From 149b0dab4d8ced2a6317e2efcaddb46008b8177f Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Tue, 23 Feb 2021 14:47:10 +0800 Subject: [PATCH] [Improvement][Docker] Sync docker conf templates to the latest conf properties (#4772) --- docker/README.md | 1 + docker/build/Dockerfile | 2 +- .../dolphinscheduler/alert.properties.tpl | 46 +++---------- .../application-api.properties.tpl | 27 ++++++-- .../dolphinscheduler/common.properties.tpl | 66 +++++++++---------- .../datasource.properties.tpl | 17 +++-- .../dolphinscheduler/logback/logback-api.xml | 4 +- .../logback/logback-master.xml | 4 +- .../logback/logback-worker.xml | 9 ++- .../dolphinscheduler/master.properties.tpl | 5 +- .../dolphinscheduler/quartz.properties.tpl | 2 +- .../dolphinscheduler/worker.properties.tpl | 14 ++-- .../dolphinscheduler/zookeeper.properties.tpl | 3 +- docker/build/startup-init-conf.sh | 27 ++------ docker/build/startup.sh | 38 +++++++++-- docker/docker-swarm/docker-compose.yml | 28 ++------ docker/docker-swarm/docker-stack.yml | 28 ++------ .../src/main/resources/alert.properties | 2 - .../main/resources/application-api.properties | 10 +-- .../src/main/resources/logback-api.xml | 4 +- .../src/main/resources/common.properties | 13 ++-- .../src/main/resources/master.properties | 6 +- .../src/main/resources/worker.properties | 2 +- .../src/main/resources/logback-zookeeper.xml | 2 +- .../src/main/resources/quartz.properties | 2 +- .../src/main/resources/zookeeper.properties | 2 +- 26 files changed, 169 insertions(+), 195 deletions(-) diff --git a/docker/README.md b/docker/README.md index e69de29bb2..dcd20981be 100644 --- a/docker/README.md +++ b/docker/README.md @@ -0,0 +1 @@ +# Dolphin Scheduler for Docker diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile index a33cf490b1..121970bf5a 100644 --- a/docker/build/Dockerfile +++ b/docker/build/Dockerfile @@ -57,6 +57,6 @@ RUN dos2unix /root/checkpoint.sh && \ echo "Set disable_coredump false" >> /etc/sudo.conf # 4. expose port -EXPOSE 5678 1234 12345 50051 +EXPOSE 5678 1234 12345 50051 50052 ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"] diff --git a/docker/build/conf/dolphinscheduler/alert.properties.tpl b/docker/build/conf/dolphinscheduler/alert.properties.tpl index 6d7450e13d..1ff6fe7adb 100644 --- a/docker/build/conf/dolphinscheduler/alert.properties.tpl +++ b/docker/build/conf/dolphinscheduler/alert.properties.tpl @@ -14,43 +14,17 @@ # See the License for the specific language governing permissions and # limitations under the License. # -#alert type is EMAIL/SMS -alert.type=EMAIL - -# alter msg template, default is html template -#alert.template=html -# mail server configuration -mail.protocol=SMTP -mail.server.host=${MAIL_SERVER_HOST} -mail.server.port=${MAIL_SERVER_PORT} -mail.sender=${MAIL_SENDER} -mail.user=${MAIL_USER} -mail.passwd=${MAIL_PASSWD} -# TLS -mail.smtp.starttls.enable=${MAIL_SMTP_STARTTLS_ENABLE} -# SSL -mail.smtp.ssl.enable=${MAIL_SMTP_SSL_ENABLE} -mail.smtp.ssl.trust=${MAIL_SMTP_SSL_TRUST} - -#xls file path,need create if not exist -xls.file.path=${XLS_FILE_PATH} - -# plugins dir -plugin.dir=${ALERT_PLUGIN_DIR} - -# Enterprise WeChat configuration -enterprise.wechat.enable=${ENTERPRISE_WECHAT_ENABLE} -enterprise.wechat.corp.id=${ENTERPRISE_WECHAT_CORP_ID} -enterprise.wechat.secret=${ENTERPRISE_WECHAT_SECRET} -enterprise.wechat.agent.id=${ENTERPRISE_WECHAT_AGENT_ID} -enterprise.wechat.users=${ENTERPRISE_WECHAT_USERS} -enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid={corpId}&corpsecret={secret} -enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={token} -enterprise.wechat.team.send.msg={\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"} -enterprise.wechat.user.send.msg={\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}} - - +#This configuration file configures the configuration parameters related to the AlertServer. +#These parameters are only related to the AlertServer, and it has nothing to do with the specific Alert Plugin. +#eg : max retry num. +#eg : Alert Server Listener port +#alert.plugin.dir config the Alert Plugin dir . AlertServer while find and load the Alert Plugin Jar from this dir when deploy and start AlertServer on the server . +alert.plugin.dir=${ALERT_PLUGIN_DIR} +#maven.local.repository=/Users/gaojun/Documents/jianguoyun/localRepository +#alert.plugin.binding config the Alert Plugin need be load when development and run in IDE +#alert.plugin.binding=\ +# ./dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/pom.xml diff --git a/docker/build/conf/dolphinscheduler/application-api.properties.tpl b/docker/build/conf/dolphinscheduler/application-api.properties.tpl index 88915923fa..f42588112b 100644 --- a/docker/build/conf/dolphinscheduler/application-api.properties.tpl +++ b/docker/build/conf/dolphinscheduler/application-api.properties.tpl @@ -24,22 +24,39 @@ server.servlet.session.timeout=7200 # servlet config server.servlet.context-path=/dolphinscheduler/ +# time zone +spring.jackson.time-zone=GMT+8 + # file size limit for upload spring.servlet.multipart.max-file-size=1024MB spring.servlet.multipart.max-request-size=1024MB +# enable response compression +server.compression.enabled=true +server.compression.mime-types=text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml + # post content -server.jetty.max-http-post-size=5000000 +server.jetty.max-http-form-post-size=5000000 # i18n spring.messages.encoding=UTF-8 -#i18n classpath folder , file prefix messages， if have many files, use "," seperator +# i18n classpath folder , file prefix messages， if have many files, use "," seperator spring.messages.basename=i18n/messages # Authentication types (supported types: PASSWORD) security.authentication.type=PASSWORD - - - +#============================================================================ +# LDAP Config +# mock ldap server from https://www.forumsys.com/tutorials/integration-how-to/ldap/online-ldap-test-server/ +#============================================================================ +# admin userId +#security.authentication.ldap.user.admin=read-only-admin +# ldap server config +#ldap.urls=ldap://ldap.forumsys.com:389/ +#ldap.base.dn=dc=example,dc=com +#ldap.username=cn=read-only-admin,dc=example,dc=com +#ldap.password=password +#ldap.user.identity.attribute=uid +#ldap.user.email.attribute=mail diff --git a/docker/build/conf/dolphinscheduler/common.properties.tpl b/docker/build/conf/dolphinscheduler/common.properties.tpl index ff74598fd4..a3ccde7c61 100644 --- a/docker/build/conf/dolphinscheduler/common.properties.tpl +++ b/docker/build/conf/dolphinscheduler/common.properties.tpl @@ -15,64 +15,64 @@ # limitations under the License. # -#============================================================================ -# System -#============================================================================ -# system env path. self configuration, please make sure the directory and file exists and have read write execute permissions -dolphinscheduler.env.path=${DOLPHINSCHEDULER_ENV_PATH} - -# user data directory path, self configuration, please make sure the directory exists and have read write permissions -data.basedir.path=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH} - -# resource upload startup type : HDFS,S3,NONE +# resource storage type : HDFS, S3, NONE resource.storage.type=${RESOURCE_STORAGE_TYPE} -#============================================================================ -# HDFS -#============================================================================ # resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended resource.upload.path=${RESOURCE_UPLOAD_PATH} +# user data local directory path, please make sure the directory exists and have read write permissions +data.basedir.path=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH} + # whether kerberos starts -#hadoop.security.authentication.startup.state=false +hadoop.security.authentication.startup.state=false # java.security.krb5.conf path -#java.security.krb5.conf.path=/opt/krb5.conf +java.security.krb5.conf.path=/opt/krb5.conf -# loginUserFromKeytab user -#login.user.keytab.username=hdfs-mycluster@ESZ.COM +# login user from keytab username +login.user.keytab.username=hdfs-mycluster@ESZ.COM -# loginUserFromKeytab path -#login.user.keytab.path=/opt/hdfs.headless.keytab +# login user from keytab path +login.user.keytab.path=/opt/hdfs.headless.keytab #resource.view.suffixs -#resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties +#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js # if resource.storage.type=HDFS, the user need to have permission to create directories under the HDFS root path hdfs.root.user=hdfs -# kerberos expire time -kerberos.expire.time=7 - -#============================================================================ -# S3 -#============================================================================ -# if resource.storage.type=S3，the value like: s3a://dolphinscheduler ; if resource.storage.type=HDFS, When namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir +# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS, When namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir fs.defaultFS=${FS_DEFAULT_FS} -# if resource.storage.type=S3，s3 endpoint +# if resource.storage.type=S3, s3 endpoint fs.s3a.endpoint=${FS_S3A_ENDPOINT} -# if resource.storage.type=S3，s3 access key +# if resource.storage.type=S3, s3 access key fs.s3a.access.key=${FS_S3A_ACCESS_KEY} -# if resource.storage.type=S3，s3 secret key +# if resource.storage.type=S3, s3 secret key fs.s3a.secret.key=${FS_S3A_SECRET_KEY} -# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty TODO +# if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx -# If resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ark1 to actual resourcemanager hostname. -yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s +# if resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname. +yarn.application.status.address=http://ds1:8088/ws/v1/cluster/apps/%s + +# job history status url when application number threshold is reached(default 10000,maybe it was set to 1000) +yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s + +# system env path, If you want to set your own path, you need to set this env file to an absolute path +dolphinscheduler.env.path=${DOLPHINSCHEDULER_ENV_PATH} +development.state=false + +# kerberos tgt expire time, unit is hours +kerberos.expire.time=2 +# datasource encryption salt +datasource.encryption.enable=false +datasource.encryption.salt=!@#$%^&* +# Network IP gets priority, default inner outer +#dolphin.scheduler.network.priority.strategy=default diff --git a/docker/build/conf/dolphinscheduler/datasource.properties.tpl b/docker/build/conf/dolphinscheduler/datasource.properties.tpl index f7c5ee6881..b414d21430 100644 --- a/docker/build/conf/dolphinscheduler/datasource.properties.tpl +++ b/docker/build/conf/dolphinscheduler/datasource.properties.tpl @@ -17,12 +17,21 @@ # db spring.datasource.driver-class-name=${DATABASE_DRIVER} -spring.datasource.url=jdbc:${DATABASE_TYPE}://${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DATABASE}?${DATABASE_PARAMS} +spring.datasource.url=jdbc:${DATABASE_TYPE}://${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DATABASE}${DATABASE_PARAMS:+?${DATABASE_PARAMS}} spring.datasource.username=${DATABASE_USERNAME} spring.datasource.password=${DATABASE_PASSWORD} -## base spring data source configuration todo need to remove -#spring.datasource.type=com.alibaba.druid.pool.DruidDataSource +# postgresql +#spring.datasource.driver-class-name=org.postgresql.Driver +#spring.datasource.url=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler +#spring.datasource.username=test +#spring.datasource.password=test + +# mysql +#spring.datasource.driver-class-name=com.mysql.jdbc.Driver +#spring.datasource.url=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 +#spring.datasource.username=xxxx +#spring.datasource.password=xxxx # connection configuration #spring.datasource.initialSize=5 @@ -63,4 +72,4 @@ spring.datasource.password=${DATABASE_PASSWORD} # open PSCache, specify count PSCache for every connection #spring.datasource.poolPreparedStatements=true -#spring.datasource.maxPoolPreparedStatementPerConnectionSize=20 \ No newline at end of file +#spring.datasource.maxPoolPreparedStatementPerConnectionSize=20 diff --git a/docker/build/conf/dolphinscheduler/logback/logback-api.xml b/docker/build/conf/dolphinscheduler/logback/logback-api.xml index 6d29f8af5f..9dcec7afce 100644 --- a/docker/build/conf/dolphinscheduler/logback/logback-api.xml +++ b/docker/build/conf/dolphinscheduler/logback/logback-api.xml @@ -23,12 +23,12 @@ - ${log.base}/dolphinscheduler-api-server.log + ${log.base}/dolphinscheduler-api.log INFO - ${log.base}/dolphinscheduler-api-server.%d{yyyy-MM-dd_HH}.%i.log + ${log.base}/dolphinscheduler-api.%d{yyyy-MM-dd_HH}.%i.log 168 64MB diff --git a/docker/build/conf/dolphinscheduler/logback/logback-master.xml b/docker/build/conf/dolphinscheduler/logback/logback-master.xml index d1bfb67aa1..202369c8dc 100644 --- a/docker/build/conf/dolphinscheduler/logback/logback-master.xml +++ b/docker/build/conf/dolphinscheduler/logback/logback-master.xml @@ -24,9 +24,9 @@ - + taskAppId diff --git a/docker/build/conf/dolphinscheduler/logback/logback-worker.xml b/docker/build/conf/dolphinscheduler/logback/logback-worker.xml index b7e08dd846..bf4dd46332 100644 --- a/docker/build/conf/dolphinscheduler/logback/logback-worker.xml +++ b/docker/build/conf/dolphinscheduler/logback/logback-worker.xml @@ -25,9 +25,9 @@ - + taskAppId @@ -48,10 +48,9 @@ ${log.base}/dolphinscheduler-worker.log - + ${log.base}/dolphinscheduler-worker.%d{yyyy-MM-dd_HH}.%i.log 168 diff --git a/docker/build/conf/dolphinscheduler/master.properties.tpl b/docker/build/conf/dolphinscheduler/master.properties.tpl index 17dd6f9d69..7b07dd7d62 100644 --- a/docker/build/conf/dolphinscheduler/master.properties.tpl +++ b/docker/build/conf/dolphinscheduler/master.properties.tpl @@ -21,6 +21,9 @@ master.exec.threads=${MASTER_EXEC_THREADS} # master execute task number in parallel master.exec.task.num=${MASTER_EXEC_TASK_NUM} +# master dispatch task number +#master.dispatch.task.num=3 + # master heartbeat interval master.heartbeat.interval=${MASTER_HEARTBEAT_INTERVAL} @@ -37,4 +40,4 @@ master.max.cpuload.avg=${MASTER_MAX_CPULOAD_AVG} master.reserved.memory=${MASTER_RESERVED_MEMORY} # master listen port -#master.listen.port=${MASTER_LISTEN_PORT} \ No newline at end of file +master.listen.port=${MASTER_LISTEN_PORT} diff --git a/docker/build/conf/dolphinscheduler/quartz.properties.tpl b/docker/build/conf/dolphinscheduler/quartz.properties.tpl index 25645795bb..10f18122bf 100644 --- a/docker/build/conf/dolphinscheduler/quartz.properties.tpl +++ b/docker/build/conf/dolphinscheduler/quartz.properties.tpl @@ -51,4 +51,4 @@ #============================================================================ # Configure Datasources #============================================================================ -#org.quartz.dataSource.myDs.connectionProvider.class = org.apache.dolphinscheduler.service.quartz.DruidConnectionProvider \ No newline at end of file +#org.quartz.dataSource.myDs.connectionProvider.class = org.apache.dolphinscheduler.service.quartz.DruidConnectionProvider diff --git a/docker/build/conf/dolphinscheduler/worker.properties.tpl b/docker/build/conf/dolphinscheduler/worker.properties.tpl index 83097dd9a4..2093da3501 100644 --- a/docker/build/conf/dolphinscheduler/worker.properties.tpl +++ b/docker/build/conf/dolphinscheduler/worker.properties.tpl @@ -21,20 +21,20 @@ worker.exec.threads=${WORKER_EXEC_THREADS} # worker heartbeat interval worker.heartbeat.interval=${WORKER_HEARTBEAT_INTERVAL} -# submit the number of tasks at a time -worker.fetch.task.num=${WORKER_FETCH_TASK_NUM} - -# only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2 +# only less than cpu avg load, worker server can work. default value -1: the number of cpu cores * 2 worker.max.cpuload.avg=${WORKER_MAX_CPULOAD_AVG} # only larger than reserved memory, worker server can work. default value : physical memory * 1/6, unit is G. worker.reserved.memory=${WORKER_RESERVED_MEMORY} # worker listener port -#worker.listen.port=${WORKER_LISTEN_PORT} +worker.listen.port=${WORKER_LISTEN_PORT} # default worker group -#worker.groups=${WORKER_GROUP} +worker.groups=${WORKER_GROUP} # default worker weight -#worker.weight=${WORKER_WEIGHT} \ No newline at end of file +worker.weight=${WORKER_WEIGHT} + +# alert server listener host +alert.listen.host=${ALERT_LISTEN_HOST} diff --git a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl b/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl index 51540aa345..8e222328f8 100644 --- a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl +++ b/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl @@ -26,4 +26,5 @@ zookeeper.dolphinscheduler.root=${ZOOKEEPER_ROOT} #zookeeper.connection.timeout=30000 #zookeeper.retry.base.sleep=100 #zookeeper.retry.max.sleep=30000 -#zookeeper.retry.maxtime=10 \ No newline at end of file +#zookeeper.retry.maxtime=10 +#zookeeper.max.wait.time=10000 diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh index c3aadcd673..9439048587 100755 --- a/docker/build/startup-init-conf.sh +++ b/docker/build/startup-init-conf.sh @@ -20,7 +20,8 @@ set -e echo "init env variables" -# Define parameters default value. +# Define parameters default value + #============================================================================ # Database Source #============================================================================ @@ -34,7 +35,7 @@ export DATABASE_DRIVER=${DATABASE_DRIVER:-"org.postgresql.Driver"} export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"} #============================================================================ -# System +# Common #============================================================================ export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"} export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"} @@ -69,35 +70,17 @@ export MASTER_LISTEN_PORT=${MASTER_LISTEN_PORT:-"5678"} #============================================================================ export WORKER_EXEC_THREADS=${WORKER_EXEC_THREADS:-"100"} export WORKER_HEARTBEAT_INTERVAL=${WORKER_HEARTBEAT_INTERVAL:-"10"} -export WORKER_FETCH_TASK_NUM=${WORKER_FETCH_TASK_NUM:-"3"} export WORKER_MAX_CPULOAD_AVG=${WORKER_MAX_CPULOAD_AVG:-"100"} export WORKER_RESERVED_MEMORY=${WORKER_RESERVED_MEMORY:-"0.1"} export WORKER_LISTEN_PORT=${WORKER_LISTEN_PORT:-"1234"} export WORKER_GROUP=${WORKER_GROUP:-"default"} export WORKER_WEIGHT=${WORKER_WEIGHT:-"100"} +export ALERT_LISTEN_HOST=${ALERT_LISTEN_HOST:-"127.0.0.1"} #============================================================================ # Alert Server #============================================================================ -# alert plugin dir -export ALERT_PLUGIN_DIR=${ALERT_PLUGIN_DIR:-"/opt/dolphinscheduler"} -# xls file -export XLS_FILE_PATH=${XLS_FILE_PATH:-"/tmp/xls"} -# mail -export MAIL_SERVER_HOST=${MAIL_SERVER_HOST:-""} -export MAIL_SERVER_PORT=${MAIL_SERVER_PORT:-""} -export MAIL_SENDER=${MAIL_SENDER:-""} -export MAIL_USER=${MAIL_USER:-""} -export MAIL_PASSWD=${MAIL_PASSWD:-""} -export MAIL_SMTP_STARTTLS_ENABLE=${MAIL_SMTP_STARTTLS_ENABLE:-"true"} -export MAIL_SMTP_SSL_ENABLE=${MAIL_SMTP_SSL_ENABLE:-"false"} -export MAIL_SMTP_SSL_TRUST=${MAIL_SMTP_SSL_TRUST:-""} -# wechat -export ENTERPRISE_WECHAT_ENABLE=${ENTERPRISE_WECHAT_ENABLE:-"false"} -export ENTERPRISE_WECHAT_CORP_ID=${ENTERPRISE_WECHAT_CORP_ID:-""} -export ENTERPRISE_WECHAT_SECRET=${ENTERPRISE_WECHAT_SECRET:-""} -export ENTERPRISE_WECHAT_AGENT_ID=${ENTERPRISE_WECHAT_AGENT_ID:-""} -export ENTERPRISE_WECHAT_USERS=${ENTERPRISE_WECHAT_USERS:-""} +export ALERT_PLUGIN_DIR=${ALERT_PLUGIN_DIR:-"lib/plugin/alert"} echo "generate app config" ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do diff --git a/docker/build/startup.sh b/docker/build/startup.sh index 8bd1895cc0..5c00c272c2 100755 --- a/docker/build/startup.sh +++ b/docker/build/startup.sh @@ -26,7 +26,7 @@ DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs waitDatabase() { echo "test ${DATABASE_TYPE} service" while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do - counter=$((counter+1)) + local counter=$((counter+1)) if [ $counter == 30 ]; then echo "Error: Couldn't connect to ${DATABASE_TYPE}." exit 1 @@ -57,12 +57,41 @@ initDatabase() { ${DOLPHINSCHEDULER_SCRIPT}/create-dolphinscheduler.sh } +# check ds version +checkDSVersion() { + if [ ${DATABASE_TYPE} = "mysql" ]; then + v=$(mysql -h${DATABASE_HOST} -P${DATABASE_PORT} -u${DATABASE_USERNAME} --password=${DATABASE_PASSWORD} -D ${DATABASE_DATABASE} -e "SELECT * FROM public.t_ds_version" 2>/dev/null) + else + v=$(PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "SELECT * FROM public.t_ds_version" 2>/dev/null) + fi + if [ -n "$v" ]; then + echo "ds version: $v" + return 0 + else + return 1 + fi +} + +# check init database +checkInitDatabase() { + echo "check init database" + while ! checkDSVersion; do + local counter=$((counter+1)) + if [ $counter == 30 ]; then + echo "Error: Couldn't check init database." + exit 1 + fi + echo "Trying to check init database. Attempt $counter." + sleep 5 + done +} + # wait zk waitZK() { echo "connect remote zookeeper" echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do while ! nc -z ${line%:*} ${line#*:}; do - counter=$((counter+1)) + local counter=$((counter+1)) if [ $counter == 30 ]; then echo "Error: Couldn't connect to zookeeper." exit 1 @@ -133,7 +162,7 @@ case "$1" in initApiServer initAlertServer initLoggerServer - LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log + LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api.log ;; (master-server) waitZK @@ -153,10 +182,11 @@ case "$1" in waitDatabase initDatabase initApiServer - LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log + LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api.log ;; (alert-server) waitDatabase + checkInitDatabase initAlertServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-alert.log ;; diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index 04a9bc556d..2360c4abed 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -87,22 +87,11 @@ services: image: apache/dolphinscheduler:latest container_name: dolphinscheduler-alert command: alert-server + ports: + - 50052:50052 environment: TZ: Asia/Shanghai - XLS_FILE_PATH: "/tmp/xls" - MAIL_SERVER_HOST: "" - MAIL_SERVER_PORT: "" - MAIL_SENDER: "" - MAIL_USER: "" - MAIL_PASSWD: "" - MAIL_SMTP_STARTTLS_ENABLE: "false" - MAIL_SMTP_SSL_ENABLE: "false" - MAIL_SMTP_SSL_TRUST: "" - ENTERPRISE_WECHAT_ENABLE: "false" - ENTERPRISE_WECHAT_CORP_ID: "" - ENTERPRISE_WECHAT_SECRET: "" - ENTERPRISE_WECHAT_AGENT_ID: "" - ENTERPRISE_WECHAT_USERS: "" + ALERT_PLUGIN_DIR: lib/plugin/alert DATABASE_HOST: dolphinscheduler-postgresql DATABASE_PORT: 5432 DATABASE_USERNAME: root @@ -169,21 +158,12 @@ services: TZ: Asia/Shanghai WORKER_EXEC_THREADS: "100" WORKER_HEARTBEAT_INTERVAL: "10" - WORKER_FETCH_TASK_NUM: "3" WORKER_MAX_CPULOAD_AVG: "100" WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUP: "default" WORKER_WEIGHT: "100" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler - XLS_FILE_PATH: "/tmp/xls" - MAIL_SERVER_HOST: "" - MAIL_SERVER_PORT: "" - MAIL_SENDER: "" - MAIL_USER: "" - MAIL_PASSWD: "" - MAIL_SMTP_STARTTLS_ENABLE: "false" - MAIL_SMTP_SSL_ENABLE: "false" - MAIL_SMTP_SSL_TRUST: "" + ALERT_LISTEN_HOST: dolphinscheduler-alert DATABASE_HOST: dolphinscheduler-postgresql DATABASE_PORT: 5432 DATABASE_USERNAME: root diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml index 39c36347bb..7035ac3bda 100644 --- a/docker/docker-swarm/docker-stack.yml +++ b/docker/docker-swarm/docker-stack.yml @@ -84,22 +84,11 @@ services: dolphinscheduler-alert: image: apache/dolphinscheduler:latest command: alert-server + ports: + - 50052:50052 environment: TZ: Asia/Shanghai - XLS_FILE_PATH: "/tmp/xls" - MAIL_SERVER_HOST: "" - MAIL_SERVER_PORT: "" - MAIL_SENDER: "" - MAIL_USER: "" - MAIL_PASSWD: "" - MAIL_SMTP_STARTTLS_ENABLE: "false" - MAIL_SMTP_SSL_ENABLE: "false" - MAIL_SMTP_SSL_TRUST: "" - ENTERPRISE_WECHAT_ENABLE: "false" - ENTERPRISE_WECHAT_CORP_ID: "" - ENTERPRISE_WECHAT_SECRET: "" - ENTERPRISE_WECHAT_AGENT_ID: "" - ENTERPRISE_WECHAT_USERS: "" + ALERT_PLUGIN_DIR: lib/plugin/alert DATABASE_HOST: dolphinscheduler-postgresql DATABASE_PORT: 5432 DATABASE_USERNAME: root @@ -163,21 +152,12 @@ services: TZ: Asia/Shanghai WORKER_EXEC_THREADS: "100" WORKER_HEARTBEAT_INTERVAL: "10" - WORKER_FETCH_TASK_NUM: "3" WORKER_MAX_CPULOAD_AVG: "100" WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUP: "default" WORKER_WEIGHT: "100" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler - XLS_FILE_PATH: "/tmp/xls" - MAIL_SERVER_HOST: "" - MAIL_SERVER_PORT: "" - MAIL_SENDER: "" - MAIL_USER: "" - MAIL_PASSWD: "" - MAIL_SMTP_STARTTLS_ENABLE: "false" - MAIL_SMTP_SSL_ENABLE: "false" - MAIL_SMTP_SSL_TRUST: "" + ALERT_LISTEN_HOST: dolphinscheduler-alert DATABASE_HOST: dolphinscheduler-postgresql DATABASE_PORT: 5432 DATABASE_USERNAME: root diff --git a/dolphinscheduler-alert/src/main/resources/alert.properties b/dolphinscheduler-alert/src/main/resources/alert.properties index 80ea87c2cd..33184cadf2 100644 --- a/dolphinscheduler-alert/src/main/resources/alert.properties +++ b/dolphinscheduler-alert/src/main/resources/alert.properties @@ -28,5 +28,3 @@ alert.plugin.dir=./lib/plugin/alert #alert.plugin.binding config the Alert Plugin need be load when development and run in IDE #alert.plugin.binding=\ # ./dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/pom.xml - - diff --git a/dolphinscheduler-api/src/main/resources/application-api.properties b/dolphinscheduler-api/src/main/resources/application-api.properties index e2cabfac67..f42588112b 100644 --- a/dolphinscheduler-api/src/main/resources/application-api.properties +++ b/dolphinscheduler-api/src/main/resources/application-api.properties @@ -21,9 +21,10 @@ server.port=12345 # session config server.servlet.session.timeout=7200 +# servlet config server.servlet.context-path=/dolphinscheduler/ -# Set time zone +# time zone spring.jackson.time-zone=GMT+8 # file size limit for upload @@ -34,12 +35,13 @@ spring.servlet.multipart.max-request-size=1024MB server.compression.enabled=true server.compression.mime-types=text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml -#post content +# post content server.jetty.max-http-form-post-size=5000000 +# i18n spring.messages.encoding=UTF-8 -#i18n classpath folder , file prefix messages， if have many files, use "," seperator +# i18n classpath folder , file prefix messages， if have many files, use "," seperator spring.messages.basename=i18n/messages # Authentication types (supported types: PASSWORD) @@ -58,5 +60,3 @@ security.authentication.type=PASSWORD #ldap.password=password #ldap.user.identity.attribute=uid #ldap.user.email.attribute=mail - - diff --git a/dolphinscheduler-api/src/main/resources/logback-api.xml b/dolphinscheduler-api/src/main/resources/logback-api.xml index e5cb37afac..0f06b9cff5 100644 --- a/dolphinscheduler-api/src/main/resources/logback-api.xml +++ b/dolphinscheduler-api/src/main/resources/logback-api.xml @@ -31,12 +31,12 @@ - ${log.base}/dolphinscheduler-api-server.log + ${log.base}/dolphinscheduler-api.log INFO - ${log.base}/dolphinscheduler-api-server.%d{yyyy-MM-dd_HH}.%i.log + ${log.base}/dolphinscheduler-api.%d{yyyy-MM-dd_HH}.%i.log 168 64MB diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index ed9a02739d..833e0491e2 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -15,7 +15,7 @@ # limitations under the License. # -# resource storage type : HDFS,S3,NONE +# resource storage type : HDFS, S3, NONE resource.storage.type=NONE # resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended @@ -33,7 +33,7 @@ java.security.krb5.conf.path=/opt/krb5.conf # login user from keytab username login.user.keytab.username=hdfs-mycluster@ESZ.COM -# loginUserFromKeytab path +# login user from keytab path login.user.keytab.path=/opt/hdfs.headless.keytab #resource.view.suffixs @@ -42,16 +42,16 @@ login.user.keytab.path=/opt/hdfs.headless.keytab # if resource.storage.type=HDFS, the user need to have permission to create directories under the HDFS root path hdfs.root.user=hdfs -# if resource.storage.type=S3,the value like: s3a://dolphinscheduler ; if resource.storage.type=HDFS, When namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir +# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS, When namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir fs.defaultFS=hdfs://mycluster:8020 -# if resource.storage.type=S3，s3 endpoint +# if resource.storage.type=S3, s3 endpoint fs.s3a.endpoint=http://192.168.xx.xx:9010 -# if resource.storage.type=S3，s3 access key +# if resource.storage.type=S3, s3 access key fs.s3a.access.key=A3DXS30FO22544RE -# if resource.storage.type=S3，s3 secret key +# if resource.storage.type=S3, s3 secret key fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK # if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty @@ -59,6 +59,7 @@ yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx # if resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname. yarn.application.status.address=http://ds1:8088/ws/v1/cluster/apps/%s + # job history status url when application number threshold is reached(default 10000,maybe it was set to 1000) yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s diff --git a/dolphinscheduler-server/src/main/resources/master.properties b/dolphinscheduler-server/src/main/resources/master.properties index 44301fb54e..01d1189507 100644 --- a/dolphinscheduler-server/src/main/resources/master.properties +++ b/dolphinscheduler-server/src/main/resources/master.properties @@ -21,9 +21,8 @@ # master execute task number in parallel #master.exec.task.num=20 - # master dispatch task number -#master.dispatch.task.num = 3 +#master.dispatch.task.num=3 # master heartbeat interval #master.heartbeat.interval=10 @@ -34,7 +33,6 @@ # master commit task interval #master.task.commit.interval=1000 - # only less than cpu avg load, master server can work. default value -1 : the number of cpu cores * 2 #master.max.cpuload.avg=-1 @@ -42,4 +40,4 @@ #master.reserved.memory=0.3 # master listen port -#master.listen.port=5678 \ No newline at end of file +#master.listen.port=5678 diff --git a/dolphinscheduler-server/src/main/resources/worker.properties b/dolphinscheduler-server/src/main/resources/worker.properties index a1236a36bd..5fdbf1d910 100644 --- a/dolphinscheduler-server/src/main/resources/worker.properties +++ b/dolphinscheduler-server/src/main/resources/worker.properties @@ -22,7 +22,7 @@ #worker.heartbeat.interval=10 # only less than cpu avg load, worker server can work. default value -1: the number of cpu cores * 2 -#worker.max.cpuload.avg= -1 +#worker.max.cpuload.avg=-1 # only larger than reserved memory, worker server can work. default value : physical memory * 1/6, unit is G. #worker.reserved.memory=0.3 diff --git a/dolphinscheduler-service/src/main/resources/logback-zookeeper.xml b/dolphinscheduler-service/src/main/resources/logback-zookeeper.xml index 34a15a7a5b..9bdf18f8cd 100644 --- a/dolphinscheduler-service/src/main/resources/logback-zookeeper.xml +++ b/dolphinscheduler-service/src/main/resources/logback-zookeeper.xml @@ -32,7 +32,7 @@ ${log.base}/dolphinscheduler-zookeeper.log - ${log.base}/dolphinscheduler-alert.%d{yyyy-MM-dd_HH}.%i.log + ${log.base}/dolphinscheduler-zookeeper.%d{yyyy-MM-dd_HH}.%i.log 20 64MB diff --git a/dolphinscheduler-service/src/main/resources/quartz.properties b/dolphinscheduler-service/src/main/resources/quartz.properties index 6e208f62d4..93ee71c6a3 100644 --- a/dolphinscheduler-service/src/main/resources/quartz.properties +++ b/dolphinscheduler-service/src/main/resources/quartz.properties @@ -51,4 +51,4 @@ #============================================================================ # Configure Datasources #============================================================================ -#org.quartz.dataSource.myDs.connectionProvider.class = org.apache.dolphinscheduler.service.quartz.DruidConnectionProvider \ No newline at end of file +#org.quartz.dataSource.myDs.connectionProvider.class = org.apache.dolphinscheduler.service.quartz.DruidConnectionProvider diff --git a/dolphinscheduler-service/src/main/resources/zookeeper.properties b/dolphinscheduler-service/src/main/resources/zookeeper.properties index 31f5f43c42..006b12036d 100644 --- a/dolphinscheduler-service/src/main/resources/zookeeper.properties +++ b/dolphinscheduler-service/src/main/resources/zookeeper.properties @@ -27,4 +27,4 @@ zookeeper.quorum=localhost:2181 #zookeeper.retry.base.sleep=100 #zookeeper.retry.max.sleep=30000 #zookeeper.retry.maxtime=10 -#zookeeper.max.wait.time=10000 \ No newline at end of file +#zookeeper.max.wait.time=10000