diff --git a/.licenserc.yaml b/.licenserc.yaml index ea6909e240..8f69da5608 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -34,6 +34,7 @@ header: - '**/*.md' - '**/*.json' - '**/*.iml' + - '**/*.ini' - '**/.babelrc' - '**/.eslintignore' - '**/.gitignore' diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile index eeea20d482..97a99e6df0 100644 --- a/docker/build/Dockerfile +++ b/docker/build/Dockerfile @@ -28,7 +28,7 @@ ENV DOCKER true # RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories # RUN sed -i 's/dl-cdn.alpinelinux.org/mirror.tuna.tsinghua.edu.cn/g' /etc/apk/repositories RUN apk update && \ - apk add --no-cache tzdata dos2unix bash python2 python3 procps sudo shadow tini postgresql-client && \ + apk add --no-cache tzdata dos2unix bash python2 python3 supervisor procps sudo shadow tini postgresql-client && \ cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ apk del tzdata && \ rm -rf /var/cache/apk/* @@ -44,6 +44,7 @@ COPY ./startup-init-conf.sh /root/startup-init-conf.sh COPY ./startup.sh /root/startup.sh COPY ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/ COPY ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/ +COPY ./conf/dolphinscheduler/supervisor/supervisor.ini /etc/supervisor.d/ COPY ./conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl /opt/dolphinscheduler/conf/env/ RUN dos2unix /root/checkpoint.sh && \ dos2unix /root/startup-init-conf.sh && \ diff --git a/docker/build/conf/dolphinscheduler/logback/logback-alert.xml b/docker/build/conf/dolphinscheduler/logback/logback-alert.xml index eec78385db..1718947dd1 100644 --- a/docker/build/conf/dolphinscheduler/logback/logback-alert.xml +++ b/docker/build/conf/dolphinscheduler/logback/logback-alert.xml @@ -20,6 +20,14 @@ + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n + + UTF-8 + + ${log.base}/dolphinscheduler-alert.log @@ -37,6 +45,7 @@ + diff --git a/docker/build/conf/dolphinscheduler/logback/logback-api.xml b/docker/build/conf/dolphinscheduler/logback/logback-api.xml index 9dcec7afce..bec3d9aa49 100644 --- a/docker/build/conf/dolphinscheduler/logback/logback-api.xml +++ b/docker/build/conf/dolphinscheduler/logback/logback-api.xml @@ -20,6 +20,14 @@ + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n + + UTF-8 + + @@ -47,6 +55,7 @@ + diff --git a/docker/build/conf/dolphinscheduler/logback/logback-master.xml b/docker/build/conf/dolphinscheduler/logback/logback-master.xml index 202369c8dc..f0d2c81df2 100644 --- a/docker/build/conf/dolphinscheduler/logback/logback-master.xml +++ b/docker/build/conf/dolphinscheduler/logback/logback-master.xml @@ -20,6 +20,14 @@ + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n + + UTF-8 + + @@ -66,6 +74,7 @@ + diff --git a/docker/build/conf/dolphinscheduler/logback/logback-worker.xml b/docker/build/conf/dolphinscheduler/logback/logback-worker.xml index bf4dd46332..7127219873 100644 --- a/docker/build/conf/dolphinscheduler/logback/logback-worker.xml +++ b/docker/build/conf/dolphinscheduler/logback/logback-worker.xml @@ -20,6 +20,14 @@ + + + + [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n + + UTF-8 + + + diff --git a/docker/build/conf/dolphinscheduler/supervisor/supervisor.ini b/docker/build/conf/dolphinscheduler/supervisor/supervisor.ini new file mode 100644 index 0000000000..c8c4e126c2 --- /dev/null +++ b/docker/build/conf/dolphinscheduler/supervisor/supervisor.ini @@ -0,0 +1,92 @@ +; Licensed to the Apache Software Foundation (ASF) under one +; or more contributor license agreements. See the NOTICE file +; distributed with this work for additional information +; regarding copyright ownership. The ASF licenses this file +; to you under the Apache License, Version 2.0 (the +; "License"); you may not use this file except in compliance +; with the License. You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, software +; distributed under the License is distributed on an "AS IS" BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions and +; limitations under the License. + +; program config file + +[program:master] +command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start master-server +directory=%(ENV_DOLPHINSCHEDULER_HOME)s +priority=999 +autostart=%(ENV_MASTER_START_ENABLED)s +autorestart=true +startsecs=10 +stopwaitsecs=3 +exitcodes=0 +stopasgroup=true +killasgroup=true +redirect_stderr=true +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 + +[program:worker] +command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start worker-server +directory=%(ENV_DOLPHINSCHEDULER_HOME)s +priority=999 +autostart=%(ENV_WORKER_START_ENABLED)s +autorestart=true +startsecs=10 +stopwaitsecs=3 +exitcodes=0 +stopasgroup=true +killasgroup=true +redirect_stderr=true +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 + +[program:api] +command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start api-server +directory=%(ENV_DOLPHINSCHEDULER_HOME)s +priority=999 +autostart=%(ENV_API_START_ENABLED)s +autorestart=true +startsecs=10 +stopwaitsecs=3 +exitcodes=0 +stopasgroup=true +killasgroup=true +redirect_stderr=true +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 + +[program:alert] +command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start alert-server +directory=%(ENV_DOLPHINSCHEDULER_HOME)s +priority=999 +autostart=%(ENV_ALERT_START_ENABLED)s +autorestart=true +startsecs=5 +stopwaitsecs=3 +exitcodes=0 +stopasgroup=true +killasgroup=true +redirect_stderr=true +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 + +[program:logger] +command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start logger-server +directory=%(ENV_DOLPHINSCHEDULER_HOME)s +priority=999 +autostart=%(ENV_LOGGER_START_ENABLED)s +autorestart=true +startsecs=5 +stopwaitsecs=3 +exitcodes=0 +stopasgroup=true +killasgroup=true +redirect_stderr=true +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh index 89be05eaf6..c4109dd0f2 100755 --- a/docker/build/startup-init-conf.sh +++ b/docker/build/startup-init-conf.sh @@ -92,11 +92,20 @@ export ALERT_LISTEN_HOST=${ALERT_LISTEN_HOST:-"127.0.0.1"} #============================================================================ export ALERT_PLUGIN_DIR=${ALERT_PLUGIN_DIR:-"lib/plugin/alert"} -echo "generate app config" -find ${DOLPHINSCHEDULER_HOME}/conf/ -name "*.tpl" | while read file; do +echo "generate dolphinscheduler config" +ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do eval "cat << EOF -$(cat ${file}) +$(cat ${DOLPHINSCHEDULER_HOME}/conf/${line}) EOF -" > ${file%.*} +" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*} done -find ${DOLPHINSCHEDULER_HOME}/conf/ -name "*.sh" -exec chmod +x {} \; + +# generate dolphinscheduler env only in docker +DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh +if [ -z "${KUBERNETES_SERVICE_HOST}" ] && [ -r "${DOLPHINSCHEDULER_ENV_PATH}.tpl" ]; then +eval "cat << EOF +$(cat ${DOLPHINSCHEDULER_ENV_PATH}.tpl) +EOF +" > ${DOLPHINSCHEDULER_ENV_PATH} +chmod +x ${DOLPHINSCHEDULER_ENV_PATH} +fi diff --git a/docker/build/startup.sh b/docker/build/startup.sh index 5c00c272c2..934fac3ae7 100755 --- a/docker/build/startup.sh +++ b/docker/build/startup.sh @@ -18,9 +18,12 @@ set -e -DOLPHINSCHEDULER_BIN=${DOLPHINSCHEDULER_HOME}/bin -DOLPHINSCHEDULER_SCRIPT=${DOLPHINSCHEDULER_HOME}/script -DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs +export DOLPHINSCHEDULER_BIN=${DOLPHINSCHEDULER_HOME}/bin +export MASTER_START_ENABLED=false +export WORKER_START_ENABLED=false +export API_START_ENABLED=false +export ALERT_START_ENABLED=false +export LOGGER_START_ENABLED=false # wait database waitDatabase() { @@ -54,7 +57,7 @@ waitDatabase() { # init database initDatabase() { echo "import sql data" - ${DOLPHINSCHEDULER_SCRIPT}/create-dolphinscheduler.sh + ${DOLPHINSCHEDULER_HOME}/script/create-dolphinscheduler.sh } # check ds version @@ -102,41 +105,6 @@ waitZK() { done } -# start master-server -initMasterServer() { - echo "start master-server" - ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh stop master-server - ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh start master-server -} - -# start worker-server -initWorkerServer() { - echo "start worker-server" - ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh stop worker-server - ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh start worker-server -} - -# start api-server -initApiServer() { - echo "start api-server" - ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh stop api-server - ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh start api-server -} - -# start logger-server -initLoggerServer() { - echo "start logger-server" - ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh stop logger-server - ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh start logger-server -} - -# start alert-server -initAlertServer() { - echo "start alert-server" - ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh stop alert-server - ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh start alert-server -} - # print usage printUsage() { echo -e "Dolphin Scheduler is a distributed and easy-to-expand visual DAG workflow scheduling system," @@ -157,38 +125,33 @@ case "$1" in waitZK waitDatabase initDatabase - initMasterServer - initWorkerServer - initApiServer - initAlertServer - initLoggerServer - LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api.log + export MASTER_START_ENABLED=true + export WORKER_START_ENABLED=true + export API_START_ENABLED=true + export ALERT_START_ENABLED=true + export LOGGER_START_ENABLED=true ;; (master-server) waitZK waitDatabase - initMasterServer - LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-master.log + export MASTER_START_ENABLED=true ;; (worker-server) waitZK waitDatabase - initWorkerServer - initLoggerServer - LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log + export WORKER_START_ENABLED=true + export LOGGER_START_ENABLED=true ;; (api-server) waitZK waitDatabase initDatabase - initApiServer - LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api.log + export API_START_ENABLED=true ;; (alert-server) waitDatabase checkInitDatabase - initAlertServer - LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-alert.log + export ALERT_START_ENABLED=true ;; (help) printUsage @@ -200,8 +163,8 @@ case "$1" in ;; esac -# init directories and log files -mkdir -p ${DOLPHINSCHEDULER_LOGS} && cat /dev/null >> ${LOGFILE} +# init directories +mkdir -p ${DOLPHINSCHEDULER_HOME}/logs -echo "tail begin" -exec bash -c "tail -n 1 -f ${LOGFILE}" +# start supervisord +supervisord -n -u root diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index a87e62e187..08aaa14d02 100755 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -54,42 +54,36 @@ fi log=$DOLPHINSCHEDULER_LOG_DIR/dolphinscheduler-$command-$HOSTNAME.out pid=$DOLPHINSCHEDULER_PID_DIR/dolphinscheduler-$command.pid -# print logs to /dev/null in docker -if [ "$DOCKER" = "true" ]; then - echo "start in docker" - log=/dev/null -fi - cd $DOLPHINSCHEDULER_HOME if [ "$command" = "api-server" ]; then HEAP_INITIAL_SIZE=1g HEAP_MAX_SIZE=1g - HEAP_NEW_GENERATION__SIZE=500m + HEAP_NEW_GENERATION_SIZE=512m LOG_FILE="-Dlogging.config=classpath:logback-api.xml -Dspring.profiles.active=api" CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer elif [ "$command" = "master-server" ]; then HEAP_INITIAL_SIZE=4g HEAP_MAX_SIZE=4g - HEAP_NEW_GENERATION__SIZE=2g + HEAP_NEW_GENERATION_SIZE=2g LOG_FILE="-Dlogging.config=classpath:logback-master.xml -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.server.master.MasterServer elif [ "$command" = "worker-server" ]; then HEAP_INITIAL_SIZE=2g HEAP_MAX_SIZE=2g - HEAP_NEW_GENERATION__SIZE=1g + HEAP_NEW_GENERATION_SIZE=1g LOG_FILE="-Dlogging.config=classpath:logback-worker.xml -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer elif [ "$command" = "alert-server" ]; then HEAP_INITIAL_SIZE=1g HEAP_MAX_SIZE=1g - HEAP_NEW_GENERATION__SIZE=500m + HEAP_NEW_GENERATION_SIZE=512m LOG_FILE="-Dlogback.configurationFile=conf/logback-alert.xml" CLASS=org.apache.dolphinscheduler.alert.AlertServer elif [ "$command" = "logger-server" ]; then HEAP_INITIAL_SIZE=1g HEAP_MAX_SIZE=1g - HEAP_NEW_GENERATION__SIZE=500m + HEAP_NEW_GENERATION_SIZE=512m CLASS=org.apache.dolphinscheduler.server.log.LoggerServer elif [ "$command" = "zookeeper-server" ]; then #note: this command just for getting a quick experience，not recommended for production. this operation will start a standalone zookeeper server @@ -100,26 +94,29 @@ else exit 1 fi -export DOLPHINSCHEDULER_OPTS="-server -Xms$HEAP_INITIAL_SIZE -Xmx$HEAP_MAX_SIZE -Xmn$HEAP_NEW_GENERATION__SIZE -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof $DOLPHINSCHEDULER_OPTS" +export DOLPHINSCHEDULER_OPTS="-server -Xms$HEAP_INITIAL_SIZE -Xmx$HEAP_MAX_SIZE -Xmn$HEAP_NEW_GENERATION_SIZE -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:$DOLPHINSCHEDULER_LOG_DIR/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof $DOLPHINSCHEDULER_OPTS" case $startStop in (start) - [ -w "$DOLPHINSCHEDULER_PID_DIR" ] || mkdir -p "$DOLPHINSCHEDULER_PID_DIR" + exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS" + if [ "$DOCKER" = "true" ]; then + echo "start in docker" + $JAVA_HOME/bin/java $exec_command + else + [ -w "$DOLPHINSCHEDULER_PID_DIR" ] || mkdir -p "$DOLPHINSCHEDULER_PID_DIR" - if [ -f $pid ]; then - if kill -0 `cat $pid` > /dev/null 2>&1; then - echo $command running as process `cat $pid`. Stop it first. - exit 1 + if [ -f $pid ]; then + if kill -0 `cat $pid` > /dev/null 2>&1; then + echo $command running as process `cat $pid`. Stop it first. + exit 1 + fi fi - fi - - echo starting $command, logging to $log - exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS" - - echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &" - nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 & - echo $! > $pid + echo starting $command, logging to $log + echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &" + nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 & + echo $! > $pid + fi ;; (stop)