Merge pull request #4900 from chengshiwen/improve-docker-supervisor-and-log

[Fix-4897][Docker] Support supervisor management and fix log stdout

.licenserc.yaml

@@ -34,6 +34,7 @@ header:
     - '**/*.md'
     - '**/*.json'
     - '**/*.iml'
+    - '**/*.ini'
     - '**/.babelrc'
     - '**/.eslintignore'
     - '**/.gitignore'

docker/build/Dockerfile

@@ -28,7 +28,7 @@ ENV DOCKER true
 # RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories
 # RUN sed -i 's/dl-cdn.alpinelinux.org/mirror.tuna.tsinghua.edu.cn/g' /etc/apk/repositories
 RUN apk update && \
-    apk add --no-cache tzdata dos2unix bash python2 python3 procps sudo shadow tini postgresql-client && \
+    apk add --no-cache tzdata dos2unix bash python2 python3 supervisor procps sudo shadow tini postgresql-client && \
     cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
     apk del tzdata && \
     rm -rf /var/cache/apk/*
@@ -44,6 +44,7 @@ COPY ./startup-init-conf.sh /root/startup-init-conf.sh
 COPY ./startup.sh /root/startup.sh
 COPY ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/
 COPY ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/
+COPY ./conf/dolphinscheduler/supervisor/supervisor.ini /etc/supervisor.d/
 COPY ./conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl /opt/dolphinscheduler/conf/env/
 RUN dos2unix /root/checkpoint.sh && \
     dos2unix /root/startup-init-conf.sh && \

docker/build/conf/dolphinscheduler/logback/logback-alert.xml

@@ -20,6 +20,14 @@
 <configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
 
     <property name="log.base" value="logs"/>
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>
+                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+            </pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>
 
     <appender name="ALERTLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
         <file>${log.base}/dolphinscheduler-alert.log</file>
@@ -37,6 +45,7 @@
     </appender>
 
     <root level="INFO">
+        <appender-ref ref="STDOUT"/>
         <appender-ref ref="ALERTLOGFILE"/>
     </root>
 

docker/build/conf/dolphinscheduler/logback/logback-api.xml

@@ -20,6 +20,14 @@
 <configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
 
     <property name="log.base" value="logs"/>
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>
+                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+            </pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>
 
     <!-- api server logback config start -->
     <appender name="APILOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
@@ -47,6 +55,7 @@
 
 
     <root level="INFO">
+        <appender-ref ref="STDOUT"/>
         <appender-ref ref="APILOGFILE"/>
     </root>
 

docker/build/conf/dolphinscheduler/logback/logback-master.xml

@@ -20,6 +20,14 @@
 <configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
 
     <property name="log.base" value="logs"/>
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>
+                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+            </pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>
 
     <conversionRule conversionWord="messsage"
                     converterClass="org.apache.dolphinscheduler.server.log.SensitiveDataConverter"/>
@@ -66,6 +74,7 @@
     <!-- master server logback config end -->
 
     <root level="INFO">
+        <appender-ref ref="STDOUT"/>
         <appender-ref ref="TASKLOGFILE"/>
         <appender-ref ref="MASTERLOGFILE"/>
     </root>

docker/build/conf/dolphinscheduler/logback/logback-worker.xml

@@ -20,6 +20,14 @@
 <configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
 
     <property name="log.base" value="logs"/>
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>
+                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+            </pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>
 
     <!-- worker server logback config start -->
     <conversionRule conversionWord="messsage"
@@ -66,6 +74,7 @@
     <!-- worker server logback config end -->
 
     <root level="INFO">
+        <appender-ref ref="STDOUT"/>
         <appender-ref ref="TASKLOGFILE"/>
         <appender-ref ref="WORKERLOGFILE"/>
     </root>

docker/build/conf/dolphinscheduler/supervisor/supervisor.ini

@@ -0,0 +1,92 @@
+; Licensed to the Apache Software Foundation (ASF) under one
+; or more contributor license agreements.  See the NOTICE file
+; distributed with this work for additional information
+; regarding copyright ownership.  The ASF licenses this file
+; to you under the Apache License, Version 2.0 (the
+; "License"); you may not use this file except in compliance
+; with the License.  You may obtain a copy of the License at
+;
+;     http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing, software
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
+
+; program config file
+
+[program:master]
+command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start master-server
+directory=%(ENV_DOLPHINSCHEDULER_HOME)s
+priority=999
+autostart=%(ENV_MASTER_START_ENABLED)s
+autorestart=true
+startsecs=10
+stopwaitsecs=3
+exitcodes=0
+stopasgroup=true
+killasgroup=true
+redirect_stderr=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+
+[program:worker]
+command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start worker-server
+directory=%(ENV_DOLPHINSCHEDULER_HOME)s
+priority=999
+autostart=%(ENV_WORKER_START_ENABLED)s
+autorestart=true
+startsecs=10
+stopwaitsecs=3
+exitcodes=0
+stopasgroup=true
+killasgroup=true
+redirect_stderr=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+
+[program:api]
+command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start api-server
+directory=%(ENV_DOLPHINSCHEDULER_HOME)s
+priority=999
+autostart=%(ENV_API_START_ENABLED)s
+autorestart=true
+startsecs=10
+stopwaitsecs=3
+exitcodes=0
+stopasgroup=true
+killasgroup=true
+redirect_stderr=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+
+[program:alert]
+command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start alert-server
+directory=%(ENV_DOLPHINSCHEDULER_HOME)s
+priority=999
+autostart=%(ENV_ALERT_START_ENABLED)s
+autorestart=true
+startsecs=5
+stopwaitsecs=3
+exitcodes=0
+stopasgroup=true
+killasgroup=true
+redirect_stderr=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+
+[program:logger]
+command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start logger-server
+directory=%(ENV_DOLPHINSCHEDULER_HOME)s
+priority=999
+autostart=%(ENV_LOGGER_START_ENABLED)s
+autorestart=true
+startsecs=5
+stopwaitsecs=3
+exitcodes=0
+stopasgroup=true
+killasgroup=true
+redirect_stderr=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0

docker/build/startup-init-conf.sh

@@ -92,11 +92,20 @@ export ALERT_LISTEN_HOST=${ALERT_LISTEN_HOST:-"127.0.0.1"}
 #============================================================================
 export ALERT_PLUGIN_DIR=${ALERT_PLUGIN_DIR:-"lib/plugin/alert"}
 
-echo "generate app config"
+echo "generate dolphinscheduler config"
-find ${DOLPHINSCHEDULER_HOME}/conf/ -name "*.tpl" | while read file; do
+ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do
 eval "cat << EOF
-$(cat ${file})
+$(cat ${DOLPHINSCHEDULER_HOME}/conf/${line})
 EOF
-" > ${file%.*}
+" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*}
 done
-find ${DOLPHINSCHEDULER_HOME}/conf/ -name "*.sh" -exec chmod +x {} \;
+
+# generate dolphinscheduler env only in docker
+DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh
+if [ -z "${KUBERNETES_SERVICE_HOST}" ] && [ -r "${DOLPHINSCHEDULER_ENV_PATH}.tpl" ]; then
+eval "cat << EOF
+$(cat ${DOLPHINSCHEDULER_ENV_PATH}.tpl)
+EOF
+" > ${DOLPHINSCHEDULER_ENV_PATH}
+chmod +x ${DOLPHINSCHEDULER_ENV_PATH}
+fi

docker/build/startup.sh

@@ -18,9 +18,12 @@
 
 set -e
 
-DOLPHINSCHEDULER_BIN=${DOLPHINSCHEDULER_HOME}/bin
+export DOLPHINSCHEDULER_BIN=${DOLPHINSCHEDULER_HOME}/bin
-DOLPHINSCHEDULER_SCRIPT=${DOLPHINSCHEDULER_HOME}/script
+export MASTER_START_ENABLED=false
-DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs
+export WORKER_START_ENABLED=false
+export API_START_ENABLED=false
+export ALERT_START_ENABLED=false
+export LOGGER_START_ENABLED=false
 
 # wait database
 waitDatabase() {
@@ -54,7 +57,7 @@ waitDatabase() {
 # init database
 initDatabase() {
     echo "import sql data"
-    ${DOLPHINSCHEDULER_SCRIPT}/create-dolphinscheduler.sh
+    ${DOLPHINSCHEDULER_HOME}/script/create-dolphinscheduler.sh
 }
 
 # check ds version
@@ -102,41 +105,6 @@ waitZK() {
     done
 }
 
-# start master-server
-initMasterServer() {
-    echo "start master-server"
-    ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh stop master-server
-    ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh start master-server
-}
-
-# start worker-server
-initWorkerServer() {
-    echo "start worker-server"
-    ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh stop worker-server
-    ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh start worker-server
-}
-
-# start api-server
-initApiServer() {
-    echo "start api-server"
-    ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh stop api-server
-    ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh start api-server
-}
-
-# start logger-server
-initLoggerServer() {
-    echo "start logger-server"
-    ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh stop logger-server
-    ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh start logger-server
-}
-
-# start alert-server
-initAlertServer() {
-    echo "start alert-server"
-    ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh stop alert-server
-    ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh start alert-server
-}
-
 # print usage
 printUsage() {
     echo -e "Dolphin Scheduler is a distributed and easy-to-expand visual DAG workflow scheduling system,"
@@ -157,38 +125,33 @@ case "$1" in
         waitZK
         waitDatabase
         initDatabase
-        initMasterServer
+        export MASTER_START_ENABLED=true
-        initWorkerServer
+        export WORKER_START_ENABLED=true
-        initApiServer
+        export API_START_ENABLED=true
-        initAlertServer
+        export ALERT_START_ENABLED=true
-        initLoggerServer
+        export LOGGER_START_ENABLED=true
-        LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api.log
     ;;
     (master-server)
         waitZK
         waitDatabase
-        initMasterServer
+        export MASTER_START_ENABLED=true
-        LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-master.log
     ;;
     (worker-server)
         waitZK
         waitDatabase
-        initWorkerServer
+        export WORKER_START_ENABLED=true
-        initLoggerServer
+        export LOGGER_START_ENABLED=true
-        LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log
     ;;
     (api-server)
         waitZK
         waitDatabase
         initDatabase
-        initApiServer
+        export API_START_ENABLED=true
-        LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api.log
     ;;
     (alert-server)
         waitDatabase
         checkInitDatabase
-        initAlertServer
+        export ALERT_START_ENABLED=true
-        LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-alert.log
     ;;
     (help)
         printUsage
@@ -200,8 +163,8 @@ case "$1" in
     ;;
 esac
 
-# init directories and log files
+# init directories
-mkdir -p ${DOLPHINSCHEDULER_LOGS} && cat /dev/null >> ${LOGFILE}
+mkdir -p ${DOLPHINSCHEDULER_HOME}/logs
 
-echo "tail begin"
+# start supervisord
-exec bash -c "tail -n 1 -f ${LOGFILE}"
+supervisord -n -u root

script/dolphinscheduler-daemon.sh

@@ -54,42 +54,36 @@ fi
 log=$DOLPHINSCHEDULER_LOG_DIR/dolphinscheduler-$command-$HOSTNAME.out
 pid=$DOLPHINSCHEDULER_PID_DIR/dolphinscheduler-$command.pid
 
-# print logs to /dev/null in docker
-if [ "$DOCKER" = "true" ]; then
-  echo "start in docker"
-  log=/dev/null
-fi
-
 cd $DOLPHINSCHEDULER_HOME
 
 if [ "$command" = "api-server" ]; then
   HEAP_INITIAL_SIZE=1g
   HEAP_MAX_SIZE=1g
-  HEAP_NEW_GENERATION__SIZE=500m
+  HEAP_NEW_GENERATION_SIZE=512m
   LOG_FILE="-Dlogging.config=classpath:logback-api.xml -Dspring.profiles.active=api"
   CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer
 elif [ "$command" = "master-server" ]; then
   HEAP_INITIAL_SIZE=4g
   HEAP_MAX_SIZE=4g
-  HEAP_NEW_GENERATION__SIZE=2g
+  HEAP_NEW_GENERATION_SIZE=2g
   LOG_FILE="-Dlogging.config=classpath:logback-master.xml -Ddruid.mysql.usePingMethod=false"
   CLASS=org.apache.dolphinscheduler.server.master.MasterServer
 elif [ "$command" = "worker-server" ]; then
   HEAP_INITIAL_SIZE=2g
   HEAP_MAX_SIZE=2g
-  HEAP_NEW_GENERATION__SIZE=1g
+  HEAP_NEW_GENERATION_SIZE=1g
   LOG_FILE="-Dlogging.config=classpath:logback-worker.xml -Ddruid.mysql.usePingMethod=false"
   CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer
 elif [ "$command" = "alert-server" ]; then
   HEAP_INITIAL_SIZE=1g
   HEAP_MAX_SIZE=1g
-  HEAP_NEW_GENERATION__SIZE=500m
+  HEAP_NEW_GENERATION_SIZE=512m
   LOG_FILE="-Dlogback.configurationFile=conf/logback-alert.xml"
   CLASS=org.apache.dolphinscheduler.alert.AlertServer
 elif [ "$command" = "logger-server" ]; then
   HEAP_INITIAL_SIZE=1g
   HEAP_MAX_SIZE=1g
-  HEAP_NEW_GENERATION__SIZE=500m
+  HEAP_NEW_GENERATION_SIZE=512m
   CLASS=org.apache.dolphinscheduler.server.log.LoggerServer
 elif [ "$command" = "zookeeper-server" ]; then
   #note: this command just for getting a quick experience，not recommended for production. this operation will start a standalone zookeeper server
@@ -100,10 +94,15 @@ else
   exit 1
 fi
 
-export DOLPHINSCHEDULER_OPTS="-server -Xms$HEAP_INITIAL_SIZE -Xmx$HEAP_MAX_SIZE -Xmn$HEAP_NEW_GENERATION__SIZE -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m  -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError  -XX:HeapDumpPath=dump.hprof $DOLPHINSCHEDULER_OPTS"
+export DOLPHINSCHEDULER_OPTS="-server -Xms$HEAP_INITIAL_SIZE -Xmx$HEAP_MAX_SIZE -Xmn$HEAP_NEW_GENERATION_SIZE -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:$DOLPHINSCHEDULER_LOG_DIR/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof $DOLPHINSCHEDULER_OPTS"
 
 case $startStop in
   (start)
+    exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS"
+    if [ "$DOCKER" = "true" ]; then
+      echo "start in docker"
+      $JAVA_HOME/bin/java $exec_command
+    else
       [ -w "$DOLPHINSCHEDULER_PID_DIR" ] || mkdir -p "$DOLPHINSCHEDULER_PID_DIR"
 
       if [ -f $pid ]; then
@@ -114,12 +113,10 @@ case $startStop in
       fi
 
       echo starting $command, logging to $log
-
-    exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS"
-
       echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &"
       nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &
       echo $! > $pid
+    fi
     ;;
 
   (stop)