Merge pull request #4900 from chengshiwen/improve-docker-supervisor-and-log

[Fix-4897][Docker] Support supervisor management and fix log stdout
4 years ago · cf13c87dae
10 changed files with 188 additions and 89 deletions
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@ -34,6 +34,7 @@ header:
    - '**/*.md'
    - '**/*.json'
    - '**/*.iml'
+    - '**/*.ini'
    - '**/.babelrc'
    - '**/.eslintignore'
    - '**/.gitignore'
--- a/docker/build/Dockerfile
+++ b/docker/build/Dockerfile
@ -28,7 +28,7 @@ ENV DOCKER true
 # RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories
 # RUN sed -i 's/dl-cdn.alpinelinux.org/mirror.tuna.tsinghua.edu.cn/g' /etc/apk/repositories
 RUN apk update && \
-    apk add --no-cache tzdata dos2unix bash python2 python3 procps sudo shadow tini postgresql-client && \
+    apk add --no-cache tzdata dos2unix bash python2 python3 supervisor procps sudo shadow tini postgresql-client && \
    cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
    apk del tzdata && \
    rm -rf /var/cache/apk/*
@ -44,6 +44,7 @@ COPY ./startup-init-conf.sh /root/startup-init-conf.sh
 COPY ./startup.sh /root/startup.sh
 COPY ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/
 COPY ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/
+COPY ./conf/dolphinscheduler/supervisor/supervisor.ini /etc/supervisor.d/
 COPY ./conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl /opt/dolphinscheduler/conf/env/
 RUN dos2unix /root/checkpoint.sh && \
    dos2unix /root/startup-init-conf.sh && \
--- a/docker/build/conf/dolphinscheduler/logback/logback-alert.xml
+++ b/docker/build/conf/dolphinscheduler/logback/logback-alert.xml
@ -20,6 +20,14 @@
 <configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->

    <property name="log.base" value="logs"/>
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>
+                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+            </pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>

    <appender name="ALERTLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <file>${log.base}/dolphinscheduler-alert.log</file>
@ -37,6 +45,7 @@
    </appender>

    <root level="INFO">
+        <appender-ref ref="STDOUT"/>
        <appender-ref ref="ALERTLOGFILE"/>
    </root>

--- a/docker/build/conf/dolphinscheduler/logback/logback-api.xml
+++ b/docker/build/conf/dolphinscheduler/logback/logback-api.xml
@ -20,6 +20,14 @@
 <configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->

    <property name="log.base" value="logs"/>
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>
+                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+            </pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>

    <!-- api server logback config start -->
    <appender name="APILOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
@ -47,6 +55,7 @@


    <root level="INFO">
+        <appender-ref ref="STDOUT"/>
        <appender-ref ref="APILOGFILE"/>
    </root>

--- a/docker/build/conf/dolphinscheduler/logback/logback-master.xml
+++ b/docker/build/conf/dolphinscheduler/logback/logback-master.xml
@ -20,6 +20,14 @@
 <configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->

    <property name="log.base" value="logs"/>
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>
+                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+            </pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>

    <conversionRule conversionWord="messsage"
                    converterClass="org.apache.dolphinscheduler.server.log.SensitiveDataConverter"/>
@ -66,6 +74,7 @@
    <!-- master server logback config end -->

    <root level="INFO">
+        <appender-ref ref="STDOUT"/>
        <appender-ref ref="TASKLOGFILE"/>
        <appender-ref ref="MASTERLOGFILE"/>
    </root>
--- a/docker/build/conf/dolphinscheduler/logback/logback-worker.xml
+++ b/docker/build/conf/dolphinscheduler/logback/logback-worker.xml
@ -20,6 +20,14 @@
 <configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->

    <property name="log.base" value="logs"/>
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>
+                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+            </pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>

    <!-- worker server logback config start -->
    <conversionRule conversionWord="messsage"
@ -66,6 +74,7 @@
    <!-- worker server logback config end -->

    <root level="INFO">
+        <appender-ref ref="STDOUT"/>
        <appender-ref ref="TASKLOGFILE"/>
        <appender-ref ref="WORKERLOGFILE"/>
    </root>
--- a/docker/build/conf/dolphinscheduler/supervisor/supervisor.ini
+++ b/docker/build/conf/dolphinscheduler/supervisor/supervisor.ini
@ -0,0 +1,92 @@
+; Licensed to the Apache Software Foundation (ASF) under one
+; or more contributor license agreements.  See the NOTICE file
+; distributed with this work for additional information
+; regarding copyright ownership.  The ASF licenses this file
+; to you under the Apache License, Version 2.0 (the
+; "License"); you may not use this file except in compliance
+; with the License.  You may obtain a copy of the License at
+;
+;     http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing, software
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
+
+; program config file
+
+[program:master]
+command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start master-server
+directory=%(ENV_DOLPHINSCHEDULER_HOME)s
+priority=999
+autostart=%(ENV_MASTER_START_ENABLED)s
+autorestart=true
+startsecs=10
+stopwaitsecs=3
+exitcodes=0
+stopasgroup=true
+killasgroup=true
+redirect_stderr=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+
+[program:worker]
+command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start worker-server
+directory=%(ENV_DOLPHINSCHEDULER_HOME)s
+priority=999
+autostart=%(ENV_WORKER_START_ENABLED)s
+autorestart=true
+startsecs=10
+stopwaitsecs=3
+exitcodes=0
+stopasgroup=true
+killasgroup=true
+redirect_stderr=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+
+[program:api]
+command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start api-server
+directory=%(ENV_DOLPHINSCHEDULER_HOME)s
+priority=999
+autostart=%(ENV_API_START_ENABLED)s
+autorestart=true
+startsecs=10
+stopwaitsecs=3
+exitcodes=0
+stopasgroup=true
+killasgroup=true
+redirect_stderr=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+
+[program:alert]
+command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start alert-server
+directory=%(ENV_DOLPHINSCHEDULER_HOME)s
+priority=999
+autostart=%(ENV_ALERT_START_ENABLED)s
+autorestart=true
+startsecs=5
+stopwaitsecs=3
+exitcodes=0
+stopasgroup=true
+killasgroup=true
+redirect_stderr=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+
+[program:logger]
+command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start logger-server
+directory=%(ENV_DOLPHINSCHEDULER_HOME)s
+priority=999
+autostart=%(ENV_LOGGER_START_ENABLED)s
+autorestart=true
+startsecs=5
+stopwaitsecs=3
+exitcodes=0
+stopasgroup=true
+killasgroup=true
+redirect_stderr=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
--- a/docker/build/startup-init-conf.sh
+++ b/docker/build/startup-init-conf.sh
@ -92,11 +92,20 @@ export ALERT_LISTEN_HOST=${ALERT_LISTEN_HOST:-"127.0.0.1"}
 #============================================================================
 export ALERT_PLUGIN_DIR=${ALERT_PLUGIN_DIR:-"lib/plugin/alert"}

-echo "generate app config"
-find ${DOLPHINSCHEDULER_HOME}/conf/ -name "*.tpl" | while read file; do
+echo "generate dolphinscheduler config"
+ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do
 eval "cat << EOF
-$(cat ${file})
+$(cat ${DOLPHINSCHEDULER_HOME}/conf/${line})
 EOF
-" > ${file%.*}
+" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*}
 done
-find ${DOLPHINSCHEDULER_HOME}/conf/ -name "*.sh" -exec chmod +x {} \;
+
+# generate dolphinscheduler env only in docker
+DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh
+if [ -z "${KUBERNETES_SERVICE_HOST}" ] && [ -r "${DOLPHINSCHEDULER_ENV_PATH}.tpl" ]; then
+eval "cat << EOF
+$(cat ${DOLPHINSCHEDULER_ENV_PATH}.tpl)
+EOF
+" > ${DOLPHINSCHEDULER_ENV_PATH}
+chmod +x ${DOLPHINSCHEDULER_ENV_PATH}
+fi
--- a/docker/build/startup.sh
+++ b/docker/build/startup.sh
@ -18,9 +18,12 @@

 set -e

-DOLPHINSCHEDULER_BIN=${DOLPHINSCHEDULER_HOME}/bin
-DOLPHINSCHEDULER_SCRIPT=${DOLPHINSCHEDULER_HOME}/script
-DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs
+export DOLPHINSCHEDULER_BIN=${DOLPHINSCHEDULER_HOME}/bin
+export MASTER_START_ENABLED=false
+export WORKER_START_ENABLED=false
+export API_START_ENABLED=false
+export ALERT_START_ENABLED=false
+export LOGGER_START_ENABLED=false

 # wait database
 waitDatabase() {
@ -54,7 +57,7 @@ waitDatabase() {
 # init database
 initDatabase() {
    echo "import sql data"
-    ${DOLPHINSCHEDULER_SCRIPT}/create-dolphinscheduler.sh
+    ${DOLPHINSCHEDULER_HOME}/script/create-dolphinscheduler.sh
 }

 # check ds version
@ -102,41 +105,6 @@ waitZK() {
    done
 }

-# start master-server
-initMasterServer() {
-    echo "start master-server"
-    ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh stop master-server
-    ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh start master-server
-}
-
-# start worker-server
-initWorkerServer() {
-    echo "start worker-server"
-    ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh stop worker-server
-    ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh start worker-server
-}
-
-# start api-server
-initApiServer() {
-    echo "start api-server"
-    ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh stop api-server
-    ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh start api-server
-}
-
-# start logger-server
-initLoggerServer() {
-    echo "start logger-server"
-    ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh stop logger-server
-    ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh start logger-server
-}
-
-# start alert-server
-initAlertServer() {
-    echo "start alert-server"
-    ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh stop alert-server
-    ${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh start alert-server
-}
-
 # print usage
 printUsage() {
    echo -e "Dolphin Scheduler is a distributed and easy-to-expand visual DAG workflow scheduling system,"
@ -157,38 +125,33 @@ case "$1" in
        waitZK
        waitDatabase
        initDatabase
-        initMasterServer
-        initWorkerServer
-        initApiServer
-        initAlertServer
-        initLoggerServer
-        LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api.log
+        export MASTER_START_ENABLED=true
+        export WORKER_START_ENABLED=true
+        export API_START_ENABLED=true
+        export ALERT_START_ENABLED=true
+        export LOGGER_START_ENABLED=true
    ;;
    (master-server)
        waitZK
        waitDatabase
-        initMasterServer
-        LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-master.log
+        export MASTER_START_ENABLED=true
    ;;
    (worker-server)
        waitZK
        waitDatabase
-        initWorkerServer
-        initLoggerServer
-        LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log
+        export WORKER_START_ENABLED=true
+        export LOGGER_START_ENABLED=true
    ;;
    (api-server)
        waitZK
        waitDatabase
        initDatabase
-        initApiServer
-        LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api.log
+        export API_START_ENABLED=true
    ;;
    (alert-server)
        waitDatabase
        checkInitDatabase
-        initAlertServer
-        LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-alert.log
+        export ALERT_START_ENABLED=true
    ;;
    (help)
        printUsage
@ -200,8 +163,8 @@ case "$1" in
    ;;
 esac

-# init directories and log files
-mkdir -p ${DOLPHINSCHEDULER_LOGS} && cat /dev/null >> ${LOGFILE}
+# init directories
+mkdir -p ${DOLPHINSCHEDULER_HOME}/logs

-echo "tail begin"
-exec bash -c "tail -n 1 -f ${LOGFILE}"
+# start supervisord
+supervisord -n -u root
--- a/script/dolphinscheduler-daemon.sh
+++ b/script/dolphinscheduler-daemon.sh
@ -54,42 +54,36 @@ fi
 log=$DOLPHINSCHEDULER_LOG_DIR/dolphinscheduler-$command-$HOSTNAME.out
 pid=$DOLPHINSCHEDULER_PID_DIR/dolphinscheduler-$command.pid

-# print logs to /dev/null in docker
-if [ "$DOCKER" = "true" ]; then
-  echo "start in docker"
-  log=/dev/null
-fi
-
 cd $DOLPHINSCHEDULER_HOME

 if [ "$command" = "api-server" ]; then
  HEAP_INITIAL_SIZE=1g
  HEAP_MAX_SIZE=1g
-  HEAP_NEW_GENERATION__SIZE=500m
+  HEAP_NEW_GENERATION_SIZE=512m
  LOG_FILE="-Dlogging.config=classpath:logback-api.xml -Dspring.profiles.active=api"
  CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer
 elif [ "$command" = "master-server" ]; then
  HEAP_INITIAL_SIZE=4g
  HEAP_MAX_SIZE=4g
-  HEAP_NEW_GENERATION__SIZE=2g
+  HEAP_NEW_GENERATION_SIZE=2g
  LOG_FILE="-Dlogging.config=classpath:logback-master.xml -Ddruid.mysql.usePingMethod=false"
  CLASS=org.apache.dolphinscheduler.server.master.MasterServer
 elif [ "$command" = "worker-server" ]; then
  HEAP_INITIAL_SIZE=2g
  HEAP_MAX_SIZE=2g
-  HEAP_NEW_GENERATION__SIZE=1g
+  HEAP_NEW_GENERATION_SIZE=1g
  LOG_FILE="-Dlogging.config=classpath:logback-worker.xml -Ddruid.mysql.usePingMethod=false"
  CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer
 elif [ "$command" = "alert-server" ]; then
  HEAP_INITIAL_SIZE=1g
  HEAP_MAX_SIZE=1g
-  HEAP_NEW_GENERATION__SIZE=500m
+  HEAP_NEW_GENERATION_SIZE=512m
  LOG_FILE="-Dlogback.configurationFile=conf/logback-alert.xml"
  CLASS=org.apache.dolphinscheduler.alert.AlertServer
 elif [ "$command" = "logger-server" ]; then
  HEAP_INITIAL_SIZE=1g
  HEAP_MAX_SIZE=1g
-  HEAP_NEW_GENERATION__SIZE=500m
+  HEAP_NEW_GENERATION_SIZE=512m
  CLASS=org.apache.dolphinscheduler.server.log.LoggerServer
 elif [ "$command" = "zookeeper-server" ]; then
  #note: this command just for getting a quick experience，not recommended for production. this operation will start a standalone zookeeper server
@ -100,26 +94,29 @@ else
  exit 1
 fi

-export DOLPHINSCHEDULER_OPTS="-server -Xms$HEAP_INITIAL_SIZE -Xmx$HEAP_MAX_SIZE -Xmn$HEAP_NEW_GENERATION__SIZE -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m  -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError  -XX:HeapDumpPath=dump.hprof $DOLPHINSCHEDULER_OPTS"
+export DOLPHINSCHEDULER_OPTS="-server -Xms$HEAP_INITIAL_SIZE -Xmx$HEAP_MAX_SIZE -Xmn$HEAP_NEW_GENERATION_SIZE -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:$DOLPHINSCHEDULER_LOG_DIR/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof $DOLPHINSCHEDULER_OPTS"

 case $startStop in
  (start)
-    [ -w "$DOLPHINSCHEDULER_PID_DIR" ] ||  mkdir -p "$DOLPHINSCHEDULER_PID_DIR"
+    exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS"
+    if [ "$DOCKER" = "true" ]; then
+      echo "start in docker"
+      $JAVA_HOME/bin/java $exec_command
+    else
+      [ -w "$DOLPHINSCHEDULER_PID_DIR" ] || mkdir -p "$DOLPHINSCHEDULER_PID_DIR"

-    if [ -f $pid ]; then
-      if kill -0 `cat $pid` > /dev/null 2>&1; then
-        echo $command running as process `cat $pid`.  Stop it first.
-        exit 1
+      if [ -f $pid ]; then
+        if kill -0 `cat $pid` > /dev/null 2>&1; then
+          echo $command running as process `cat $pid`.  Stop it first.
+          exit 1
+        fi
      fi
-    fi
-
-    echo starting $command, logging to $log

-    exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS"
-
-    echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &"
-    nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &
-    echo $! > $pid
+      echo starting $command, logging to $log
+      echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &"
+      nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &
+      echo $! > $pid
+    fi
    ;;

  (stop)