Browse Source

Merge pull request #4900 from chengshiwen/improve-docker-supervisor-and-log

[Fix-4897][Docker] Support supervisor management and fix log stdout
pull/3/MERGE
xingchun-chen 4 years ago committed by GitHub
parent
commit
cf13c87dae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      .licenserc.yaml
  2. 3
      docker/build/Dockerfile
  3. 9
      docker/build/conf/dolphinscheduler/logback/logback-alert.xml
  4. 9
      docker/build/conf/dolphinscheduler/logback/logback-api.xml
  5. 9
      docker/build/conf/dolphinscheduler/logback/logback-master.xml
  6. 9
      docker/build/conf/dolphinscheduler/logback/logback-worker.xml
  7. 92
      docker/build/conf/dolphinscheduler/supervisor/supervisor.ini
  8. 19
      docker/build/startup-init-conf.sh
  9. 79
      docker/build/startup.sh
  10. 47
      script/dolphinscheduler-daemon.sh

1
.licenserc.yaml

@ -34,6 +34,7 @@ header:
- '**/*.md'
- '**/*.json'
- '**/*.iml'
- '**/*.ini'
- '**/.babelrc'
- '**/.eslintignore'
- '**/.gitignore'

3
docker/build/Dockerfile

@ -28,7 +28,7 @@ ENV DOCKER true
# RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories
# RUN sed -i 's/dl-cdn.alpinelinux.org/mirror.tuna.tsinghua.edu.cn/g' /etc/apk/repositories
RUN apk update && \
apk add --no-cache tzdata dos2unix bash python2 python3 procps sudo shadow tini postgresql-client && \
apk add --no-cache tzdata dos2unix bash python2 python3 supervisor procps sudo shadow tini postgresql-client && \
cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
apk del tzdata && \
rm -rf /var/cache/apk/*
@ -44,6 +44,7 @@ COPY ./startup-init-conf.sh /root/startup-init-conf.sh
COPY ./startup.sh /root/startup.sh
COPY ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/
COPY ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/
COPY ./conf/dolphinscheduler/supervisor/supervisor.ini /etc/supervisor.d/
COPY ./conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl /opt/dolphinscheduler/conf/env/
RUN dos2unix /root/checkpoint.sh && \
dos2unix /root/startup-init-conf.sh && \

9
docker/build/conf/dolphinscheduler/logback/logback-alert.xml

@ -20,6 +20,14 @@
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<appender name="ALERTLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${log.base}/dolphinscheduler-alert.log</file>
@ -37,6 +45,7 @@
</appender>
<root level="INFO">
<appender-ref ref="STDOUT"/>
<appender-ref ref="ALERTLOGFILE"/>
</root>

9
docker/build/conf/dolphinscheduler/logback/logback-api.xml

@ -20,6 +20,14 @@
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<!-- api server logback config start -->
<appender name="APILOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
@ -47,6 +55,7 @@
<root level="INFO">
<appender-ref ref="STDOUT"/>
<appender-ref ref="APILOGFILE"/>
</root>

9
docker/build/conf/dolphinscheduler/logback/logback-master.xml

@ -20,6 +20,14 @@
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<conversionRule conversionWord="messsage"
converterClass="org.apache.dolphinscheduler.server.log.SensitiveDataConverter"/>
@ -66,6 +74,7 @@
<!-- master server logback config end -->
<root level="INFO">
<appender-ref ref="STDOUT"/>
<appender-ref ref="TASKLOGFILE"/>
<appender-ref ref="MASTERLOGFILE"/>
</root>

9
docker/build/conf/dolphinscheduler/logback/logback-worker.xml

@ -20,6 +20,14 @@
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<!-- worker server logback config start -->
<conversionRule conversionWord="messsage"
@ -66,6 +74,7 @@
<!-- worker server logback config end -->
<root level="INFO">
<appender-ref ref="STDOUT"/>
<appender-ref ref="TASKLOGFILE"/>
<appender-ref ref="WORKERLOGFILE"/>
</root>

92
docker/build/conf/dolphinscheduler/supervisor/supervisor.ini

@ -0,0 +1,92 @@
; Licensed to the Apache Software Foundation (ASF) under one
; or more contributor license agreements. See the NOTICE file
; distributed with this work for additional information
; regarding copyright ownership. The ASF licenses this file
; to you under the Apache License, Version 2.0 (the
; "License"); you may not use this file except in compliance
; with the License. You may obtain a copy of the License at
;
; http://www.apache.org/licenses/LICENSE-2.0
;
; Unless required by applicable law or agreed to in writing, software
; distributed under the License is distributed on an "AS IS" BASIS,
; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
; See the License for the specific language governing permissions and
; limitations under the License.
; program config file
[program:master]
command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start master-server
directory=%(ENV_DOLPHINSCHEDULER_HOME)s
priority=999
autostart=%(ENV_MASTER_START_ENABLED)s
autorestart=true
startsecs=10
stopwaitsecs=3
exitcodes=0
stopasgroup=true
killasgroup=true
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
[program:worker]
command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start worker-server
directory=%(ENV_DOLPHINSCHEDULER_HOME)s
priority=999
autostart=%(ENV_WORKER_START_ENABLED)s
autorestart=true
startsecs=10
stopwaitsecs=3
exitcodes=0
stopasgroup=true
killasgroup=true
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
[program:api]
command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start api-server
directory=%(ENV_DOLPHINSCHEDULER_HOME)s
priority=999
autostart=%(ENV_API_START_ENABLED)s
autorestart=true
startsecs=10
stopwaitsecs=3
exitcodes=0
stopasgroup=true
killasgroup=true
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
[program:alert]
command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start alert-server
directory=%(ENV_DOLPHINSCHEDULER_HOME)s
priority=999
autostart=%(ENV_ALERT_START_ENABLED)s
autorestart=true
startsecs=5
stopwaitsecs=3
exitcodes=0
stopasgroup=true
killasgroup=true
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
[program:logger]
command=%(ENV_DOLPHINSCHEDULER_BIN)s/dolphinscheduler-daemon.sh start logger-server
directory=%(ENV_DOLPHINSCHEDULER_HOME)s
priority=999
autostart=%(ENV_LOGGER_START_ENABLED)s
autorestart=true
startsecs=5
stopwaitsecs=3
exitcodes=0
stopasgroup=true
killasgroup=true
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0

19
docker/build/startup-init-conf.sh

@ -92,11 +92,20 @@ export ALERT_LISTEN_HOST=${ALERT_LISTEN_HOST:-"127.0.0.1"}
#============================================================================
export ALERT_PLUGIN_DIR=${ALERT_PLUGIN_DIR:-"lib/plugin/alert"}
echo "generate app config"
find ${DOLPHINSCHEDULER_HOME}/conf/ -name "*.tpl" | while read file; do
echo "generate dolphinscheduler config"
ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do
eval "cat << EOF
$(cat ${file})
$(cat ${DOLPHINSCHEDULER_HOME}/conf/${line})
EOF
" > ${file%.*}
" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*}
done
find ${DOLPHINSCHEDULER_HOME}/conf/ -name "*.sh" -exec chmod +x {} \;
# generate dolphinscheduler env only in docker
DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh
if [ -z "${KUBERNETES_SERVICE_HOST}" ] && [ -r "${DOLPHINSCHEDULER_ENV_PATH}.tpl" ]; then
eval "cat << EOF
$(cat ${DOLPHINSCHEDULER_ENV_PATH}.tpl)
EOF
" > ${DOLPHINSCHEDULER_ENV_PATH}
chmod +x ${DOLPHINSCHEDULER_ENV_PATH}
fi

79
docker/build/startup.sh

@ -18,9 +18,12 @@
set -e
DOLPHINSCHEDULER_BIN=${DOLPHINSCHEDULER_HOME}/bin
DOLPHINSCHEDULER_SCRIPT=${DOLPHINSCHEDULER_HOME}/script
DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs
export DOLPHINSCHEDULER_BIN=${DOLPHINSCHEDULER_HOME}/bin
export MASTER_START_ENABLED=false
export WORKER_START_ENABLED=false
export API_START_ENABLED=false
export ALERT_START_ENABLED=false
export LOGGER_START_ENABLED=false
# wait database
waitDatabase() {
@ -54,7 +57,7 @@ waitDatabase() {
# init database
initDatabase() {
echo "import sql data"
${DOLPHINSCHEDULER_SCRIPT}/create-dolphinscheduler.sh
${DOLPHINSCHEDULER_HOME}/script/create-dolphinscheduler.sh
}
# check ds version
@ -102,41 +105,6 @@ waitZK() {
done
}
# start master-server
initMasterServer() {
echo "start master-server"
${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh stop master-server
${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh start master-server
}
# start worker-server
initWorkerServer() {
echo "start worker-server"
${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh stop worker-server
${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh start worker-server
}
# start api-server
initApiServer() {
echo "start api-server"
${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh stop api-server
${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh start api-server
}
# start logger-server
initLoggerServer() {
echo "start logger-server"
${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh stop logger-server
${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh start logger-server
}
# start alert-server
initAlertServer() {
echo "start alert-server"
${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh stop alert-server
${DOLPHINSCHEDULER_BIN}/dolphinscheduler-daemon.sh start alert-server
}
# print usage
printUsage() {
echo -e "Dolphin Scheduler is a distributed and easy-to-expand visual DAG workflow scheduling system,"
@ -157,38 +125,33 @@ case "$1" in
waitZK
waitDatabase
initDatabase
initMasterServer
initWorkerServer
initApiServer
initAlertServer
initLoggerServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api.log
export MASTER_START_ENABLED=true
export WORKER_START_ENABLED=true
export API_START_ENABLED=true
export ALERT_START_ENABLED=true
export LOGGER_START_ENABLED=true
;;
(master-server)
waitZK
waitDatabase
initMasterServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-master.log
export MASTER_START_ENABLED=true
;;
(worker-server)
waitZK
waitDatabase
initWorkerServer
initLoggerServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log
export WORKER_START_ENABLED=true
export LOGGER_START_ENABLED=true
;;
(api-server)
waitZK
waitDatabase
initDatabase
initApiServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api.log
export API_START_ENABLED=true
;;
(alert-server)
waitDatabase
checkInitDatabase
initAlertServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-alert.log
export ALERT_START_ENABLED=true
;;
(help)
printUsage
@ -200,8 +163,8 @@ case "$1" in
;;
esac
# init directories and log files
mkdir -p ${DOLPHINSCHEDULER_LOGS} && cat /dev/null >> ${LOGFILE}
# init directories
mkdir -p ${DOLPHINSCHEDULER_HOME}/logs
echo "tail begin"
exec bash -c "tail -n 1 -f ${LOGFILE}"
# start supervisord
supervisord -n -u root

47
script/dolphinscheduler-daemon.sh

@ -54,42 +54,36 @@ fi
log=$DOLPHINSCHEDULER_LOG_DIR/dolphinscheduler-$command-$HOSTNAME.out
pid=$DOLPHINSCHEDULER_PID_DIR/dolphinscheduler-$command.pid
# print logs to /dev/null in docker
if [ "$DOCKER" = "true" ]; then
echo "start in docker"
log=/dev/null
fi
cd $DOLPHINSCHEDULER_HOME
if [ "$command" = "api-server" ]; then
HEAP_INITIAL_SIZE=1g
HEAP_MAX_SIZE=1g
HEAP_NEW_GENERATION__SIZE=500m
HEAP_NEW_GENERATION_SIZE=512m
LOG_FILE="-Dlogging.config=classpath:logback-api.xml -Dspring.profiles.active=api"
CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer
elif [ "$command" = "master-server" ]; then
HEAP_INITIAL_SIZE=4g
HEAP_MAX_SIZE=4g
HEAP_NEW_GENERATION__SIZE=2g
HEAP_NEW_GENERATION_SIZE=2g
LOG_FILE="-Dlogging.config=classpath:logback-master.xml -Ddruid.mysql.usePingMethod=false"
CLASS=org.apache.dolphinscheduler.server.master.MasterServer
elif [ "$command" = "worker-server" ]; then
HEAP_INITIAL_SIZE=2g
HEAP_MAX_SIZE=2g
HEAP_NEW_GENERATION__SIZE=1g
HEAP_NEW_GENERATION_SIZE=1g
LOG_FILE="-Dlogging.config=classpath:logback-worker.xml -Ddruid.mysql.usePingMethod=false"
CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer
elif [ "$command" = "alert-server" ]; then
HEAP_INITIAL_SIZE=1g
HEAP_MAX_SIZE=1g
HEAP_NEW_GENERATION__SIZE=500m
HEAP_NEW_GENERATION_SIZE=512m
LOG_FILE="-Dlogback.configurationFile=conf/logback-alert.xml"
CLASS=org.apache.dolphinscheduler.alert.AlertServer
elif [ "$command" = "logger-server" ]; then
HEAP_INITIAL_SIZE=1g
HEAP_MAX_SIZE=1g
HEAP_NEW_GENERATION__SIZE=500m
HEAP_NEW_GENERATION_SIZE=512m
CLASS=org.apache.dolphinscheduler.server.log.LoggerServer
elif [ "$command" = "zookeeper-server" ]; then
#note: this command just for getting a quick experience,not recommended for production. this operation will start a standalone zookeeper server
@ -100,26 +94,29 @@ else
exit 1
fi
export DOLPHINSCHEDULER_OPTS="-server -Xms$HEAP_INITIAL_SIZE -Xmx$HEAP_MAX_SIZE -Xmn$HEAP_NEW_GENERATION__SIZE -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof $DOLPHINSCHEDULER_OPTS"
export DOLPHINSCHEDULER_OPTS="-server -Xms$HEAP_INITIAL_SIZE -Xmx$HEAP_MAX_SIZE -Xmn$HEAP_NEW_GENERATION_SIZE -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:$DOLPHINSCHEDULER_LOG_DIR/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof $DOLPHINSCHEDULER_OPTS"
case $startStop in
(start)
[ -w "$DOLPHINSCHEDULER_PID_DIR" ] || mkdir -p "$DOLPHINSCHEDULER_PID_DIR"
exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS"
if [ "$DOCKER" = "true" ]; then
echo "start in docker"
$JAVA_HOME/bin/java $exec_command
else
[ -w "$DOLPHINSCHEDULER_PID_DIR" ] || mkdir -p "$DOLPHINSCHEDULER_PID_DIR"
if [ -f $pid ]; then
if kill -0 `cat $pid` > /dev/null 2>&1; then
echo $command running as process `cat $pid`. Stop it first.
exit 1
if [ -f $pid ]; then
if kill -0 `cat $pid` > /dev/null 2>&1; then
echo $command running as process `cat $pid`. Stop it first.
exit 1
fi
fi
fi
echo starting $command, logging to $log
exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS"
echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &"
nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &
echo $! > $pid
echo starting $command, logging to $log
echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &"
nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &
echo $! > $pid
fi
;;
(stop)

Loading…
Cancel
Save