diff --git a/.github/workflows/cluster-test/mysql/dolphinscheduler_env.sh b/.github/workflows/cluster-test/mysql/dolphinscheduler_env.sh index 3727538d5e..10be4d8453 100755 --- a/.github/workflows/cluster-test/mysql/dolphinscheduler_env.sh +++ b/.github/workflows/cluster-test/mysql/dolphinscheduler_env.sh @@ -45,6 +45,9 @@ export DATAX_HOME=${DATAX_HOME:-/opt/soft/datax} export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH +export MASTER_RESERVED_MEMORY=0.01 +export WORKER_RESERVED_MEMORY=0.01 + # applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log #export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/* #export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH diff --git a/.github/workflows/cluster-test/mysql/running_test.sh b/.github/workflows/cluster-test/mysql/running_test.sh index eeaa06c2d6..7582c3ccc5 100644 --- a/.github/workflows/cluster-test/mysql/running_test.sh +++ b/.github/workflows/cluster-test/mysql/running_test.sh @@ -24,7 +24,7 @@ WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http:/ ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health" #Cluster start health check -TIMEOUT=120 +TIMEOUT=180 START_HEALTHCHECK_EXITCODE=0 for ((i=1; i<=TIMEOUT; i++)) @@ -45,11 +45,28 @@ do fi if [[ $i -eq $TIMEOUT ]];then - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log" - echo "cluster start health check failed" + if [[ $MASTER_HTTP_STATUS -ne 200 ]];then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/*.out" + echo "master start health check failed" + fi + if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/dolphinscheduler-worker.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/*.out" + echo "worker start health check failed" + fi + if [[ $API_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/dolphinscheduler-api.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/*.out" + echo "api start health check failed" + fi + if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/dolphinscheduler-alert.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/*.out" + echo "alert start health check failed" + fi exit $START_HEALTHCHECK_EXITCODE fi - sleep 1 done diff --git a/.github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh b/.github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh index 63437283b6..445b5bae99 100644 --- a/.github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh +++ b/.github/workflows/cluster-test/postgresql/dolphinscheduler_env.sh @@ -45,6 +45,9 @@ export DATAX_HOME=${DATAX_HOME:-/opt/soft/datax} export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH +export MASTER_RESERVED_MEMORY=0.01 +export WORKER_RESERVED_MEMORY=0.01 + # applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log #export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/* #export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH diff --git a/.github/workflows/cluster-test/postgresql/running_test.sh b/.github/workflows/cluster-test/postgresql/running_test.sh index eeaa06c2d6..0bc861c389 100644 --- a/.github/workflows/cluster-test/postgresql/running_test.sh +++ b/.github/workflows/cluster-test/postgresql/running_test.sh @@ -24,7 +24,7 @@ WORKER_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http:/ ALERT_HEALTHCHECK_COMMAND="curl -I -m 10 -o /dev/null -s -w %{http_code} http://0.0.0.0:50053/actuator/health" #Cluster start health check -TIMEOUT=120 +TIMEOUT=180 START_HEALTHCHECK_EXITCODE=0 for ((i=1; i<=TIMEOUT; i++)) @@ -45,10 +45,28 @@ do fi if [[ $i -eq $TIMEOUT ]];then - docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log" - echo "cluster start health check failed" - exit $START_HEALTHCHECK_EXITCODE - fi + if [[ $MASTER_HTTP_STATUS -ne 200 ]];then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/dolphinscheduler-master.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/master-server/logs/*.out" + echo "master start health check failed" + fi + if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/dolphinscheduler-worker.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/worker-server/logs/*.out" + echo "worker start health check failed" + fi + if [[ $API_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/dolphinscheduler-api.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/api-server/logs/*.out" + echo "api start health check failed" + fi + if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/dolphinscheduler-alert.log" + docker exec -u root ds bash -c "cat /root/apache-dolphinscheduler-*-SNAPSHOT-bin/alert-server/logs/*.out" + echo "alert start health check failed" + fi + exit $START_HEALTHCHECK_EXITCODE + fi sleep 1 done diff --git a/docs/docs/en/architecture/configuration.md b/docs/docs/en/architecture/configuration.md index 423d79fa8f..05d9e99ebc 100644 --- a/docs/docs/en/architecture/configuration.md +++ b/docs/docs/en/architecture/configuration.md @@ -32,6 +32,7 @@ The directory structure of DolphinScheduler is as follows: ├── alert-server directory of DolphinScheduler alert-server commands, configurations scripts and libs │   ├── bin │   │   └── start.sh script to start DolphinScheduler alert-server +│   │   └── jvm_args_env.sh script to set JVM args of DolphinScheduler alert-server │   ├── conf │   │   ├── application.yaml configurations of alert-server │   │   ├── bootstrap.yaml configurations for Spring Cloud bootstrap, mostly you don't need to modify this, @@ -43,6 +44,7 @@ The directory structure of DolphinScheduler is as follows: ├── api-server directory of DolphinScheduler api-server commands, configurations scripts and libs │   ├── bin │   │   └── start.sh script to start DolphinScheduler api-server +│   │   └── jvm_args_env.sh script to set JVM args of DolphinScheduler api-server │   ├── conf │   │   ├── application.yaml configurations of api-server │   │   ├── bootstrap.yaml configurations for Spring Cloud bootstrap, mostly you don't need to modify this, @@ -55,6 +57,7 @@ The directory structure of DolphinScheduler is as follows: ├── master-server directory of DolphinScheduler master-server commands, configurations scripts and libs │   ├── bin │   │   └── start.sh script to start DolphinScheduler master-server +│   │   └── jvm_args_env.sh script to set JVM args of DolphinScheduler master-server │   ├── conf │   │   ├── application.yaml configurations of master-server │   │   ├── bootstrap.yaml configurations for Spring Cloud bootstrap, mostly you don't need to modify this, @@ -66,6 +69,7 @@ The directory structure of DolphinScheduler is as follows: ├── standalone-server directory of DolphinScheduler standalone-server commands, configurations scripts and libs │   ├── bin │   │   └── start.sh script to start DolphinScheduler standalone-server +│   │   └── jvm_args_env.sh script to set JVM args of DolphinScheduler standalone-server │   ├── conf │   │   ├── application.yaml configurations of standalone-server │   │   ├── bootstrap.yaml configurations for Spring Cloud bootstrap, mostly you don't need to modify this, @@ -86,15 +90,16 @@ The directory structure of DolphinScheduler is as follows: │   └── sql .sql files to create or upgrade DolphinScheduler metadata │   ├── worker-server directory of DolphinScheduler worker-server commands, configurations scripts and libs -│ ├── bin -│ │   └── start.sh script to start DolphinScheduler worker-server -│ ├── conf -│ │   ├── application.yaml configurations of worker-server -│ │   ├── bootstrap.yaml configurations for Spring Cloud bootstrap, mostly you don't need to modify this, -│ │   ├── common.properties configurations of common-service like storage, credentials, etc. -│ │   ├── dolphinscheduler_env.sh script to load environment variables for worker-server -│ │   └── logback-spring.xml configurations of worker-service log -│ └── libs directory of worker-server libs +│   ├── bin +│   │   └── start.sh script to start DolphinScheduler worker-server +│   │   └── jvm_args_env.sh script to set JVM args of DolphinScheduler worker-server +│   ├── conf +│   │   ├── application.yaml configurations of worker-server +│   │   ├── bootstrap.yaml configurations for Spring Cloud bootstrap, mostly you don't need to modify this, +│   │   ├── common.properties configurations of common-service like storage, credentials, etc. +│   │   ├── dolphinscheduler_env.sh script to load environment variables for worker-server +│   │   └── logback-spring.xml configurations of worker-service log +│   └── libs directory of worker-server libs │ └── ui directory of front-end web resources ``` diff --git a/docs/docs/zh/architecture/configuration.md b/docs/docs/zh/architecture/configuration.md index 47d0715179..3d91a8797d 100644 --- a/docs/docs/zh/architecture/configuration.md +++ b/docs/docs/zh/architecture/configuration.md @@ -30,6 +30,7 @@ DolphinScheduler的目录结构如下: ├── alert-server DolphinScheduler alert-server命令、配置和依赖存放目录 │   ├── bin │   │   └── start.sh DolphinScheduler alert-server启动脚本 +│   │   └── jvm_args_env.sh DolphinScheduler alert-server jvm参数配置脚本 │   ├── conf │   │   ├── application.yaml alert-server配置文件 │   │   ├── bootstrap.yaml Spring Cloud 启动阶段配置文件, 通常不需要修改 @@ -41,6 +42,7 @@ DolphinScheduler的目录结构如下: ├── api-server DolphinScheduler api-server命令、配置和依赖存放目录 │   ├── bin │   │   └── start.sh DolphinScheduler api-server启动脚本 +│   │   └── jvm_args_env.sh DolphinScheduler api-server jvm参数配置脚本 │   ├── conf │   │   ├── application.yaml api-server配置文件 │   │   ├── bootstrap.yaml Spring Cloud 启动阶段配置文件, 通常不需要修改 @@ -53,6 +55,7 @@ DolphinScheduler的目录结构如下: ├── master-server DolphinScheduler master-server命令、配置和依赖存放目录 │   ├── bin │   │   └── start.sh DolphinScheduler master-server启动脚本 +│   │   └── jvm_args_env.sh DolphinScheduler master-server jvm参数配置脚本 │   ├── conf │   │   ├── application.yaml master-server配置文件 │   │   ├── bootstrap.yaml Spring Cloud 启动阶段配置文件, 通常不需要修改 @@ -64,6 +67,7 @@ DolphinScheduler的目录结构如下: ├── standalone-server DolphinScheduler standalone-server命令、配置和依赖存放目录 │   ├── bin │   │   └── start.sh DolphinScheduler standalone-server启动脚本 +│   │   └── jvm_args_env.sh DolphinScheduler standalone-server jvm参数配置脚本 │   ├── conf │   │   ├── application.yaml standalone-server配置文件 │   │   ├── bootstrap.yaml Spring Cloud 启动阶段配置文件, 通常不需要修改 @@ -74,6 +78,7 @@ DolphinScheduler的目录结构如下: │   ├── libs standalone-server依赖jar包存放目录 │   └── ui standalone-server相关前端WEB资源存放目录 │   +| ├── tools DolphinScheduler元数据工具命令、配置和依赖存放目录 │   ├── bin │   │   └── upgrade-schema.sh DolphinScheduler元数据创建/升级脚本 @@ -83,16 +88,18 @@ DolphinScheduler的目录结构如下: │   ├── libs 元数据工具依赖jar包存放目录 │   └── sql DolphinScheduler元数据创建/升级sql文件 │   +| ├── worker-server DolphinScheduler worker-server命令、配置和依赖存放目录 -│ ├── bin -│ │   └── start.sh DolphinScheduler worker-server启动脚本 -│ ├── conf -│ │   ├── application.yaml worker-server配置文件 -│ │   ├── bootstrap.yaml Spring Cloud 启动阶段配置文件, 通常不需要修改 -│ │   ├── common.properties 公共服务(存储等信息)配置文件 -│ │   ├── dolphinscheduler_env.sh worker-server环境变量配置加载脚本 -│ │   └── logback-spring.xml worker-service日志配置文件 -│ └── libs worker-server依赖jar包存放目录 +│   ├── bin +│   │   └── start.sh DolphinScheduler worker-server 启动脚本 +│   │   └── jvm_args_env.sh DolphinScheduler worker-server jvm参数配置脚本 +│   ├── conf +│   │   ├── application.yaml worker-server配置文件 +│   │   ├── bootstrap.yaml Spring Cloud 启动阶段配置文件, 通常不需要修改 +│   │   ├── common.properties 公共服务(存储等信息)配置文件 +│   │   ├── dolphinscheduler_env.sh worker-server环境变量配置加载脚本 +│   │   └── logback-spring.xml worker-service日志配置文件 +│   └── libs worker-server依赖jar包存放目录 │ └── ui 前端WEB资源目录 ``` diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/jvm_args_env.sh b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/jvm_args_env.sh new file mode 100644 index 0000000000..d6ccc3c571 --- /dev/null +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/jvm_args_env.sh @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +-Xms1g +-Xmx1g +-Xmn512m + +-XX:+IgnoreUnrecognizedVMOptions +-XX:+PrintGCDateStamps +-XX:+PrintGCDetails +-Xloggc:gc.log + +-XX:+HeapDumpOnOutOfMemoryError +-XX:HeapDumpPath=dump.hprof + +-Duser.timezone=${SPRING_JACKSON_TIME_ZONE} \ No newline at end of file diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/start.sh b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/start.sh index 16145e411d..09239cbc49 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/start.sh +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/start.sh @@ -21,12 +21,27 @@ DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)} source "$DOLPHINSCHEDULER_HOME/conf/dolphinscheduler_env.sh" -JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+IgnoreUnrecognizedVMOptions -XX:+PrintGCDateStamps -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} +JVM_ARGS_ENV_FILE=${BIN_DIR}/jvm_args_env.sh +JVM_ARGS="-server" + +if [ -f $JVM_ARGS_ENV_FILE ]; then + while read line + do + if [[ "$line" == -* ]]; then + JVM_ARGS="${JVM_ARGS} $line" + fi + done < $JVM_ARGS_ENV_FILE +fi + +JAVA_OPTS=${JAVA_OPTS:-"${JVM_ARGS}"} if [[ "$DOCKER" == "true" ]]; then JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport" fi +echo "JAVA_HOME=${JAVA_HOME}" +echo "JAVA_OPTS=${JAVA_OPTS}" + $JAVA_HOME/bin/java $JAVA_OPTS \ -cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \ org.apache.dolphinscheduler.alert.AlertServer diff --git a/dolphinscheduler-api/src/main/bin/jvm_args_env.sh b/dolphinscheduler-api/src/main/bin/jvm_args_env.sh new file mode 100644 index 0000000000..d6ccc3c571 --- /dev/null +++ b/dolphinscheduler-api/src/main/bin/jvm_args_env.sh @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +-Xms1g +-Xmx1g +-Xmn512m + +-XX:+IgnoreUnrecognizedVMOptions +-XX:+PrintGCDateStamps +-XX:+PrintGCDetails +-Xloggc:gc.log + +-XX:+HeapDumpOnOutOfMemoryError +-XX:HeapDumpPath=dump.hprof + +-Duser.timezone=${SPRING_JACKSON_TIME_ZONE} \ No newline at end of file diff --git a/dolphinscheduler-api/src/main/bin/start.sh b/dolphinscheduler-api/src/main/bin/start.sh index 93def0882f..a0ecaa0819 100644 --- a/dolphinscheduler-api/src/main/bin/start.sh +++ b/dolphinscheduler-api/src/main/bin/start.sh @@ -21,12 +21,27 @@ DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)} source "$DOLPHINSCHEDULER_HOME/conf/dolphinscheduler_env.sh" -JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+IgnoreUnrecognizedVMOptions -XX:+PrintGCDateStamps -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} +JVM_ARGS_ENV_FILE=${BIN_DIR}/jvm_args_env.sh +JVM_ARGS="-server" + +if [ -f $JVM_ARGS_ENV_FILE ]; then + while read line + do + if [[ "$line" == -* ]]; then + JVM_ARGS="${JVM_ARGS} $line" + fi + done < $JVM_ARGS_ENV_FILE +fi + +JAVA_OPTS=${JAVA_OPTS:-"${JVM_ARGS}"} if [[ "$DOCKER" == "true" ]]; then JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport" fi +echo "JAVA_HOME=${JAVA_HOME}" +echo "JAVA_OPTS=${JAVA_OPTS}" + $JAVA_HOME/bin/java $JAVA_OPTS \ -cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \ org.apache.dolphinscheduler.api.ApiApplicationServer diff --git a/dolphinscheduler-master/src/main/bin/jvm_args_env.sh b/dolphinscheduler-master/src/main/bin/jvm_args_env.sh new file mode 100644 index 0000000000..c831da7ac7 --- /dev/null +++ b/dolphinscheduler-master/src/main/bin/jvm_args_env.sh @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +-Xms4g +-Xmx4g +-Xmn2g + +-XX:+IgnoreUnrecognizedVMOptions +-XX:+PrintGCDateStamps +-XX:+PrintGCDetails +-Xloggc:gc.log + +-XX:+HeapDumpOnOutOfMemoryError +-XX:HeapDumpPath=dump.hprof + +-Duser.timezone=${SPRING_JACKSON_TIME_ZONE} diff --git a/dolphinscheduler-master/src/main/bin/start.sh b/dolphinscheduler-master/src/main/bin/start.sh index d00ad19f70..9ffc834f84 100644 --- a/dolphinscheduler-master/src/main/bin/start.sh +++ b/dolphinscheduler-master/src/main/bin/start.sh @@ -21,12 +21,27 @@ DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)} source "$DOLPHINSCHEDULER_HOME/conf/dolphinscheduler_env.sh" -JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms4g -Xmx4g -Xmn2g -XX:+IgnoreUnrecognizedVMOptions -XX:+PrintGCDateStamps -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} +JVM_ARGS_ENV_FILE=${BIN_DIR}/jvm_args_env.sh +JVM_ARGS="-server" + +if [ -f $JVM_ARGS_ENV_FILE ]; then + while read line + do + if [[ "$line" == -* ]]; then + JVM_ARGS="${JVM_ARGS} $line" + fi + done < $JVM_ARGS_ENV_FILE +fi + +JAVA_OPTS=${JAVA_OPTS:-"${JVM_ARGS}"} if [[ "$DOCKER" == "true" ]]; then JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport" fi +echo "JAVA_HOME=${JAVA_HOME}" +echo "JAVA_OPTS=${JAVA_OPTS}" + $JAVA_HOME/bin/java $JAVA_OPTS \ -cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \ org.apache.dolphinscheduler.server.master.MasterServer diff --git a/dolphinscheduler-master/src/main/docker/Dockerfile b/dolphinscheduler-master/src/main/docker/Dockerfile index 4d9d12843a..6699eb9d20 100644 --- a/dolphinscheduler-master/src/main/docker/Dockerfile +++ b/dolphinscheduler-master/src/main/docker/Dockerfile @@ -25,6 +25,6 @@ WORKDIR $DOLPHINSCHEDULER_HOME ADD ./target/master-server $DOLPHINSCHEDULER_HOME -EXPOSE 12345 +EXPOSE 5678 5679 CMD [ "/bin/bash", "./bin/start.sh" ] diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcClient.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcClient.java index 413523d527..225b27b060 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcClient.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcClient.java @@ -38,7 +38,7 @@ public class MasterRpcClient { public MasterRpcClient() { client = new NettyRemotingClient(new NettyClientConfig()); - log.info("Success initialized ApiServerRPCClient..."); + log.info("Success initialized MasterRPCClient..."); } public Message sendSyncCommand(@NonNull Host host, diff --git a/dolphinscheduler-standalone-server/src/main/bin/jvm_args_env.sh b/dolphinscheduler-standalone-server/src/main/bin/jvm_args_env.sh new file mode 100644 index 0000000000..8bb0a623f9 --- /dev/null +++ b/dolphinscheduler-standalone-server/src/main/bin/jvm_args_env.sh @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +-Xms4g +-Xmx4g +-Xmn2g + +-XX:+IgnoreUnrecognizedVMOptions +-XX:+PrintGCDateStamps +-XX:+PrintGCDetails +-Xloggc:gc.log + +-XX:+HeapDumpOnOutOfMemoryError +-XX:HeapDumpPath=dump.hprof + +-Duser.timezone=${SPRING_JACKSON_TIME_ZONE} \ No newline at end of file diff --git a/dolphinscheduler-standalone-server/src/main/bin/start.sh b/dolphinscheduler-standalone-server/src/main/bin/start.sh index f6591ee984..332c15e170 100755 --- a/dolphinscheduler-standalone-server/src/main/bin/start.sh +++ b/dolphinscheduler-standalone-server/src/main/bin/start.sh @@ -22,7 +22,19 @@ DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)} export DATABASE=${DATABASE:-h2} source "$DOLPHINSCHEDULER_HOME/conf/dolphinscheduler_env.sh" -JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} +JVM_ARGS_ENV_FILE=${BIN_DIR}/jvm_args_env.sh +JVM_ARGS="-server" + +if [ -f $JVM_ARGS_ENV_FILE ]; then + while read line + do + if [[ "$line" == -* ]]; then + JVM_ARGS="${JVM_ARGS} $line" + fi + done < $JVM_ARGS_ENV_FILE +fi + +JAVA_OPTS=${JAVA_OPTS:-"${JVM_ARGS}"} if [[ "$DOCKER" == "true" ]]; then JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport" diff --git a/dolphinscheduler-worker/src/main/bin/jvm_args_env.sh b/dolphinscheduler-worker/src/main/bin/jvm_args_env.sh new file mode 100644 index 0000000000..8bb0a623f9 --- /dev/null +++ b/dolphinscheduler-worker/src/main/bin/jvm_args_env.sh @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +-Xms4g +-Xmx4g +-Xmn2g + +-XX:+IgnoreUnrecognizedVMOptions +-XX:+PrintGCDateStamps +-XX:+PrintGCDetails +-Xloggc:gc.log + +-XX:+HeapDumpOnOutOfMemoryError +-XX:HeapDumpPath=dump.hprof + +-Duser.timezone=${SPRING_JACKSON_TIME_ZONE} \ No newline at end of file diff --git a/dolphinscheduler-worker/src/main/bin/start.sh b/dolphinscheduler-worker/src/main/bin/start.sh index 289185434b..d9bcdb132e 100644 --- a/dolphinscheduler-worker/src/main/bin/start.sh +++ b/dolphinscheduler-worker/src/main/bin/start.sh @@ -23,12 +23,27 @@ source "$DOLPHINSCHEDULER_HOME/conf/dolphinscheduler_env.sh" export DOLPHINSCHEDULER_WORK_HOME=${DOLPHINSCHEDULER_HOME} -JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms4g -Xmx4g -Xmn2g -XX:+IgnoreUnrecognizedVMOptions -XX:+PrintGCDateStamps -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} +JVM_ARGS_ENV_FILE=${BIN_DIR}/jvm_args_env.sh +JVM_ARGS="-server" + +if [ -f $JVM_ARGS_ENV_FILE ]; then + while read line + do + if [[ "$line" == -* ]]; then + JVM_ARGS="${JVM_ARGS} $line" + fi + done < $JVM_ARGS_ENV_FILE +fi + +JAVA_OPTS=${JAVA_OPTS:-"${JVM_ARGS}"} if [[ "$DOCKER" == "true" ]]; then JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport" fi +echo "JAVA_HOME=${JAVA_HOME}" +echo "JAVA_OPTS=${JAVA_OPTS}" + $JAVA_HOME/bin/java $JAVA_OPTS \ -cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \ org.apache.dolphinscheduler.server.worker.WorkerServer