From 6e656ca1099d6dd1451e71bb84c0fb47f6b7fd2a Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Sat, 1 Aug 2020 15:15:58 +0800 Subject: [PATCH 1/9] [Feature] JVM parameter optimization , related issue #3370 --- script/dolphinscheduler-daemon.sh | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index b0437a30c2..7ffc84f73e 100644 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -46,7 +46,6 @@ export DOLPHINSCHEDULER_LOG_DIR=$DOLPHINSCHEDULER_HOME/logs export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/* -export DOLPHINSCHEDULER_OPTS="-server -Xmx16g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" export STOP_TIMEOUT=5 if [ ! -d "$DOLPHINSCHEDULER_LOG_DIR" ]; then @@ -59,24 +58,37 @@ pid=$DOLPHINSCHEDULER_PID_DIR/dolphinscheduler-$command.pid cd $DOLPHINSCHEDULER_HOME if [ "$command" = "api-server" ]; then + HEAP_INITIAL_SIZE=1g + HEAP_MAX_SIZE=1g + HEAP_NEW_GENERATION__SIZE=500m LOG_FILE="-Dlogging.config=classpath:logback-api.xml -Dspring.profiles.active=api" CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer elif [ "$command" = "master-server" ]; then - LOG_FILE="-Dlogging.config=classpath:logback-master.xml -Ddruid.mysql.usePingMethod=false" - CLASS=org.apache.dolphinscheduler.server.master.MasterServer + HEAP_INITIAL_SIZE=4g elif [ "$command" = "worker-server" ]; then + HEAP_INITIAL_SIZE=2g + HEAP_MAX_SIZE=2g + HEAP_NEW_GENERATION__SIZE=1g LOG_FILE="-Dlogging.config=classpath:logback-worker.xml -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer elif [ "$command" = "alert-server" ]; then + HEAP_INITIAL_SIZE=1g + HEAP_MAX_SIZE=1g + HEAP_NEW_GENERATION__SIZE=500m LOG_FILE="-Dlogback.configurationFile=conf/logback-alert.xml" CLASS=org.apache.dolphinscheduler.alert.AlertServer elif [ "$command" = "logger-server" ]; then + HEAP_INITIAL_SIZE=1g + HEAP_MAX_SIZE=1g + HEAP_NEW_GENERATION__SIZE=500m CLASS=org.apache.dolphinscheduler.server.log.LoggerServer else echo "Error: No command named \`$command' was found." exit 1 fi +export DOLPHINSCHEDULER_OPTS="-server -Xms$HEAP_INITIAL_SIZE -Xmx$HEAP_MAX_SIZE -Xmn$HEAP_NEW_GENERATION__SIZE -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof" + case $startStop in (start) [ -w "$DOLPHINSCHEDULER_PID_DIR" ] || mkdir -p "$DOLPHINSCHEDULER_PID_DIR" From 07a1c0a7b4228901c98f2e5ec5a1f0922eba423e Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Sat, 1 Aug 2020 15:21:01 +0800 Subject: [PATCH 2/9] [Feature] JVM parameter optimization , related issue #3370 --- script/dolphinscheduler-daemon.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index 7ffc84f73e..09f3fe838b 100644 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -65,6 +65,10 @@ if [ "$command" = "api-server" ]; then CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer elif [ "$command" = "master-server" ]; then HEAP_INITIAL_SIZE=4g + HEAP_MAX_SIZE=4g + HEAP_NEW_GENERATION__SIZE=2g + LOG_FILE="-Dlogging.config=classpath:logback-master.xml -Ddruid.mysql.usePingMethod=false" + CLASS=org.apache.dolphinscheduler.server.master.MasterServer elif [ "$command" = "worker-server" ]; then HEAP_INITIAL_SIZE=2g HEAP_MAX_SIZE=2g From b4b8c801ede82f5138c48585feecf5fd37ee05e6 Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Fri, 14 Aug 2020 17:24:07 +0800 Subject: [PATCH 3/9] test release 1.3.2 version rollback --- dolphinscheduler-alert/pom.xml | 2 +- dolphinscheduler-api/pom.xml | 2 +- dolphinscheduler-common/pom.xml | 2 +- dolphinscheduler-common/src/main/resources/common.properties | 4 ++-- dolphinscheduler-dao/pom.xml | 2 +- dolphinscheduler-dist/pom.xml | 2 +- dolphinscheduler-plugin-api/pom.xml | 2 +- dolphinscheduler-remote/pom.xml | 2 +- dolphinscheduler-server/pom.xml | 2 +- dolphinscheduler-service/pom.xml | 2 +- dolphinscheduler-ui/pom.xml | 2 +- pom.xml | 2 +- 12 files changed, 13 insertions(+), 13 deletions(-) diff --git a/dolphinscheduler-alert/pom.xml b/dolphinscheduler-alert/pom.xml index a416f2f08e..5958bfbb8f 100644 --- a/dolphinscheduler-alert/pom.xml +++ b/dolphinscheduler-alert/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.3-SNAPSHOT + 1.3.2-SNAPSHOT dolphinscheduler-alert ${project.artifactId} diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index c49e186198..5c822ee362 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.3-SNAPSHOT + 1.3.2-SNAPSHOT dolphinscheduler-api ${project.artifactId} diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index 86e9de6cf8..290fa91b54 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.3-SNAPSHOT + 1.3.2-SNAPSHOT dolphinscheduler-common dolphinscheduler-common diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index ef49a1962f..6e74964d87 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -16,7 +16,7 @@ # # resource storage type : HDFS,S3,NONE -resource.storage.type=NONE +resource.storage.type=HDFS # resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended resource.upload.path=/dolphinscheduler @@ -43,7 +43,7 @@ login.user.keytab.path=/opt/hdfs.headless.keytab hdfs.root.user=hdfs # if resource.storage.type=S3，the value like: s3a://dolphinscheduler ; if resource.storage.type=HDFS, When namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir -fs.defaultFS=hdfs://mycluster:8020 +fs.defaultFS=hdfs://192.168.220.188:8020 # if resource.storage.type=S3，s3 endpoint fs.s3a.endpoint=http://192.168.xx.xx:9010 diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml index 8ef43b51b6..c97b52d201 100644 --- a/dolphinscheduler-dao/pom.xml +++ b/dolphinscheduler-dao/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.3-SNAPSHOT + 1.3.2-SNAPSHOT dolphinscheduler-dao ${project.artifactId} diff --git a/dolphinscheduler-dist/pom.xml b/dolphinscheduler-dist/pom.xml index 8accc18f4c..ca3834e90b 100644 --- a/dolphinscheduler-dist/pom.xml +++ b/dolphinscheduler-dist/pom.xml @@ -20,7 +20,7 @@ dolphinscheduler org.apache.dolphinscheduler - 1.3.3-SNAPSHOT + 1.3.2-SNAPSHOT 4.0.0 diff --git a/dolphinscheduler-plugin-api/pom.xml b/dolphinscheduler-plugin-api/pom.xml index b5963a0ea3..2bffba811d 100644 --- a/dolphinscheduler-plugin-api/pom.xml +++ b/dolphinscheduler-plugin-api/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.3-SNAPSHOT + 1.3.2-SNAPSHOT dolphinscheduler-plugin-api ${project.artifactId} diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml index 7a6aef783e..dcc57633bb 100644 --- a/dolphinscheduler-remote/pom.xml +++ b/dolphinscheduler-remote/pom.xml @@ -19,7 +19,7 @@ dolphinscheduler org.apache.dolphinscheduler - 1.3.3-SNAPSHOT + 1.3.2-SNAPSHOT 4.0.0 diff --git a/dolphinscheduler-server/pom.xml b/dolphinscheduler-server/pom.xml index f32207bc77..4cbce0ab47 100644 --- a/dolphinscheduler-server/pom.xml +++ b/dolphinscheduler-server/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.3-SNAPSHOT + 1.3.2-SNAPSHOT dolphinscheduler-server dolphinscheduler-server diff --git a/dolphinscheduler-service/pom.xml b/dolphinscheduler-service/pom.xml index 4e0bdbb983..3c4ea53ebb 100644 --- a/dolphinscheduler-service/pom.xml +++ b/dolphinscheduler-service/pom.xml @@ -19,7 +19,7 @@ dolphinscheduler org.apache.dolphinscheduler - 1.3.3-SNAPSHOT + 1.3.2-SNAPSHOT 4.0.0 diff --git a/dolphinscheduler-ui/pom.xml b/dolphinscheduler-ui/pom.xml index 53b872929b..13644bad91 100644 --- a/dolphinscheduler-ui/pom.xml +++ b/dolphinscheduler-ui/pom.xml @@ -20,7 +20,7 @@ dolphinscheduler org.apache.dolphinscheduler - 1.3.3-SNAPSHOT + 1.3.2-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index f13cf9a05d..80dceb71f6 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ 4.0.0 org.apache.dolphinscheduler dolphinscheduler - 1.3.3-SNAPSHOT + 1.3.2-SNAPSHOT pom ${project.artifactId} http://dolphinscheduler.apache.org From 3191bfd19c2d53f8f95752f3ff5e73aab6a4dfe1 Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Fri, 14 Aug 2020 17:27:03 +0800 Subject: [PATCH 4/9] test release 1.3.2 version rollback --- dolphinscheduler-common/src/main/resources/common.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index 6e74964d87..ef49a1962f 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -16,7 +16,7 @@ # # resource storage type : HDFS,S3,NONE -resource.storage.type=HDFS +resource.storage.type=NONE # resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended resource.upload.path=/dolphinscheduler @@ -43,7 +43,7 @@ login.user.keytab.path=/opt/hdfs.headless.keytab hdfs.root.user=hdfs # if resource.storage.type=S3，the value like: s3a://dolphinscheduler ; if resource.storage.type=HDFS, When namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir -fs.defaultFS=hdfs://192.168.220.188:8020 +fs.defaultFS=hdfs://mycluster:8020 # if resource.storage.type=S3，s3 endpoint fs.s3a.endpoint=http://192.168.xx.xx:9010 From e26b654ee68a428c2b9d88a2d7fcbc9746ad25e8 Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Fri, 14 Aug 2020 18:16:22 +0800 Subject: [PATCH 5/9] test --- .../org/apache/dolphinscheduler/api/Demo.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/Demo.java diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/Demo.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/Demo.java new file mode 100644 index 0000000000..3c4b0cf556 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/Demo.java @@ -0,0 +1,18 @@ +package org.apache.dolphinscheduler.api; + +import java.io.FileWriter; +import java.io.Writer; +import java.text.SimpleDateFormat; +import java.util.Date; + +public class Demo { + + public static void main(String[] args) throws Exception{ + Writer writer = new FileWriter("D:\\demo.txt"); + + for (int i = 0 ; i < 3000; i++){ + writer.append("echo 日志打印测试" + (i+1) + "\r\n"); + } + writer.close(); + } +} From ed6f5791fe9da79b0349db1718083d726641cd7c Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Fri, 14 Aug 2020 18:17:05 +0800 Subject: [PATCH 6/9] test release 1.3.2 version rollback --- .../org/apache/dolphinscheduler/api/Demo.java | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/Demo.java diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/Demo.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/Demo.java deleted file mode 100644 index 3c4b0cf556..0000000000 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/Demo.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.apache.dolphinscheduler.api; - -import java.io.FileWriter; -import java.io.Writer; -import java.text.SimpleDateFormat; -import java.util.Date; - -public class Demo { - - public static void main(String[] args) throws Exception{ - Writer writer = new FileWriter("D:\\demo.txt"); - - for (int i = 0 ; i < 3000; i++){ - writer.append("echo 日志打印测试" + (i+1) + "\r\n"); - } - writer.close(); - } -} From 2ed5f062c16cb5e646290f9468517daf40bd7810 Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Mon, 17 Aug 2020 11:26:42 +0800 Subject: [PATCH 7/9] modify tag 1.3.0 to HEAD --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 80dceb71f6..7d9f79ccd1 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,7 @@ scm:git:https://github.com/apache/incubator-dolphinscheduler.git scm:git:https://github.com/apache/incubator-dolphinscheduler.git https://github.com/apache/incubator-dolphinscheduler - 1.3.0 + HEAD From 94e453cbd50f1ea72d52040d6edf9c33ed13e74d Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Tue, 18 Aug 2020 10:14:10 +0800 Subject: [PATCH 8/9] modify general user can't create token --- .../dolphinscheduler/api/service/AccessTokenService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java index 8b40a25743..8ccbff1e2f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java @@ -120,9 +120,11 @@ public class AccessTokenService extends BaseService { */ public Map generateToken(User loginUser, int userId, String expireTime) { Map result = new HashMap<>(5); - if(check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)){ + if (!hasPerm(loginUser,userId)){ + putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } + String token = EncryptionUtils.getMd5(userId + expireTime + String.valueOf(System.currentTimeMillis())); result.put(Constants.DATA_LIST, token); putMsg(result, Status.SUCCESS); From b4974241c71715eba9904ab91116f3288d21e4e6 Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Tue, 18 Aug 2020 10:57:25 +0800 Subject: [PATCH 9/9] if task is null , set task type is null instead of "null" --- .../dolphinscheduler/api/service/ProcessDefinitionService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index 19b38ea46f..91ea4f9bd0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -1305,7 +1305,7 @@ public class ProcessDefinitionService extends BaseDAGService { ProcessInstance processInstance = processInstanceList.get(i); TaskInstance taskInstance = taskInstanceMapper.queryByInstanceIdAndName(processInstance.getId(), nodeName); if (taskInstance == null) { - treeViewDto.getInstances().add(new Instance(-1, "not running", "null")); + treeViewDto.getInstances().add(new Instance(-1, "not running", null)); } else { Date startTime = taskInstance.getStartTime() == null ? new Date() : taskInstance.getStartTime(); Date endTime = taskInstance.getEndTime() == null ? new Date() : taskInstance.getEndTime();