From 3744167d52a52892b5ff98d3b07ab250970fdb5d Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Mon, 3 Aug 2020 10:31:27 +0800 Subject: [PATCH 1/2] [Fix-3364]Running the spark task fail because the can't find the jar (#3378) --- .../common/process/ResourceInfo.java | 16 ++++++++++++++++ .../consumer/TaskPriorityQueueConsumer.java | 2 +- .../service/process/ProcessService.java | 4 +++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ResourceInfo.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ResourceInfo.java index a7fc0839eb..c460354fe5 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ResourceInfo.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ResourceInfo.java @@ -43,5 +43,21 @@ public class ResourceInfo { this.res = res; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ResourceInfo that = (ResourceInfo) o; + + if (id != that.id) return false; + return res.equals(that.res); + } + + @Override + public int hashCode() { + int result = id; + result = 31 * result + res.hashCode(); + return result; + } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java index b1681bc3f3..1a3bf6d02e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java @@ -385,7 +385,7 @@ public class TaskPriorityQueueConsumer extends Thread{ // get the resource id in order to get the resource names in batch Stream resourceIdStream = projectResourceFiles.stream().map(resourceInfo -> resourceInfo.getId()); - Set resourceIdsSet = resourceIdStream.collect(Collectors.toSet()); + Set resourceIdsSet = resourceIdStream.filter(resId-> resId != 0).collect(Collectors.toSet()); if (CollectionUtils.isNotEmpty(resourceIdsSet)) { Integer[] resourceIds = resourceIdsSet.toArray(new Integer[resourceIdsSet.size()]); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index 617826cd32..40101fc3bc 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -1554,7 +1554,9 @@ public class ProcessService { * @return tenant code */ public String queryTenantCodeByResName(String resName,ResourceType resourceType){ - return resourceMapper.queryTenantCodeByResourceName(resName, resourceType.ordinal()); + // in order to query tenant code successful although the version is older + String fullName = resName.startsWith("/") ? resName : String.format("/%s",resName); + return resourceMapper.queryTenantCodeByResourceName(fullName, resourceType.ordinal()); } /** From e80e57ab07865cdbed88a78e48679ce48e3db72b Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Mon, 3 Aug 2020 10:39:51 +0800 Subject: [PATCH 2/2] [Feature] JVM parameter optimization , related issue #3370 (#3373) * [Feature] JVM parameter optimization , related issue #3370 * [Feature] JVM parameter optimization , related issue #3370 Co-authored-by: qiaozhanwei --- script/dolphinscheduler-daemon.sh | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index b0437a30c2..09f3fe838b 100644 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -46,7 +46,6 @@ export DOLPHINSCHEDULER_LOG_DIR=$DOLPHINSCHEDULER_HOME/logs export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/* -export DOLPHINSCHEDULER_OPTS="-server -Xmx16g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" export STOP_TIMEOUT=5 if [ ! -d "$DOLPHINSCHEDULER_LOG_DIR" ]; then @@ -59,24 +58,41 @@ pid=$DOLPHINSCHEDULER_PID_DIR/dolphinscheduler-$command.pid cd $DOLPHINSCHEDULER_HOME if [ "$command" = "api-server" ]; then + HEAP_INITIAL_SIZE=1g + HEAP_MAX_SIZE=1g + HEAP_NEW_GENERATION__SIZE=500m LOG_FILE="-Dlogging.config=classpath:logback-api.xml -Dspring.profiles.active=api" CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer elif [ "$command" = "master-server" ]; then + HEAP_INITIAL_SIZE=4g + HEAP_MAX_SIZE=4g + HEAP_NEW_GENERATION__SIZE=2g LOG_FILE="-Dlogging.config=classpath:logback-master.xml -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.server.master.MasterServer elif [ "$command" = "worker-server" ]; then + HEAP_INITIAL_SIZE=2g + HEAP_MAX_SIZE=2g + HEAP_NEW_GENERATION__SIZE=1g LOG_FILE="-Dlogging.config=classpath:logback-worker.xml -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer elif [ "$command" = "alert-server" ]; then + HEAP_INITIAL_SIZE=1g + HEAP_MAX_SIZE=1g + HEAP_NEW_GENERATION__SIZE=500m LOG_FILE="-Dlogback.configurationFile=conf/logback-alert.xml" CLASS=org.apache.dolphinscheduler.alert.AlertServer elif [ "$command" = "logger-server" ]; then + HEAP_INITIAL_SIZE=1g + HEAP_MAX_SIZE=1g + HEAP_NEW_GENERATION__SIZE=500m CLASS=org.apache.dolphinscheduler.server.log.LoggerServer else echo "Error: No command named \`$command' was found." exit 1 fi +export DOLPHINSCHEDULER_OPTS="-server -Xms$HEAP_INITIAL_SIZE -Xmx$HEAP_MAX_SIZE -Xmn$HEAP_NEW_GENERATION__SIZE -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof" + case $startStop in (start) [ -w "$DOLPHINSCHEDULER_PID_DIR" ] || mkdir -p "$DOLPHINSCHEDULER_PID_DIR"