From 6d1b45c14a77f439d0353a41d8f3852c2acffb96 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Sat, 1 Aug 2020 16:45:40 +0800 Subject: [PATCH 1/6] Restore constants to variables --- .../pages/dag/_source/plugIn/jsPlumbHandle.js | 11 +++++------ .../conf/home/pages/dag/_source/plugIn/util.js | 16 ++++++++-------- .../src/js/conf/home/store/dag/mutations.js | 4 ++-- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js index fc91f6ebd4..3e2d7d944e 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js @@ -478,9 +478,9 @@ JSP.prototype.handleEventRemove = function () { JSP.prototype.removeNodes = function ($id) { // Delete node processing(data-targetarr) _.map(tasksAll(), v => { - const targetarr = v.targetarr.split(',') + let targetarr = v.targetarr.split(',') if (targetarr.length) { - const newArr = _.filter(targetarr, v1 => v1 !== $id) + let newArr = _.filter(targetarr, v1 => v1 !== $id) $(`#${v.id}`).attr('data-targetarr', newArr.toString()) } }) @@ -496,8 +496,7 @@ JSP.prototype.removeNodes = function ($id) { _.map(this.JspInstance.getConnections(), v => { connects.push({ endPointSourceId: v.sourceId, - endPointTargetId: v.targetId, - label: v._jsPlumb.overlays.label.canvas.innerText + endPointTargetId: v.targetId }) }) // Storage line dependence @@ -512,8 +511,8 @@ JSP.prototype.removeConnect = function ($connect) { return } // Remove connections and remove node and node dependencies - const targetId = $connect.targetId - const sourceId = $connect.sourceId + let targetId = $connect.targetId + let sourceId = $connect.sourceId let targetarr = rtTargetarrArr(targetId) if (targetarr.length) { targetarr = _.filter(targetarr, v => v !== sourceId) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js index 0b472b6c10..e4c78d29fb 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js @@ -23,7 +23,7 @@ import store from '@/conf/home/store' * Node, to array */ const rtTargetarrArr = (id) => { - const ids = $(`#${id}`).attr('data-targetarr') + let ids = $(`#${id}`).attr('data-targetarr') return ids ? ids.split(',') : [] } @@ -31,8 +31,8 @@ const rtTargetarrArr = (id) => { * Store node id to targetarr */ const saveTargetarr = (valId, domId) => { - const $target = $(`#${domId}`) - const targetStr = $target.attr('data-targetarr') ? $target.attr('data-targetarr') + `,${valId}` : `${valId}` + let $target = $(`#${domId}`) + let targetStr = $target.attr('data-targetarr') ? $target.attr('data-targetarr') + `,${valId}` : `${valId}` $target.attr('data-targetarr', targetStr) } @@ -66,9 +66,9 @@ const rtTasksTpl = ({ id, name, x, y, targetarr, isAttachment, taskType, runFlag * Get all tasks nodes */ const tasksAll = () => { - const a = [] + let a = [] $('#canvas .w').each(function (idx, elem) { - const e = $(elem) + let e = $(elem) a.push({ id: e.attr('id'), name: e.find('.name-p').text(), @@ -117,10 +117,10 @@ const setSvgColor = (e, color) => { * Get all node ids */ const allNodesId = () => { - const idArr = [] + let idArr = [] $('.w').each((i, o) => { - const $obj = $(o) - const $span = $obj.find('.name-p').text() + let $obj = $(o) + let $span = $obj.find('.name-p').text() if ($span) { idArr.push({ id: $obj.attr('id'), diff --git a/dolphinscheduler-ui/src/js/conf/home/store/dag/mutations.js b/dolphinscheduler-ui/src/js/conf/home/store/dag/mutations.js index 509ae3c103..cbd828fe01 100755 --- a/dolphinscheduler-ui/src/js/conf/home/store/dag/mutations.js +++ b/dolphinscheduler-ui/src/js/conf/home/store/dag/mutations.js @@ -120,7 +120,7 @@ export default { * object {} */ addTasks (state, payload) { - const i = _.findIndex(state.tasks, v => v.id === payload.id) + let i = _.findIndex(state.tasks, v => v.id === payload.id) if (i !== -1) { state.tasks[i] = Object.assign(state.tasks[i], {}, payload) } else { @@ -131,7 +131,7 @@ export default { } else { state.cacheTasks[payload.id] = payload } - const dom = $(`#${payload.id}`) + let dom = $(`#${payload.id}`) state.locations[payload.id] = _.assign(state.locations[payload.id], { name: dom.find('.name-p').text(), targetarr: dom.attr('data-targetarr'), From 72153cc88572861d7648e2063737b4f71c1b9281 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Sat, 1 Aug 2020 17:51:02 +0800 Subject: [PATCH 2/6] i18n format modification --- .../src/js/conf/home/pages/dag/_source/formModel/formModel.vue | 2 +- dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index 9db5483943..29c4cf664b 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -38,7 +38,7 @@ type="text" v-model="name" :disabled="isDetails" - :placeholder="$t('Please enter name(required)')" + :placeholder="$t('Please enter name (required)')" maxlength="100" @on-blur="_verifName()" autocomplete="off"> diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js index f84f41b023..c38eac25e5 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -34,7 +34,7 @@ export default { 'View log': '查看日志', 'Enter this child node': '进入该子节点', 'Node name': '节点名称', - 'Please enter name(required)': '请输入名称(必填)', + 'Please enter name (required)': '请输入名称(必填)', 'Run flag': '运行标志', Normal: '正常', 'Prohibition execution': '禁止执行', From 340d409757be5b505764cb53c8f4bfc8516dd99d Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Sat, 1 Aug 2020 18:17:56 +0800 Subject: [PATCH 3/6] fix --- .../src/js/conf/home/pages/dag/_source/udp/udp.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue index b9610cbc7b..4acd550f5f 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue @@ -26,7 +26,7 @@ type="text" v-model="name" :disabled="router.history.current.name === 'projects-instance-details'" - :placeholder="$t('Please enter name(required)')"> + :placeholder="$t('Please enter name (required)')"> From e174506cbddaf4615d2d1818b330f6dcf56ba019 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Sat, 1 Aug 2020 18:18:26 +0800 Subject: [PATCH 4/6] fix --- dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js | 1 - 1 file changed, 1 deletion(-) diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js index 504534eb34..009f2bdac2 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js @@ -34,7 +34,6 @@ export default { 'View log': 'View log', 'Enter this child node': 'Enter this child node', 'Node name': 'Node name', - 'Please enter name(required)': 'Please enter name(required)', 'Run flag': 'Run flag', Normal: 'Normal', 'Prohibition execution': 'Prohibition execution', From 3744167d52a52892b5ff98d3b07ab250970fdb5d Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Mon, 3 Aug 2020 10:31:27 +0800 Subject: [PATCH 5/6] [Fix-3364]Running the spark task fail because the can't find the jar (#3378) --- .../common/process/ResourceInfo.java | 16 ++++++++++++++++ .../consumer/TaskPriorityQueueConsumer.java | 2 +- .../service/process/ProcessService.java | 4 +++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ResourceInfo.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ResourceInfo.java index a7fc0839eb..c460354fe5 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ResourceInfo.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ResourceInfo.java @@ -43,5 +43,21 @@ public class ResourceInfo { this.res = res; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ResourceInfo that = (ResourceInfo) o; + + if (id != that.id) return false; + return res.equals(that.res); + } + + @Override + public int hashCode() { + int result = id; + result = 31 * result + res.hashCode(); + return result; + } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java index b1681bc3f3..1a3bf6d02e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java @@ -385,7 +385,7 @@ public class TaskPriorityQueueConsumer extends Thread{ // get the resource id in order to get the resource names in batch Stream resourceIdStream = projectResourceFiles.stream().map(resourceInfo -> resourceInfo.getId()); - Set resourceIdsSet = resourceIdStream.collect(Collectors.toSet()); + Set resourceIdsSet = resourceIdStream.filter(resId-> resId != 0).collect(Collectors.toSet()); if (CollectionUtils.isNotEmpty(resourceIdsSet)) { Integer[] resourceIds = resourceIdsSet.toArray(new Integer[resourceIdsSet.size()]); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index 617826cd32..40101fc3bc 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -1554,7 +1554,9 @@ public class ProcessService { * @return tenant code */ public String queryTenantCodeByResName(String resName,ResourceType resourceType){ - return resourceMapper.queryTenantCodeByResourceName(resName, resourceType.ordinal()); + // in order to query tenant code successful although the version is older + String fullName = resName.startsWith("/") ? resName : String.format("/%s",resName); + return resourceMapper.queryTenantCodeByResourceName(fullName, resourceType.ordinal()); } /** From e80e57ab07865cdbed88a78e48679ce48e3db72b Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Mon, 3 Aug 2020 10:39:51 +0800 Subject: [PATCH 6/6] [Feature] JVM parameter optimization , related issue #3370 (#3373) * [Feature] JVM parameter optimization , related issue #3370 * [Feature] JVM parameter optimization , related issue #3370 Co-authored-by: qiaozhanwei --- script/dolphinscheduler-daemon.sh | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index b0437a30c2..09f3fe838b 100644 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -46,7 +46,6 @@ export DOLPHINSCHEDULER_LOG_DIR=$DOLPHINSCHEDULER_HOME/logs export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/* -export DOLPHINSCHEDULER_OPTS="-server -Xmx16g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" export STOP_TIMEOUT=5 if [ ! -d "$DOLPHINSCHEDULER_LOG_DIR" ]; then @@ -59,24 +58,41 @@ pid=$DOLPHINSCHEDULER_PID_DIR/dolphinscheduler-$command.pid cd $DOLPHINSCHEDULER_HOME if [ "$command" = "api-server" ]; then + HEAP_INITIAL_SIZE=1g + HEAP_MAX_SIZE=1g + HEAP_NEW_GENERATION__SIZE=500m LOG_FILE="-Dlogging.config=classpath:logback-api.xml -Dspring.profiles.active=api" CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer elif [ "$command" = "master-server" ]; then + HEAP_INITIAL_SIZE=4g + HEAP_MAX_SIZE=4g + HEAP_NEW_GENERATION__SIZE=2g LOG_FILE="-Dlogging.config=classpath:logback-master.xml -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.server.master.MasterServer elif [ "$command" = "worker-server" ]; then + HEAP_INITIAL_SIZE=2g + HEAP_MAX_SIZE=2g + HEAP_NEW_GENERATION__SIZE=1g LOG_FILE="-Dlogging.config=classpath:logback-worker.xml -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer elif [ "$command" = "alert-server" ]; then + HEAP_INITIAL_SIZE=1g + HEAP_MAX_SIZE=1g + HEAP_NEW_GENERATION__SIZE=500m LOG_FILE="-Dlogback.configurationFile=conf/logback-alert.xml" CLASS=org.apache.dolphinscheduler.alert.AlertServer elif [ "$command" = "logger-server" ]; then + HEAP_INITIAL_SIZE=1g + HEAP_MAX_SIZE=1g + HEAP_NEW_GENERATION__SIZE=500m CLASS=org.apache.dolphinscheduler.server.log.LoggerServer else echo "Error: No command named \`$command' was found." exit 1 fi +export DOLPHINSCHEDULER_OPTS="-server -Xms$HEAP_INITIAL_SIZE -Xmx$HEAP_MAX_SIZE -Xmn$HEAP_NEW_GENERATION__SIZE -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof" + case $startStop in (start) [ -w "$DOLPHINSCHEDULER_PID_DIR" ] || mkdir -p "$DOLPHINSCHEDULER_PID_DIR"