From cd349c9c7f78ddaa8a65b91cdbbeabb407964492 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Wed, 5 Aug 2020 13:39:58 +0800 Subject: [PATCH 01/20] [Bug][ui]Fix front-end bug #3413 --- .../js/conf/home/pages/dag/_source/formModel/tasks/shell.vue | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue index 7a462a1f27..712a2081b7 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue @@ -364,9 +364,10 @@ } }, mounted () { + // Added delay loading in script input box setTimeout(() => { this._handlerEditor() - }, 200) + }, 350) }, destroyed () { if (editor) { From 21eaf51e1b8fccfab478b87478643fb1bbd03b90 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Thu, 6 Aug 2020 09:38:39 +0800 Subject: [PATCH 02/20] Optimize dag --- .../src/js/conf/home/pages/dag/_source/dag.js | 4 ++-- .../src/js/conf/home/pages/dag/_source/dag.vue | 6 +++--- .../js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js | 6 +++--- .../src/js/conf/home/pages/dag/_source/plugIn/util.js | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js index 5a41091029..fef884c387 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js @@ -118,8 +118,8 @@ Dag.prototype.toolbarEvent = function ({ item, code, is }) { Dag.prototype.backfill = function (arg) { if (arg) { let locationsValue = store.state.dag.locations - const locationsValue1 = store.state.dag.locations - const locationsValue2 = store.state.dag.locations + const locationsValue1 = _.cloneDeep(store.state.dag.locations) + const locationsValue2 = _.cloneDeep(store.state.dag.locations) const arr = [] for (const i in locationsValue1) { const objs = {} diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue index 25516bf265..a73b1cb410 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue @@ -522,9 +522,9 @@ } else { rearList = [] } - let targetarr = $(`#${id}`).attr('data-targetarr') - if (targetarr) { - let nodearr = targetarr.split(',') + let target = $(`#${id}`).attr('data-targetarr') + if (target) { + let nodearr = target.split(',') nodearr.forEach(v => { let nodeobj = {} nodeobj.value = $(`#${v}`).find('.name-p').text() diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js index 3e2d7d944e..62fe764a74 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js @@ -687,14 +687,14 @@ JSP.prototype.saveStore = function () { JSP.prototype.handleEvent = function () { this.JspInstance.bind('beforeDrop', function (info) { - const sourceId = info.sourceId// 出 - const targetId = info.targetId// 入 + let sourceId = info['sourceId']// 出 + let targetId = info['targetId']// 入 /** * Recursive search for nodes */ let recursiveVal const recursiveTargetarr = (arr, targetId) => { - for (const i in arr) { + for (let i in arr) { if (arr[i] === targetId) { recursiveVal = targetId } else { diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js index e4c78d29fb..d356ff3715 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js @@ -32,7 +32,7 @@ const rtTargetarrArr = (id) => { */ const saveTargetarr = (valId, domId) => { let $target = $(`#${domId}`) - let targetStr = $target.attr('data-targetarr') ? $target.attr('data-targetarr') + `,${valId}` : `${valId}` + let targetStr = $target.attr('data-targetarr') ? $target.attr('data-targetarr') + `,${valId}` : `${valId}`; $target.attr('data-targetarr', targetStr) } From c30713d88c8500ad65ab5aecd3c49aa64a57e68f Mon Sep 17 00:00:00 2001 From: JinyLeeChina <42576980+JinyLeeChina@users.noreply.github.com> Date: Wed, 5 Aug 2020 18:59:49 +0800 Subject: [PATCH 03/20] Update actions.js (#3401) --- .../src/js/conf/home/store/dag/actions.js | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js b/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js index f933eaede4..a549aafaa2 100644 --- a/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js +++ b/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js @@ -19,6 +19,25 @@ import _ from 'lodash' import io from '@/module/io' import { tasksState } from '@/conf/home/pages/dag/_source/config' +// delete 'definitionList' from tasks +const deleteDefinitionList = (tasks) => { + const newTasks = []; + tasks.forEach(item => { + const newItem = Object.assign({}, item); + if(newItem.dependence && newItem.dependence.dependTaskList) { + newItem.dependence.dependTaskList.forEach(dependTaskItem => { + if (dependTaskItem.dependItemList) { + dependTaskItem.dependItemList.forEach(dependItem => { + Reflect.deleteProperty(dependItem, 'definitionList'); + }) + } + }) + } + newTasks.push(newItem); + }); + return newTasks; +} + export default { /** * Task status acquisition @@ -193,7 +212,7 @@ export default { return new Promise((resolve, reject) => { const data = { globalParams: state.globalParams, - tasks: state.tasks, + tasks: deleteDefinitionList(state.tasks), tenantId: state.tenantId, timeout: state.timeout } @@ -217,7 +236,7 @@ export default { return new Promise((resolve, reject) => { const data = { globalParams: state.globalParams, - tasks: state.tasks, + tasks: deleteDefinitionList(state.tasks), tenantId: state.tenantId, timeout: state.timeout } From 0ef75493c4ec365dabcd9f98697f6b0f328fe000 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Fri, 7 Aug 2020 14:54:07 +0800 Subject: [PATCH 04/20] [Fix-3256][ui] Fix admin user info update error (#3425) (#3428) * [PROPOSAL-3139] Datasource selection changes from radio to select * [PROPOSAL-3139] Datasource selection changes from radio to select * [BUG FIX] issues #3256 * [BUG FIX] issues #3256 * [BUG FIX] issues #3256 * [Fix-3256][ui] Fix admin user info update error * [Fix-3256][ui] Fix admin user info update error * [Fix-3256][ui] Fix admin user info update error * [Fix-3256][ui] Fix admin user info update error * reset createUser.vue * [Fix-3256][ui] Fix admin user info update error Co-authored-by: dailidong Co-authored-by: wuchunfu <319355703@qq.com> Co-authored-by: dailidong --- sql/upgrade/1.2.0_schema/mysql/dolphinscheduler_dml.sql | 3 ++- sql/upgrade/1.3.0_schema/mysql/dolphinscheduler_dml.sql | 3 ++- sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_dml.sql | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/sql/upgrade/1.2.0_schema/mysql/dolphinscheduler_dml.sql b/sql/upgrade/1.2.0_schema/mysql/dolphinscheduler_dml.sql index 6aeafcd700..e2a08756a6 100644 --- a/sql/upgrade/1.2.0_schema/mysql/dolphinscheduler_dml.sql +++ b/sql/upgrade/1.2.0_schema/mysql/dolphinscheduler_dml.sql @@ -22,4 +22,5 @@ UPDATE QRTZ_FIRED_TRIGGERS SET SCHED_NAME='DolphinScheduler' WHERE SCHED_NAME='E UPDATE QRTZ_JOB_DETAILS SET SCHED_NAME='DolphinScheduler' WHERE SCHED_NAME='EasyScheduler'; UPDATE QRTZ_JOB_DETAILS SET JOB_CLASS_NAME='org.apache.dolphinscheduler.dao.quartz.ProcessScheduleJob' WHERE JOB_CLASS_NAME='cn.escheduler.server.quartz.ProcessScheduleJob'; UPDATE QRTZ_LOCKS SET SCHED_NAME='DolphinScheduler' WHERE SCHED_NAME='EasyScheduler'; -UPDATE QRTZ_SCHEDULER_STATE SET SCHED_NAME='DolphinScheduler' WHERE SCHED_NAME='EasyScheduler'; \ No newline at end of file +UPDATE QRTZ_SCHEDULER_STATE SET SCHED_NAME='DolphinScheduler' WHERE SCHED_NAME='EasyScheduler'; +UPDATE t_ds_user SET phone = '' WHERE phone = 'xx'; \ No newline at end of file diff --git a/sql/upgrade/1.3.0_schema/mysql/dolphinscheduler_dml.sql b/sql/upgrade/1.3.0_schema/mysql/dolphinscheduler_dml.sql index 6f0e145176..661ed9e827 100644 --- a/sql/upgrade/1.3.0_schema/mysql/dolphinscheduler_dml.sql +++ b/sql/upgrade/1.3.0_schema/mysql/dolphinscheduler_dml.sql @@ -23,4 +23,5 @@ UPDATE t_ds_process_instance instance SET `worker_group`=IFNULL((SELECT name fro UPDATE t_ds_task_instance instance SET `worker_group`=IFNULL((SELECT name from t_ds_worker_group WHERE instance.worker_group=CONCAT(id,'')),'default'); UPDATE t_ds_schedules schedule SET `worker_group`=IFNULL((SELECT name from t_ds_worker_group WHERE schedule.worker_group=CONCAT(id,'')),'default'); UPDATE t_ds_command command SET `worker_group`=IFNULL((SELECT name from t_ds_worker_group WHERE command.worker_group=CONCAT(id,'')),'default'); -UPDATE t_ds_error_command command SET `worker_group`=IFNULL((SELECT name from t_ds_worker_group WHERE command.worker_group=CONCAT(id,'')),'default'); \ No newline at end of file +UPDATE t_ds_error_command command SET `worker_group`=IFNULL((SELECT name from t_ds_worker_group WHERE command.worker_group=CONCAT(id,'')),'default'); +UPDATE t_ds_user SET phone = '' WHERE phone = 'xx'; \ No newline at end of file diff --git a/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_dml.sql b/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_dml.sql index fba03152ee..a748eae6cf 100644 --- a/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_dml.sql +++ b/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_dml.sql @@ -21,4 +21,5 @@ UPDATE t_ds_process_instance instance SET worker_group=COALESCE((SELECT name fro UPDATE t_ds_task_instance instance SET worker_group=COALESCE((SELECT name from t_ds_worker_group WHERE instance.worker_group=CONCAT(id,'')),'default'); UPDATE t_ds_schedules schedule SET worker_group=COALESCE((SELECT name from t_ds_worker_group WHERE schedule.worker_group=CONCAT(id,'')),'default'); UPDATE t_ds_command command SET worker_group=COALESCE((SELECT name from t_ds_worker_group WHERE command.worker_group=CONCAT(id,'')),'default'); -UPDATE t_ds_error_command command SET worker_group=COALESCE((SELECT name from t_ds_worker_group WHERE command.worker_group=CONCAT(id,'')),'default'); \ No newline at end of file +UPDATE t_ds_error_command command SET worker_group=COALESCE((SELECT name from t_ds_worker_group WHERE command.worker_group=CONCAT(id,'')),'default'); +UPDATE t_ds_user SET phone = '' WHERE phone = 'xx'; \ No newline at end of file From 2f6cf5cc411b4f5e6fc93898e172889564f7c3f7 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Fri, 7 Aug 2020 15:13:24 +0800 Subject: [PATCH 05/20] [Fix-3433][api]Fixed that release the imported process definition which version is below 1.3.0 will be failure --- .../dolphinscheduler/api/service/ProcessDefinitionService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index 881e2fed1a..19b38ea46f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -38,6 +38,7 @@ import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.process.ProcessDag; import org.apache.dolphinscheduler.common.process.Property; +import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.utils.*; @@ -176,7 +177,7 @@ public class ProcessDefinitionService extends BaseDAGService { String taskParameter = taskNode.getParams(); AbstractParameters params = TaskParametersUtils.getParameters(taskNode.getType(),taskParameter); if (CollectionUtils.isNotEmpty(params.getResourceFilesList())) { - Set tempSet = params.getResourceFilesList().stream().map(t->t.getId()).collect(Collectors.toSet()); + Set tempSet = params.getResourceFilesList().stream().filter(t->t.getId()!=0).map(ResourceInfo::getId).collect(Collectors.toSet()); resourceIds.addAll(tempSet); } } From ba43c8c3f736c105386c74453708b40d6d63f52a Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Mon, 10 Aug 2020 14:24:08 +0800 Subject: [PATCH 06/20] dag connection add check --- dolphinscheduler-ui/build/webpack.config.prod.js | 7 +------ .../home/pages/dag/_source/plugIn/jsPlumbHandle.js | 13 +++++++++++-- .../js/conf/home/pages/dag/_source/plugIn/util.js | 1 + .../src/js/module/i18n/locale/en_US.js | 3 ++- .../src/js/module/i18n/locale/zh_CN.js | 3 ++- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/dolphinscheduler-ui/build/webpack.config.prod.js b/dolphinscheduler-ui/build/webpack.config.prod.js index 4bb90d54d1..1024ac6724 100644 --- a/dolphinscheduler-ui/build/webpack.config.prod.js +++ b/dolphinscheduler-ui/build/webpack.config.prod.js @@ -51,12 +51,7 @@ const config = merge.smart(baseConfig, { minimizer: [ new TerserPlugin({ terserOptions: { - compress: { - warnings: false, - drop_console: true, - drop_debugger: true, - pure_funcs: ['console.log'] - } + compress: {} }, cache: true, parallel: true, diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js index 62fe764a74..d6b01f1778 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js @@ -36,7 +36,7 @@ import { } from './util' import mStart from '@/conf/home/pages/projects/pages/definition/pages/list/_source/start' -const JSP = function () { +let JSP = function () { this.dag = {} this.selectedElement = {} @@ -687,8 +687,17 @@ JSP.prototype.saveStore = function () { JSP.prototype.handleEvent = function () { this.JspInstance.bind('beforeDrop', function (info) { + console.log(info) let sourceId = info['sourceId']// 出 let targetId = info['targetId']// 入 + console.log(sourceId,targetId) + /** + * When connecting, connection is prohibited when the sourceId and target nodes are empty + */ + if(!sourceId && !targetId) { + Vue.$message.warning(`${i18n.$t('This canvas is abnormal and the node connection cannot be made. Please save or exit the current workflow')}`) + return false + } /** * Recursive search for nodes */ @@ -730,7 +739,7 @@ JSP.prototype.handleEvent = function () { }) } /** - * Backfill data processing + * Backfill data processingJSP */ JSP.prototype.jspBackfill = function ({ connects, locations, largeJson }) { // Backfill nodes diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js index d356ff3715..65c345921d 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js @@ -34,6 +34,7 @@ const saveTargetarr = (valId, domId) => { let $target = $(`#${domId}`) let targetStr = $target.attr('data-targetarr') ? $target.attr('data-targetarr') + `,${valId}` : `${valId}`; $target.attr('data-targetarr', targetStr) + console.log(targetStr) } const rtBantpl = () => { diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js index 170ebbc38d..5f7a707f87 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js @@ -600,5 +600,6 @@ export default { 'Directory detail': 'Directory detail', 'Connection name': 'Connection name', 'Current connection settings': 'Current connection settings', - 'Please save the DAG before formatting': 'Please save the DAG before formatting' + 'Please save the DAG before formatting': 'Please save the DAG before formatting', + 'This canvas is abnormal and the node connection cannot be made. Please save or exit the current workflow': 'This canvas is abnormal and the node connection cannot be made. Please save or exit the current workflow' } diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js index 704b447b12..5dec5096b3 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -606,5 +606,6 @@ export default { 'Directory detail': '查看目录详情', 'Connection name': '连线名', 'Current connection settings': '当前连线设置', - 'Please save the DAG before formatting': '格式化前请先保存DAG' + 'Please save the DAG before formatting': '格式化前请先保存DAG', + 'This canvas is abnormal and the node connection cannot be made. Please save or exit the current workflow': '此画布异常，无法进行节点连线，请保存或退出当前工作流' } From 100c87b9d8dfb7f01d9de137c59d2b0624e82f7a Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Mon, 10 Aug 2020 14:26:07 +0800 Subject: [PATCH 07/20] fix --- .../src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js index d6b01f1778..c662339b8e 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js @@ -739,7 +739,7 @@ JSP.prototype.handleEvent = function () { }) } /** - * Backfill data processingJSP + * Backfill data processing */ JSP.prototype.jspBackfill = function ({ connects, locations, largeJson }) { // Backfill nodes From efef631b68ef5fcac72c6ef2848aa6b84d718da7 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Mon, 10 Aug 2020 19:05:05 +0800 Subject: [PATCH 08/20] [Fix-3423][dao][sql]Fixed that the resource file of the task node can't be found when upgrade from 1.2.0 to 1.3.x (#3454) --- .../dao/upgrade/DolphinSchedulerManager.java | 2 + .../dao/upgrade/ResourceDao.java | 67 +++++++++++++++++ .../dao/upgrade/UpgradeDao.java | 71 +++++++++++++++++-- .../mysql/dolphinscheduler_ddl.sql | 16 +++++ .../mysql/dolphinscheduler_dml.sql | 19 +++++ .../postgresql/dolphinscheduler_ddl.sql | 16 +++++ .../postgresql/dolphinscheduler_dml.sql | 17 +++++ 7 files changed, 203 insertions(+), 5 deletions(-) create mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ResourceDao.java create mode 100644 sql/upgrade/1.3.2_schema/mysql/dolphinscheduler_ddl.sql create mode 100644 sql/upgrade/1.3.2_schema/mysql/dolphinscheduler_dml.sql create mode 100644 sql/upgrade/1.3.2_schema/postgresql/dolphinscheduler_ddl.sql create mode 100644 sql/upgrade/1.3.2_schema/postgresql/dolphinscheduler_dml.sql diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java index 8d1d862640..b2daae28cb 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java @@ -117,6 +117,8 @@ public class DolphinSchedulerManager { upgradeDao.upgradeDolphinScheduler(schemaDir); if ("1.3.0".equals(schemaVersion)) { upgradeDao.upgradeDolphinSchedulerWorkerGroup(); + } else if ("1.3.2".equals(schemaVersion)) { + upgradeDao.upgradeDolphinSchedulerResourceList(); } version = schemaVersion; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ResourceDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ResourceDao.java new file mode 100644 index 0000000000..b47971e1f0 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ResourceDao.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.upgrade; + +import org.apache.dolphinscheduler.common.utils.ConnectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.HashMap; +import java.util.Map; + +/** + * resource dao + */ +public class ResourceDao { + public static final Logger logger = LoggerFactory.getLogger(ProcessDefinitionDao.class); + + /** + * list all resources + * @param conn connection + * @return map that key is full_name and value is id + */ + Map listAllResources(Connection conn){ + Map resourceMap = new HashMap<>(); + + String sql = String.format("SELECT id,full_name FROM t_ds_resources"); + ResultSet rs = null; + PreparedStatement pstmt = null; + try { + pstmt = conn.prepareStatement(sql); + rs = pstmt.executeQuery(); + + while (rs.next()){ + Integer id = rs.getInt(1); + String fullName = rs.getString(2); + resourceMap.put(fullName,id); + } + + } catch (Exception e) { + logger.error(e.getMessage(),e); + throw new RuntimeException("sql: " + sql, e); + } finally { + ConnectionUtils.releaseResource(rs, pstmt, conn); + } + + return resourceMap; + } + +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java index f9458a82bf..59c2cbc874 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java @@ -19,10 +19,8 @@ package org.apache.dolphinscheduler.dao.upgrade; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.ConnectionUtils; -import org.apache.dolphinscheduler.common.utils.SchemaUtils; -import org.apache.dolphinscheduler.common.utils.ScriptRunner; -import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.process.ResourceInfo; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.AbstractBaseDao; import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory; import org.slf4j.Logger; @@ -36,7 +34,9 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.text.MessageFormat; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public abstract class UpgradeDao extends AbstractBaseDao { @@ -270,6 +270,15 @@ public abstract class UpgradeDao extends AbstractBaseDao { public void upgradeDolphinSchedulerWorkerGroup() { updateProcessDefinitionJsonWorkerGroup(); } + + /** + * upgrade DolphinScheduler resource list + * ds-1.3.2 modify the resource list for process definition json + */ + public void upgradeDolphinSchedulerResourceList() { + updateProcessDefinitionJsonResourceList(); + } + /** * updateProcessDefinitionJsonWorkerGroup */ @@ -288,7 +297,7 @@ public abstract class UpgradeDao extends AbstractBaseDao { for (int i = 0 ;i < tasks.size() ; i++){ JSONObject task = tasks.getJSONObject(i); Integer workerGroupId = task.getInteger("workerGroupId"); - if (workerGroupId == -1) { + if (workerGroupId == null || workerGroupId == -1) { task.put("workerGroup", "default"); }else { task.put("workerGroup", oldWorkerGroupMap.get(workerGroupId)); @@ -310,6 +319,58 @@ public abstract class UpgradeDao extends AbstractBaseDao { } + /** + * updateProcessDefinitionJsonResourceList + */ + protected void updateProcessDefinitionJsonResourceList(){ + ResourceDao resourceDao = new ResourceDao(); + ProcessDefinitionDao processDefinitionDao = new ProcessDefinitionDao(); + Map replaceProcessDefinitionMap = new HashMap<>(); + try { + Map resourcesMap = resourceDao.listAllResources(dataSource.getConnection()); + Map processDefinitionJsonMap = processDefinitionDao.queryAllProcessDefinition(dataSource.getConnection()); + + for (Map.Entry entry : processDefinitionJsonMap.entrySet()){ + JSONObject jsonObject = JSONObject.parseObject(entry.getValue()); + JSONArray tasks = JSONArray.parseArray(jsonObject.getString("tasks")); + + for (int i = 0 ;i < tasks.size() ; i++){ + JSONObject task = tasks.getJSONObject(i); + JSONObject param = (JSONObject) task.get("params"); + if (param != null) { + + List resourceList = JSONUtils.toList(param.getString("resourceList"), ResourceInfo.class); + + if (CollectionUtils.isNotEmpty(resourceList)) { + List newResourceList = resourceList.stream().map(resInfo -> { + String fullName = resInfo.getRes().startsWith("/") ? resInfo.getRes() : String.format("/%s",resInfo.getRes()); + if (resInfo.getId() == 0 && resourcesMap.containsKey(fullName)) { + resInfo.setId(resourcesMap.get(fullName)); + } + return resInfo; + }).collect(Collectors.toList()); + param.put("resourceList",JSONArray.parse(JSONObject.toJSONString(newResourceList))); + } + } + task.put("params",param); + + } + + jsonObject.remove(jsonObject.getString("tasks")); + + jsonObject.put("tasks",tasks); + + replaceProcessDefinitionMap.put(entry.getKey(),jsonObject.toJSONString()); + } + if (replaceProcessDefinitionMap.size() > 0){ + processDefinitionDao.updateProcessDefinitionJson(dataSource.getConnection(),replaceProcessDefinitionMap); + } + }catch (Exception e){ + logger.error("update process definition json resource list error",e); + } + + } + /** * upgradeDolphinScheduler DML * @param schemaDir schemaDir diff --git a/sql/upgrade/1.3.2_schema/mysql/dolphinscheduler_ddl.sql b/sql/upgrade/1.3.2_schema/mysql/dolphinscheduler_ddl.sql new file mode 100644 index 0000000000..38964cc551 --- /dev/null +++ b/sql/upgrade/1.3.2_schema/mysql/dolphinscheduler_ddl.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ \ No newline at end of file diff --git a/sql/upgrade/1.3.2_schema/mysql/dolphinscheduler_dml.sql b/sql/upgrade/1.3.2_schema/mysql/dolphinscheduler_dml.sql new file mode 100644 index 0000000000..383d8a4407 --- /dev/null +++ b/sql/upgrade/1.3.2_schema/mysql/dolphinscheduler_dml.sql @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +SET sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY','')); +SET FOREIGN_KEY_CHECKS=0; +UPDATE t_ds_user SET phone = '' WHERE phone = 'xx'; \ No newline at end of file diff --git a/sql/upgrade/1.3.2_schema/postgresql/dolphinscheduler_ddl.sql b/sql/upgrade/1.3.2_schema/postgresql/dolphinscheduler_ddl.sql new file mode 100644 index 0000000000..38964cc551 --- /dev/null +++ b/sql/upgrade/1.3.2_schema/postgresql/dolphinscheduler_ddl.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ \ No newline at end of file diff --git a/sql/upgrade/1.3.2_schema/postgresql/dolphinscheduler_dml.sql b/sql/upgrade/1.3.2_schema/postgresql/dolphinscheduler_dml.sql new file mode 100644 index 0000000000..bf043ad469 --- /dev/null +++ b/sql/upgrade/1.3.2_schema/postgresql/dolphinscheduler_dml.sql @@ -0,0 +1,17 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +UPDATE t_ds_user SET phone = '' WHERE phone = 'xx'; \ No newline at end of file From 288a2547d0abedad98d12f72f30a7246b8bb8820 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Tue, 11 Aug 2020 09:39:49 +0800 Subject: [PATCH 09/20] Remove node deep monitoring --- dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue | 2 -- 1 file changed, 2 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue index a73b1cb410..98aaaa5ff4 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue @@ -602,9 +602,7 @@ }, watch: { 'tasks': { - deep: true, handler (o) { - // Edit state does not allow deletion of node a... this.setIsEditDag(true) } From 46f058698170fff573bbd02af74762ded639769c Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 11 Aug 2020 10:42:59 +0800 Subject: [PATCH 10/20] If worker group id is null,don't need to set the value of the worker group (#3460) * [Fix-3423][dao][sql]Fixed that the resource file of the task node can't be found when upgrade from 1.2.0 to 1.3.x * [Fix-3423][dao]If worker group id is null,don't need to set the value of the worker group --- .../dolphinscheduler/dao/upgrade/UpgradeDao.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java index 59c2cbc874..3d35b68aac 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java @@ -297,11 +297,14 @@ public abstract class UpgradeDao extends AbstractBaseDao { for (int i = 0 ;i < tasks.size() ; i++){ JSONObject task = tasks.getJSONObject(i); Integer workerGroupId = task.getInteger("workerGroupId"); - if (workerGroupId == null || workerGroupId == -1) { - task.put("workerGroup", "default"); - }else { - task.put("workerGroup", oldWorkerGroupMap.get(workerGroupId)); + if (workerGroupId != null) { + if (workerGroupId == -1) { + task.put("workerGroup", "default"); + } else { + task.put("workerGroup", oldWorkerGroupMap.get(workerGroupId)); + } } + } jsonObject.remove(jsonObject.getString("tasks")); From d0037b46684c201ac690cd9e73cdfbee713e8c45 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Tue, 11 Aug 2020 18:15:56 +0800 Subject: [PATCH 11/20] [ui]Code optimization --- .../pages/dag/_source/plugIn/jsPlumbHandle.js | 31 +++++++++++-------- .../home/pages/dag/_source/plugIn/util.js | 2 +- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js index c662339b8e..c39802d091 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js @@ -691,6 +691,9 @@ JSP.prototype.handleEvent = function () { let sourceId = info['sourceId']// 出 let targetId = info['targetId']// 入 console.log(sourceId,targetId) + let rtTargetarr = rtTargetarrArr(targetId) + let rtSoucearr = rtTargetarrArr(sourceId) + console.log(rtTargetarr,rtSoucearr) /** * When connecting, connection is prohibited when the sourceId and target nodes are empty */ @@ -703,30 +706,32 @@ JSP.prototype.handleEvent = function () { */ let recursiveVal const recursiveTargetarr = (arr, targetId) => { - for (let i in arr) { - if (arr[i] === targetId) { - recursiveVal = targetId - } else { - recursiveTargetarr(rtTargetarrArr(arr[i]), targetId) + for (let i in arr) { + if (arr[i] === targetId) { + recursiveVal = targetId + } else { + recursiveTargetarr(rtTargetarrArr(arr[i]), targetId) + } } - } return recursiveVal } - + // Connection to connected nodes is not allowed - if (_.findIndex(rtTargetarrArr(targetId), v => v === sourceId) !== -1) { + if (_.findIndex(rtTargetarr, v => v == sourceId) !== -1) { + console.log(rtTargetarr,'not allowed') return false } - + // Recursive form to find if the target Targetarr has a sourceId - if (recursiveTargetarr(rtTargetarrArr(sourceId), targetId)) { + if (recursiveTargetarr(rtSoucearr, targetId)) { + console.log('has a sourceId') return false } - - if ($(`#${sourceId}`).attr('data-tasks-type') === 'CONDITIONS' && $(`#${sourceId}`).attr('data-nodenumber') === 2) { + if ($(`#${sourceId}`).attr('data-tasks-type') === 'CONDITIONS' && parseInt($(`#${sourceId}`).attr('data-nodenumber')) === 2) { return false } else { - $(`#${sourceId}`).attr('data-nodenumber', Number($(`#${sourceId}`).attr('data-nodenumber')) + 1) + console.log('data-nodenumber') + $(`#${sourceId}`).attr('data-nodenumber', parseInt($(`#${sourceId}`).attr('data-nodenumber')) + 1) } // Storage node dependency information diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js index 65c345921d..c0037dca7f 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js @@ -34,7 +34,7 @@ const saveTargetarr = (valId, domId) => { let $target = $(`#${domId}`) let targetStr = $target.attr('data-targetarr') ? $target.attr('data-targetarr') + `,${valId}` : `${valId}`; $target.attr('data-targetarr', targetStr) - console.log(targetStr) + console.log('Store node id to targetarr',targetStr) } const rtBantpl = () => { From cafa2ae0c621ccc617d0427dc8a05b063b0f0ddb Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Tue, 11 Aug 2020 18:32:29 +0800 Subject: [PATCH 12/20] fix --- .../src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js index c39802d091..0b4146e490 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js @@ -710,7 +710,8 @@ JSP.prototype.handleEvent = function () { if (arr[i] === targetId) { recursiveVal = targetId } else { - recursiveTargetarr(rtTargetarrArr(arr[i]), targetId) + let targetArr = rtTargetarrArr(arr[i]) + recursiveTargetarr(targetArr, targetId) } } return recursiveVal From b6186bf31847bdaf02712c3fc14cf7fe8c3fa191 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Tue, 11 Aug 2020 18:39:42 +0800 Subject: [PATCH 13/20] fix --- .../src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js index 0b4146e490..2feb5e587d 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js @@ -718,7 +718,7 @@ JSP.prototype.handleEvent = function () { } // Connection to connected nodes is not allowed - if (_.findIndex(rtTargetarr, v => v == sourceId) !== -1) { + if (_.findIndex(rtTargetarr, v => v === sourceId) !== -1) { console.log(rtTargetarr,'not allowed') return false } From e44aa1394e4276ceb1f799b9e94ba5ffefb4d4d7 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Wed, 12 Aug 2020 11:50:27 +0800 Subject: [PATCH 14/20] [fix-3058][ui]Move rtTargetArr to jsPlumbHandle.js --- .../pages/dag/_source/plugIn/jsPlumbHandle.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js index 2feb5e587d..34f78b03b8 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js @@ -688,12 +688,15 @@ JSP.prototype.saveStore = function () { JSP.prototype.handleEvent = function () { this.JspInstance.bind('beforeDrop', function (info) { console.log(info) + const rtTargetArr = (id) => { + let ids = $(`#${id}`).attr('data-targetarr') + return ids ? ids.split(',') : [] + } let sourceId = info['sourceId']// 出 let targetId = info['targetId']// 入 console.log(sourceId,targetId) - let rtTargetarr = rtTargetarrArr(targetId) - let rtSoucearr = rtTargetarrArr(sourceId) - console.log(rtTargetarr,rtSoucearr) + let rtTargetArrs = rtTargetArr(targetId) + let rtSouceArrs = rtTargetArr(sourceId) /** * When connecting, connection is prohibited when the sourceId and target nodes are empty */ @@ -710,7 +713,7 @@ JSP.prototype.handleEvent = function () { if (arr[i] === targetId) { recursiveVal = targetId } else { - let targetArr = rtTargetarrArr(arr[i]) + let targetArr = rtTargetArr(arr[i]) recursiveTargetarr(targetArr, targetId) } } @@ -718,13 +721,13 @@ JSP.prototype.handleEvent = function () { } // Connection to connected nodes is not allowed - if (_.findIndex(rtTargetarr, v => v === sourceId) !== -1) { - console.log(rtTargetarr,'not allowed') + if (_.findIndex(rtTargetArrs, v => v === sourceId) !== -1) { + console.log(rtTargetArrs,'not allowed') return false } // Recursive form to find if the target Targetarr has a sourceId - if (recursiveTargetarr(rtSoucearr, targetId)) { + if (recursiveTargetarr(rtSouceArrs, targetId)) { console.log('has a sourceId') return false } From 12aca8ff516f9defc456789de3d007917adab2c6 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Wed, 12 Aug 2020 14:34:34 +0800 Subject: [PATCH 15/20] [optimization][ui]Prevent the shell script input box from being empty --- .../pages/dag/_source/formModel/tasks/shell.vue | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue index 712a2081b7..2f4609a6f5 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue @@ -20,11 +20,7 @@
{{$t('Script')}}
- + @@ -365,9 +361,11 @@ }, mounted () { // Added delay loading in script input box - setTimeout(() => { - this._handlerEditor() - }, 350) + this.$nextTick(() => { + setTimeout(() => { + this._handlerEditor() + }, 350) + }) }, destroyed () { if (editor) { From a409306601891c08d76f7bbfdc842012db1af537 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Wed, 12 Aug 2020 14:59:38 +0800 Subject: [PATCH 16/20] [Fix-3462][api]If login user is admin,need list all udfs (#3465) * [Fix-3462][api]If login user is admin,need list all udfs * [Fix-3462][api]add the test on the method of QueryUdfFuncList * [Fix-3462][api]fix the code smell * [Fix-3462][api]fix the code smell * [Fix-3462][api]fix the code smell --- .../api/controller/ResourcesController.java | 14 ++++++++------ .../api/service/UdfFuncService.java | 14 +++++++++----- .../api/service/UdfFuncServiceTest.java | 8 +++++--- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java index 2af8ccd767..2f2cb9ca6f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java @@ -564,7 +564,7 @@ public class ResourcesController extends BaseController { @GetMapping(value = "/udf-func/list-paging") @ResponseStatus(HttpStatus.OK) @ApiException(QUERY_UDF_FUNCTION_LIST_PAGING_ERROR) - public Result queryUdfFuncList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + public Result queryUdfFuncListPaging(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("pageNo") Integer pageNo, @RequestParam(value = "searchVal", required = false) String searchVal, @RequestParam("pageSize") Integer pageSize @@ -581,23 +581,25 @@ public class ResourcesController extends BaseController { } /** - * query resource list by type + * query udf func list by type * * @param loginUser login user * @param type resource type * @return resource list */ - @ApiOperation(value = "queryResourceList", notes = "QUERY_RESOURCE_LIST_NOTES") + @ApiOperation(value = "queryUdfFuncList", notes = "QUERY_UDF_FUNC_LIST_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "type", value = "UDF_TYPE", required = true, dataType = "UdfType") }) @GetMapping(value = "/udf-func/list") @ResponseStatus(HttpStatus.OK) @ApiException(QUERY_DATASOURCE_BY_TYPE_ERROR) - public Result queryResourceList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + public Result queryUdfFuncList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("type") UdfType type) { - logger.info("query datasource list, user:{}, type:{}", loginUser.getUserName(), type); - Map result = udfFuncService.queryResourceList(loginUser, type.ordinal()); + String userName = loginUser.getUserName(); + userName = userName.replaceAll("[\n|\r|\t]", "_"); + logger.info("query udf func list, user:{}, type:{}", userName, type); + Map result = udfFuncService.queryUdfFuncList(loginUser, type.ordinal()); return returnDataList(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java index 8a0bf748bb..4d4aec738f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java @@ -279,15 +279,19 @@ public class UdfFuncService extends BaseService{ } /** - * query data resource by type + * query udf list * * @param loginUser login user - * @param type resource type - * @return resource list + * @param type udf type + * @return udf func list */ - public Map queryResourceList(User loginUser, Integer type) { + public Map queryUdfFuncList(User loginUser, Integer type) { Map result = new HashMap<>(5); - List udfFuncList = udfFuncMapper.getUdfFuncByType(loginUser.getId(), type); + int userId = loginUser.getId(); + if (isAdmin(loginUser)) { + userId = 0; + } + List udfFuncList = udfFuncMapper.getUdfFuncByType(userId, type); result.put(Constants.DATA_LIST, udfFuncList); putMsg(result, Status.SUCCESS); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java index 9ec24bbb50..884e9b6b36 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java @@ -149,9 +149,11 @@ public class UdfFuncServiceTest { } @Test - public void testQueryResourceList(){ - Mockito.when(udfFuncMapper.getUdfFuncByType(1, 1)).thenReturn(getList()); - Map result = udfFuncService.queryResourceList(getLoginUser(),1); + public void testQueryUdfFuncList(){ + User user = getLoginUser(); + user.setUserType(UserType.GENERAL_USER); + Mockito.when(udfFuncMapper.getUdfFuncByType(user.getId(), UdfType.HIVE.ordinal())).thenReturn(getList()); + Map result = udfFuncService.queryUdfFuncList(user,UdfType.HIVE.ordinal()); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); List udfFuncList = (List) result.get(Constants.DATA_LIST); From c8322482bbd021a89c407809abfcdd50cf3b2dc6 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Thu, 13 Aug 2020 16:19:11 +0800 Subject: [PATCH 17/20] [Fix-3463][api]Fixed that run the sql task will be failure after rename the udf resource (#3482) --- .../api/service/ResourcesService.java | 26 ++++++++++++++++++- .../dao/mapper/UdfFuncMapper.java | 9 ++++++- .../dao/mapper/UdfFuncMapper.xml | 13 ++++++++++ .../dao/mapper/UdfFuncMapperTest.java | 13 ++++++++++ 4 files changed, 59 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index e0ee7117c4..e1a9c85aa1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -436,14 +436,38 @@ public class ResourcesService extends BaseService { if (CollectionUtils.isNotEmpty(childrenResource)) { String matcherFullName = Matcher.quoteReplacement(fullName); List childResourceList = new ArrayList<>(); - List resourceList = resourcesMapper.listResourceByIds(childrenResource.toArray(new Integer[childrenResource.size()])); + Integer[] childResIdArray = childrenResource.toArray(new Integer[childrenResource.size()]); + List resourceList = resourcesMapper.listResourceByIds(childResIdArray); childResourceList = resourceList.stream().map(t -> { t.setFullName(t.getFullName().replaceFirst(originFullName, matcherFullName)); t.setUpdateTime(now); return t; }).collect(Collectors.toList()); resourcesMapper.batchUpdateResource(childResourceList); + + if (ResourceType.UDF.equals(resource.getType())) { + List udfFuncs = udfFunctionMapper.listUdfByResourceId(childResIdArray); + if (CollectionUtils.isNotEmpty(udfFuncs)) { + udfFuncs = udfFuncs.stream().map(t -> { + t.setResourceName(t.getResourceName().replaceFirst(originFullName, matcherFullName)); + t.setUpdateTime(now); + return t; + }).collect(Collectors.toList()); + udfFunctionMapper.batchUpdateUdfFunc(udfFuncs); + } + } } + } else if (ResourceType.UDF.equals(resource.getType())) { + List udfFuncs = udfFunctionMapper.listUdfByResourceId(new Integer[]{resourceId}); + if (CollectionUtils.isNotEmpty(udfFuncs)) { + udfFuncs = udfFuncs.stream().map(t -> { + t.setResourceName(fullName); + t.setUpdateTime(now); + return t; + }).collect(Collectors.toList()); + udfFunctionMapper.batchUpdateUdfFunc(udfFuncs); + } + } putMsg(result, Status.SUCCESS); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java index a2ce6b29b8..db5e6a454d 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java @@ -16,9 +16,9 @@ */ package org.apache.dolphinscheduler.dao.mapper; -import org.apache.dolphinscheduler.dao.entity.UdfFunc; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.core.metadata.IPage; +import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.ibatis.annotations.Param; import java.util.List; @@ -100,5 +100,12 @@ public interface UdfFuncMapper extends BaseMapper { */ List listAuthorizedUdfByResourceId(@Param("userId") int userId,@Param("resourceIds") int[] resourceIds); + /** + * batch update udf func + * @param udfFuncList udf list + * @return update num + */ + int batchUpdateUdfFunc(@Param("udfFuncList") List udfFuncList); + } diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml index e38d1637d6..18de6db620 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml @@ -111,4 +111,17 @@ + + + + update t_ds_udfs + + resource_name=#{udf.resourceName}, + update_time=#{udf.updateTime} + + + id=#{udf.id} + + + \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapperTest.java index 47d8d89b40..9399c51f3e 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapperTest.java @@ -33,6 +33,7 @@ import org.springframework.test.annotation.Rollback; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.transaction.annotation.Transactional; +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.List; @@ -303,4 +304,16 @@ public class UdfFuncMapperTest { authorizedUdfFunc = udfFuncMapper.listAuthorizedUdfFunc(generalUser1.getId(), udfFuncIds); Assert.assertTrue(authorizedUdfFunc.stream().map(t -> t.getId()).collect(toList()).containsAll(Arrays.asList(udfFuncIds))); } + + @Test + public void batchUpdateUdfFuncTest(){ + //create general user + User generalUser1 = createGeneralUser("user1"); + UdfFunc udfFunc = insertOne(generalUser1); + udfFunc.setResourceName("/updateTest"); + List udfFuncList = new ArrayList<>(); + udfFuncList.add(udfFunc); + Assert.assertTrue(udfFuncMapper.batchUpdateUdfFunc(udfFuncList)>0); + + } } \ No newline at end of file From a78f5c0337c760adab66114fc5f0513be73403ca Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Thu, 13 Aug 2020 17:13:28 +0800 Subject: [PATCH 18/20] [fixBug-3058][ui]Fix connection abnormalities in historical workflow instance data --- .../conf/home/pages/dag/_source/formModel/tasks/flink.vue | 8 +++----- .../js/conf/home/pages/dag/_source/formModel/tasks/mr.vue | 8 +++----- .../home/pages/dag/_source/formModel/tasks/python.vue | 6 ++---- .../conf/home/pages/dag/_source/formModel/tasks/shell.vue | 6 ++---- .../conf/home/pages/dag/_source/formModel/tasks/spark.vue | 8 +++----- 5 files changed, 13 insertions(+), 23 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue index 3f85f36992..0fe5d0cbdb 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue @@ -375,10 +375,8 @@ resourceIdArr = isResourceId.map(item=>{ return item.id }) - Array.prototype.diff = function(a) { - return this.filter(function(i) {return a.indexOf(i) < 0;}); - }; - let diffSet = this.resourceList.diff(resourceIdArr); + let diffSet + diffSet = _.xorWith(this.resourceList, resourceIdArr, _.isEqual) let optionsCmp = [] if(diffSet.length>0) { diffSet.forEach(item=>{ @@ -477,7 +475,7 @@ // Non-null objects represent backfill if (!_.isEmpty(o)) { this.mainClass = o.params.mainClass || '' - if(o.params.mainJar.res) { + if(!o.params.mainJar.id) { this.marjarId(o.params.mainJar.res) } else if(o.params.mainJar.res=='') { this.mainJar = '' diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue index d05c6983d6..26d74bd990 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue @@ -216,10 +216,8 @@ resourceIdArr = isResourceId.map(item=>{ return item.id }) - Array.prototype.diff = function(a) { - return this.filter(function(i) {return a.indexOf(i) < 0;}); - }; - let diffSet = this.resourceList.diff(resourceIdArr); + let diffSet + diffSet = _.xorWith(this.resourceList, resourceIdArr, _.isEqual) let optionsCmp = [] if(diffSet.length>0) { diffSet.forEach(item=>{ @@ -356,7 +354,7 @@ // Non-null objects represent backfill if (!_.isEmpty(o)) { this.mainClass = o.params.mainClass || '' - if(o.params.mainJar.res) { + if(!o.params.mainJar.id) { this.marjarId(o.params.mainJar.res) } else if(o.params.mainJar.res=='') { this.mainJar = '' diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue index 851f8bee8a..40b8c6a0b7 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue @@ -208,10 +208,8 @@ resourceIdArr = isResourceId.map(item=>{ return item.id }) - Array.prototype.diff = function(a) { - return this.filter(function(i) {return a.indexOf(i) < 0;}); - }; - let diffSet = this.resourceList.diff(resourceIdArr); + let diffSet + diffSet = _.xorWith(this.resourceList, resourceIdArr, _.isEqual) let optionsCmp = [] if(diffSet.length>0) { diffSet.forEach(item=>{ diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue index 2f4609a6f5..48c5322b16 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue @@ -246,10 +246,8 @@ resourceIdArr = isResourceId.map(item=>{ return item.id }) - Array.prototype.diff = function(a) { - return this.filter(function(i) {return a.indexOf(i) < 0;}); - }; - let diffSet = this.resourceList.diff(resourceIdArr); + let diffSet + diffSet = _.xorWith(this.resourceList, resourceIdArr, _.isEqual) let optionsCmp = [] if(diffSet.length>0) { diffSet.forEach(item=>{ diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue index 4cceb3b27b..2c59707c69 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue @@ -335,10 +335,8 @@ resourceIdArr = isResourceId.map(item=>{ return item.id }) - Array.prototype.diff = function(a) { - return this.filter(function(i) {return a.indexOf(i) < 0;}); - }; - let diffSet = this.resourceList.diff(resourceIdArr); + let diffSet + diffSet = _.xorWith(this.resourceList, resourceIdArr, _.isEqual) let optionsCmp = [] if(diffSet.length>0) { diffSet.forEach(item=>{ @@ -525,7 +523,7 @@ // Non-null objects represent backfill if (!_.isEmpty(o)) { this.mainClass = o.params.mainClass || '' - if(o.params.mainJar.res) { + if(!o.params.mainJar.id) { this.marjarId(o.params.mainJar.res) } else if(o.params.mainJar.res=='') { this.mainJar = '' From 7a866ff20ae23fdbbe8c43226c2e3eaffc75a593 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Thu, 13 Aug 2020 17:42:08 +0800 Subject: [PATCH 19/20] [Feture-3327][ui]Add the function of re-uploading files in the udf subdirectory --- .../pages/subUdfDirectory/_source/list.vue | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/_source/list.vue index ddb097ee0f..610eb1dbd8 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/_source/list.vue @@ -43,7 +43,7 @@ {{$t('Update Time')}} - + {{$t('Operation')}} @@ -76,6 +76,16 @@ - + + Date: Thu, 13 Aug 2020 19:02:14 +0800 Subject: [PATCH 20/20] fix bug: Fix master task dependency check bug (#3473) Co-authored-by: lenboo --- .../server/master/runner/MasterExecThread.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 6692598ce6..15cc814b26 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -667,12 +667,10 @@ public class MasterExecThread implements Runnable { List depNameList = taskNode.getDepList(); for(String depsNode : depNameList ){ if(!dag.containsNode(depsNode) + || skipTaskNodeList.containsKey(depsNode) || forbiddenTaskList.containsKey(depsNode)){ continue; } - if(skipTaskNodeList.containsKey(depsNode)){ - return DependResult.FAILED; - } // all the dependencies must be completed if(!completeTaskList.containsKey(depsNode)){ return DependResult.WAITING;