From 6467875a87d3c19bde205e83ea5c0dd40b132298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9B=B9=E8=81=9A=E9=98=B3?= Date: Sat, 15 Aug 2020 05:04:58 +0800 Subject: [PATCH 01/45] delay execution ui --- .../js/conf/home/pages/dag/_source/config.js | 7 + .../pages/dag/_source/formModel/formModel.vue | 21 ++- .../pages/list/_source/createDataSource.vue | 7 - .../_source/instanceConditions/common.js | 3 + .../pages/index/_source/taskStatusCount.vue | 147 ++++++++++++++++++ .../home/pages/projects/pages/index/index.vue | 8 +- .../js/conf/home/store/datasource/actions.js | 4 +- .../js/conf/home/store/projects/actions.js | 2 +- .../src/js/module/i18n/locale/en_US.js | 2 + .../src/js/module/i18n/locale/zh_CN.js | 2 + 10 files changed, 188 insertions(+), 15 deletions(-) create mode 100644 dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/taskStatusCount.vue diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js index a4960f7ac5..2e60929577 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js @@ -229,6 +229,13 @@ const tasksState = { color: '#5101be', icoUnicode: 'ans-icon-dependence', isSpin: false + }, + DELAY_EXECUTION: { + id: 12, + desc: `${i18n.$t('Delay execution')}`, + color: '#5102ce', + icoUnicode: 'ans-icon-coin', + isSpin: false } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index 6f07f97f02..8444863aea 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -109,6 +109,20 @@ ({{$t('Minute')}}) + + +
+
+ {{$t('Delay execution time')}} +
+
+ + + ({{$t('Minute')}}) +
+
+ +
{{$t('State')}} @@ -127,7 +141,6 @@
-
{{$t('State')}} @@ -339,6 +352,8 @@ maxRetryTimes: '0', // Failure retry interval retryInterval: '1', + // Delay execution time + delayTime: '0', // Task timeout alarm timeout: {}, // Task priority @@ -466,6 +481,7 @@ dependence: this.cacheDependence, maxRetryTimes: this.maxRetryTimes, retryInterval: this.retryInterval, + delayTime: this.delayTime, timeout: this.timeout, taskInstancePriority: this.taskInstancePriority, workerGroup: this.workerGroup, @@ -544,6 +560,7 @@ dependence: this.dependence, maxRetryTimes: this.maxRetryTimes, retryInterval: this.retryInterval, + delayTime: this.delayTime, timeout: this.timeout, taskInstancePriority: this.taskInstancePriority, workerGroup: this.workerGroup, @@ -634,6 +651,7 @@ this.description = o.description this.maxRetryTimes = o.maxRetryTimes this.retryInterval = o.retryInterval + this.delayTime = o.delayTime if(o.conditionResult) { this.successBranch = o.conditionResult.successNode[0] this.failedBranch = o.conditionResult.failedNode[0] @@ -699,6 +717,7 @@ dependence: this.cacheDependence, maxRetryTimes: this.maxRetryTimes, retryInterval: this.retryInterval, + delayTime: this.delayTime, timeout: this.timeout, taskInstancePriority: this.taskInstancePriority, workerGroup: this.workerGroup, diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue b/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue index adf4753f4f..1e15688c5d 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue @@ -227,10 +227,6 @@ { value: 'DB2', label: 'DB2' - }, - { - value: 'PRESTO', - label: 'PRESTO' } ] } @@ -437,9 +433,6 @@ case 'DB2': defaultPort = '50000' break - case 'PRESTO': - defaultPort = '8080' - break default: break diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/_source/instanceConditions/common.js b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/_source/instanceConditions/common.js index 694d04748c..8a13aeacb4 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/_source/instanceConditions/common.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/_source/instanceConditions/common.js @@ -60,6 +60,9 @@ const stateType = [ }, { code: 'WAITTING_DEPEND', label: `${i18n.$t('Waiting for dependency to complete')}` + }, { + code: 'DELAY_EXECUTION', + label: `${i18n.$t('Delay execution')}` } ] diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/taskStatusCount.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/taskStatusCount.vue new file mode 100644 index 0000000000..f24b0f3402 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/taskStatusCount.vue @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue index 2b5cbbc017..7ca6e3a0f6 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue @@ -35,8 +35,8 @@ {{$t('Task status statistics')}}
- - + +
@@ -68,7 +68,7 @@ import dayjs from 'dayjs' import mDefineUserCount from './_source/defineUserCount' import mCommandStateCount from './_source/commandStateCount' - import mTaskCtatusCount from './_source/taskCtatusCount' + import mTaskStatusCount from './_source/taskStatusCount' import mProcessStateCount from './_source/processStateCount' import mQueueCount from './_source/queueCount' import localStore from '@/module/util/localStorage' @@ -105,7 +105,7 @@ mListConstruction, mDefineUserCount, mCommandStateCount, - mTaskCtatusCount, + mTaskStatusCount, mProcessStateCount, mQueueCount } diff --git a/dolphinscheduler-ui/src/js/conf/home/store/datasource/actions.js b/dolphinscheduler-ui/src/js/conf/home/store/datasource/actions.js index 0743621e21..f8166d610c 100644 --- a/dolphinscheduler-ui/src/js/conf/home/store/datasource/actions.js +++ b/dolphinscheduler-ui/src/js/conf/home/store/datasource/actions.js @@ -20,7 +20,7 @@ import io from '@/module/io' export default { /** * Data source creation - * @param "type": string,//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE, ORACLE, SQLSERVER, PRESTO + * @param "type": string,//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE, ORACLE, SQLSERVER * @param "name": string, * @param "desc": string, * @param "parameter":string //{"address":"jdbc:hive2://192.168.220.189:10000","autoReconnect":"true","characterEncoding":"utf8","database":"default","initialTimeout":3000,"jdbcUrl":"jdbc:hive2://192.168.220.189:10000/default","maxReconnect":10,"password":"","useUnicode":true,"user":"hive"} @@ -49,7 +49,7 @@ export default { }, /** * Query data source list - no paging - * @param "type": string//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE, ORACLE, SQLSERVER, PRESTO + * @param "type": string//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE, ORACLE, SQLSERVER */ getDatasourcesList ({ state }, payload) { return new Promise((resolve, reject) => { diff --git a/dolphinscheduler-ui/src/js/conf/home/store/projects/actions.js b/dolphinscheduler-ui/src/js/conf/home/store/projects/actions.js index 43273de9e2..6a18fdaf9c 100644 --- a/dolphinscheduler-ui/src/js/conf/home/store/projects/actions.js +++ b/dolphinscheduler-ui/src/js/conf/home/store/projects/actions.js @@ -69,7 +69,7 @@ export default { /** * Task status statistics */ - getTaskCtatusCount ({ state }, payload) { + getTaskStatusCount ({ state }, payload) { return new Promise((resolve, reject) => { io.get('projects/analysis/task-state-count', payload, res => { resolve(res) diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js index 0ef5340488..e5e96131f5 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js @@ -42,6 +42,8 @@ export default { Times: 'Times', 'Failed retry interval': 'Failed retry interval', Minute: 'Minute', + 'Delay execution time': 'Delay execution time', + 'Delay execution': 'Delay execution', Cancel: 'Cancel', 'Confirm add': 'Confirm add', 'The newly created sub-Process has not yet been executed and cannot enter the sub-Process': 'The newly created sub-Process has not yet been executed and cannot enter the sub-Process', diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js index a352183fca..26326d3fe2 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -43,6 +43,7 @@ export default { Times: '次', 'Failed retry interval': '失败重试间隔', Minute: '分', + 'Delay execution time': '延时执行时间', Cancel: '取消', 'Confirm add': '确认添加', 'The newly created sub-Process has not yet been executed and cannot enter the sub-Process': '新创建子工作流还未执行，不能进入子工作流', @@ -425,6 +426,7 @@ export default { hour: '时', Running: '正在运行', 'Waiting for dependency to complete': '等待依赖完成', + 'Delay execution': '延时执行', Selected: '已选', CurrentHour: '当前小时', Last1Hour: '前1小时', From 8c0180d2118f8258802d32559ea40c90b5ed23b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9B=B9=E8=81=9A=E9=98=B3?= Date: Sat, 15 Aug 2020 09:46:07 +0800 Subject: [PATCH 02/45] delay execution ui[fixed] --- .../datasource/pages/list/_source/createDataSource.vue | 9 ++++++++- .../src/js/conf/home/store/datasource/actions.js | 6 +++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue b/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue index 1e15688c5d..5377db7d25 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue @@ -227,6 +227,10 @@ { value: 'DB2', label: 'DB2' + }, + { + value: 'PRESTO', + label: 'PRESTO' } ] } @@ -433,6 +437,9 @@ case 'DB2': defaultPort = '50000' break + case 'PRESTO': + defaultPort = '8080' + break default: break @@ -536,4 +543,4 @@ } - + \ No newline at end of file diff --git a/dolphinscheduler-ui/src/js/conf/home/store/datasource/actions.js b/dolphinscheduler-ui/src/js/conf/home/store/datasource/actions.js index f8166d610c..3a80cc3a29 100644 --- a/dolphinscheduler-ui/src/js/conf/home/store/datasource/actions.js +++ b/dolphinscheduler-ui/src/js/conf/home/store/datasource/actions.js @@ -20,7 +20,7 @@ import io from '@/module/io' export default { /** * Data source creation - * @param "type": string,//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE, ORACLE, SQLSERVER + * @param "type": string,//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE, ORACLE, SQLSERVER, PRESTO * @param "name": string, * @param "desc": string, * @param "parameter":string //{"address":"jdbc:hive2://192.168.220.189:10000","autoReconnect":"true","characterEncoding":"utf8","database":"default","initialTimeout":3000,"jdbcUrl":"jdbc:hive2://192.168.220.189:10000/default","maxReconnect":10,"password":"","useUnicode":true,"user":"hive"} @@ -49,7 +49,7 @@ export default { }, /** * Query data source list - no paging - * @param "type": string//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE, ORACLE, SQLSERVER + * @param "type": string//MYSQL, POSTGRESQL, HIVE, SPARK, CLICKHOUSE, ORACLE, SQLSERVER, PRESTO */ getDatasourcesList ({ state }, payload) { return new Promise((resolve, reject) => { @@ -126,4 +126,4 @@ export default { }) }) } -} +} \ No newline at end of file From 3b723c0cc51c40b7b8023ce41ea1834fd3a2a7b5 Mon Sep 17 00:00:00 2001 From: yinancx Date: Sat, 15 Aug 2020 11:20:32 +0800 Subject: [PATCH 03/45] restore --- .../pages/datasource/pages/list/_source/createDataSource.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue b/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue index 5377db7d25..adf4753f4f 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue @@ -543,4 +543,4 @@ } - \ No newline at end of file + From faeb97e70f7bac996dc89ac0da9165722feccfd7 Mon Sep 17 00:00:00 2001 From: yinancx Date: Sat, 15 Aug 2020 11:20:58 +0800 Subject: [PATCH 04/45] restore --- .../src/js/conf/home/store/datasource/actions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/store/datasource/actions.js b/dolphinscheduler-ui/src/js/conf/home/store/datasource/actions.js index 3a80cc3a29..0743621e21 100644 --- a/dolphinscheduler-ui/src/js/conf/home/store/datasource/actions.js +++ b/dolphinscheduler-ui/src/js/conf/home/store/datasource/actions.js @@ -126,4 +126,4 @@ export default { }) }) } -} \ No newline at end of file +} From f9f910187042f6a6043ffaad87e34e8ceb3d38b8 Mon Sep 17 00:00:00 2001 From: yinancx Date: Sat, 15 Aug 2020 11:36:27 +0800 Subject: [PATCH 05/45] update an annotation --- .../home/pages/projects/pages/index/_source/taskStatusCount.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/taskStatusCount.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/taskStatusCount.vue index f24b0f3402..90ae53f4c9 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/taskStatusCount.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/taskStatusCount.vue @@ -93,7 +93,7 @@ const myChart = Chart.pie('#task-status-pie', this.taskStatusList, { title: '' }) myChart.echart.setOption(pie) - // 首页不允许跳转 + // Jump forbidden in index page if (this.searchParams.projectId) { myChart.echart.on('click', e => { this._goTask(e.data.name) From 9bf52f9c95abc228d5a49bc33c4e08f4b1016310 Mon Sep 17 00:00:00 2001 From: zhuangchong Date: Tue, 8 Sep 2020 13:01:18 +0800 Subject: [PATCH 06/45] definition and instance bulk action buttons show changes. --- .../pages/definition/pages/list/_source/list.vue | 11 ++++------- .../pages/instance/pages/list/_source/list.vue | 3 +-- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue index 01b478f6bd..db8d465d36 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue @@ -124,7 +124,6 @@
@@ -134,14 +133,12 @@ {{$t('Confirm')}} - + {{$t('Export')}} + {{$t('Batch copy')}} + {{$t('Batch move')}} diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue index dc1acf5c7f..dfc9f3d82c 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue @@ -288,7 +288,6 @@ @@ -298,7 +297,7 @@ {{$t('Confirm')}} From c42e769ced281fa9668185d9fbb04782e775af0c Mon Sep 17 00:00:00 2001 From: muzhongjiang Date: Wed, 9 Sep 2020 14:13:29 +0800 Subject: [PATCH 07/45] [Fix] fix bug #3487 Create folder duplicate name under multithreading (#3697) * add state * fixed bug "jackson enum conversion : InvalidFormatException" * Word spelling modification Comment modification Word spelling modification,Comment modification,Log level modification * Update EmailManager.java * Update FlinkParameters.java * Update SqlTask.java * fixed "getNotifyGroupList cache" bug * fix bug "Creating folders with multiple threads will result in multiple identical folders #3487" * fix "Creating folders with multiple threads will result in multiple identical folders" #3487 * fix "Creating folders with multiple threads will result in multiple identical folders" #3487 * fix bug #3487 Create folder duplicate name under multithreading * fix bug #3487 Create folder duplicate name under multithreading * fix bug #3487 Create folder duplicate name under multithreading * fix bug #3487 Create folder duplicate name under multithreading Co-authored-by: mzjnumber1@163.com Co-authored-by: dailidong --- .../api/service/ResourcesService.java | 5 ++++ sql/dolphinscheduler-postgre.sql | 3 +- sql/dolphinscheduler_mysql.sql | 3 +- .../mysql/dolphinscheduler_ddl.sql | 26 +++++++++++++++++ .../postgresql/dolphinscheduler_ddl.sql | 28 ++++++++++++++++++- 5 files changed, 62 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index 56d40d9cab..bd7598979d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -37,6 +37,7 @@ import org.apache.dolphinscheduler.dao.utils.ResourceProcessDefinitionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.dao.DuplicateKeyException; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; @@ -139,6 +140,10 @@ public class ResourcesService extends BaseService { } } result.setData(resultMap); + } catch (DuplicateKeyException e) { + logger.error("resource directory {} has exist, can't recreate", fullName); + putMsg(result, Status.RESOURCE_EXIST); + return result; } catch (Exception e) { logger.error("resource already exists, can't recreate ", e); throw new RuntimeException("resource already exists, can't recreate"); diff --git a/sql/dolphinscheduler-postgre.sql b/sql/dolphinscheduler-postgre.sql index 1fedf05cfd..5ae37e1be8 100644 --- a/sql/dolphinscheduler-postgre.sql +++ b/sql/dolphinscheduler-postgre.sql @@ -523,7 +523,8 @@ CREATE TABLE t_ds_resources ( pid int, full_name varchar(64), is_directory int, - PRIMARY KEY (id) + PRIMARY KEY (id), + CONSTRAINT t_ds_resources_un UNIQUE (full_name, type) ) ; diff --git a/sql/dolphinscheduler_mysql.sql b/sql/dolphinscheduler_mysql.sql index 70bb7cddfc..61e697568a 100644 --- a/sql/dolphinscheduler_mysql.sql +++ b/sql/dolphinscheduler_mysql.sql @@ -657,7 +657,8 @@ CREATE TABLE `t_ds_resources` ( `pid` int(11) DEFAULT NULL, `full_name` varchar(64) DEFAULT NULL, `is_directory` tinyint(4) DEFAULT NULL, - PRIMARY KEY (`id`) + PRIMARY KEY (`id`), + UNIQUE KEY `t_ds_resources_un` (`full_name`,`type`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; -- ---------------------------- diff --git a/sql/upgrade/1.3.3_schema/mysql/dolphinscheduler_ddl.sql b/sql/upgrade/1.3.3_schema/mysql/dolphinscheduler_ddl.sql index e9f8b5b6b5..43488272e2 100644 --- a/sql/upgrade/1.3.3_schema/mysql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.3.3_schema/mysql/dolphinscheduler_ddl.sql @@ -87,3 +87,29 @@ delimiter ; CALL ct_dolphin_T_t_ds_process_definition_version; DROP PROCEDURE ct_dolphin_T_t_ds_process_definition_version; + + + +-- add t_ds_resources_un +DROP PROCEDURE IF EXISTS uc_dolphin_T_t_ds_resources_un; +delimiter d// +CREATE PROCEDURE uc_dolphin_T_t_ds_resources_un() +BEGIN + IF NOT EXISTS ( + SELECT * FROM information_schema.KEY_COLUMN_USAGE + WHERE TABLE_NAME = 't_ds_resources' + AND CONSTRAINT_NAME = 't_ds_resources_un' + ) + THEN + ALTER TABLE t_ds_resources ADD CONSTRAINT t_ds_resources_un UNIQUE KEY (full_name,`type`); + END IF; +END; + +d// + +delimiter ; +CALL uc_dolphin_T_t_ds_resources_un(); +DROP PROCEDURE IF EXISTS uc_dolphin_T_t_ds_resources_un; + + + diff --git a/sql/upgrade/1.3.3_schema/postgresql/dolphinscheduler_ddl.sql b/sql/upgrade/1.3.3_schema/postgresql/dolphinscheduler_ddl.sql index 52045f61f0..e2767617df 100644 --- a/sql/upgrade/1.3.3_schema/postgresql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.3.3_schema/postgresql/dolphinscheduler_ddl.sql @@ -79,4 +79,30 @@ d// delimiter ; SELECT ct_dolphin_T_t_ds_process_definition_version(); -DROP FUNCTION IF EXISTS ct_dolphin_T_t_ds_process_definition_version(); \ No newline at end of file +DROP FUNCTION IF EXISTS ct_dolphin_T_t_ds_process_definition_version(); + + + + +-- add t_ds_resources_un +CREATE OR REPLACE FUNCTION uc_dolphin_T_t_ds_resources_un() RETURNS void AS $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM information_schema.KEY_COLUMN_USAGE + WHERE TABLE_NAME = 't_ds_resources' + AND CONSTRAINT_NAME = 't_ds_resources_un' + ) + THEN +ALTER TABLE t_ds_resources ADD CONSTRAINT t_ds_resources_un UNIQUE (full_name,"type"); +END IF; +END; +$$ LANGUAGE plpgsql; + +SELECT uc_dolphin_T_t_ds_resources_un(); +DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_resources_un(); + + + + + + From 1e8fe73506d4aa555fcdd56401f7060972804691 Mon Sep 17 00:00:00 2001 From: yolo Date: Wed, 9 Sep 2020 14:26:14 +0800 Subject: [PATCH 08/45] [Feature-3701][ui] Shortcut connection action for node in same flow (#3699) * [Improvement][ui] Shortcut connection action for node in same flow * Set pre-tasks to get current node connect to pre-nodes automatically. * Only SHELL and SUB-PROCESS node enabled this method. * Add license * Fix code-smell * Replace Chinese comments * Try to re-trigger e2eTest --- .../pages/dag/_source/formModel/formModel.vue | 71 +++++++++++- .../dag/_source/formModel/tasks/pre_tasks.vue | 108 ++++++++++++++++++ .../src/js/module/i18n/locale/en_US.js | 3 +- .../src/js/module/i18n/locale/zh_CN.js | 3 +- 4 files changed, 181 insertions(+), 4 deletions(-) create mode 100644 dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/pre_tasks.vue diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index 8444863aea..7cd63c07db 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -277,6 +277,12 @@ :backfill-item="backfillItem" :pre-node="preNode"> + +
@@ -310,6 +316,7 @@ import mSelectInput from './_source/selectInput' import mTimeoutAlarm from './_source/timeoutAlarm' import mWorkerGroups from './_source/workerGroups' + import mPreTasks from './tasks/pre_tasks' import clickoutside from '@/module/util/clickoutside' import disabledState from '@/module/mixin/disabledState' import { isNameExDag, rtBantpl } from './../plugIn/util' @@ -369,7 +376,11 @@ value: 'failed', label: `${i18n.$t('failed')}` } - ] + ], + // preTasks + preTaskIdsInWorkflow: [], + preTasksToAdd: [], // pre-taskIds to add, used in jsplumb connects + preTasksToDelete: [], // pre-taskIds to delete, used in jsplumb connects } }, /** @@ -393,6 +404,14 @@ _onDependent (o) { this.dependence = Object.assign(this.dependence, {}, o) }, + /** + * Pre-tasks in workflow + */ + _onPreTasks (o) { + this.preTaskIdsInWorkflow = o.preTasks + this.preTasksToAdd = o.preTasksToAdd + this.preTasksToDelete = o.preTasksToDelete + }, /** * cache dependent */ @@ -543,6 +562,43 @@ if (!this.$refs[this.taskType]._verification()) { return } + // Verify preTasks and update dag-things + if (this.$refs['PRE_TASK']) { + if (!this.$refs['PRE_TASK']._verification()) { + return + } + else { + // Sync data-targetarr + $(`#${this.id}`).attr( + 'data-targetarr', this.preTaskIdsInWorkflow ? this.preTaskIdsInWorkflow.join(',') : '') + + // Update JSP connections + let plumbIns = JSP.JspInstance + var targetId = this.id + + // Update new connections + this.preTasksToAdd.map(sourceId => { + plumbIns.connect({ + source: sourceId, + target: targetId, + type: 'basic', + paintStyle: { strokeWidth: 2, stroke: '#2d8cf0' }, + HoverPaintStyle: {stroke: '#ccc', strokeWidth: 3} + }) + }) + + // Update remove connections + let currentConnects = plumbIns.getAllConnections() + let len = currentConnects.length + for (let i = 0; i < len; i++) { + if (this.preTasksToDelete.indexOf(currentConnects[i].sourceId) > -1 && currentConnects[i].targetId == targetId) { + plumbIns.deleteConnection(currentConnects[i]) + i -= 1 + len -= 1 + } + } + } + } $(`#${this.id}`).find('span').text(this.name) this.conditionResult.successNode[0] = this.successBranch @@ -684,6 +740,16 @@ } this.cacheBackfillItem = JSON.parse(JSON.stringify(o)) this.isContentBox = true + + // Init value of preTask selector + let preTaskIds = $(`#${this.id}`).attr('data-targetarr') + if (!_.isEmpty(this.backfillItem)) { + if (preTaskIds && preTaskIds.length) { + this.backfillItem.preTasks = preTaskIds.split(',') + } else { + this.backfillItem.preTasks = [] + } + } }, mounted () { let self = this @@ -745,7 +811,8 @@ mSelectInput, mTimeoutAlarm, mPriority, - mWorkerGroups + mWorkerGroups, + mPreTasks, } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/pre_tasks.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/pre_tasks.vue new file mode 100644 index 0000000000..adf889e958 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/pre_tasks.vue @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js index e1ecedc9ae..07dfb7c5a7 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js @@ -642,5 +642,6 @@ export default { 'Related items': 'Related items', 'Project name is required': 'Project name is required', 'Batch move': 'Batch move', - Version: 'Version' + Version: 'Version', + 'Pre tasks': 'Pre tasks', } diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js index f69142d3b5..e3f2562f6f 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -642,5 +642,6 @@ export default { 'Related items': '关联项目', 'Project name is required': '项目名称必填', 'Batch move': '批量移动', - Version: '版本' + Version: '版本', + 'Pre tasks': '前置任务', } From 092ce2a54e243d8325ca3403e7136ec6093aef97 Mon Sep 17 00:00:00 2001 From: zhuangchong Date: Wed, 9 Sep 2020 20:09:37 +0800 Subject: [PATCH 09/45] The batch delete function in the workflow definition and workflow instance pages cannot be canceled if selected. --- .../projects/pages/definition/pages/list/_source/list.vue | 8 ++++++++ .../projects/pages/instance/pages/list/_source/list.vue | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue index 01b478f6bd..f4bf743a73 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue @@ -268,6 +268,12 @@ * Close the delete layer */ _closeDelete (i) { + // close batch + if (i < 0) { + this.$refs['poptipDeleteAll'].doClose() + return + } + // close one this.$refs[`poptip-delete-${i}`][0].doClose() }, /** @@ -596,8 +602,10 @@ }).then(res => { this._onUpdate() this.checkAll = false + this.strSelectIds = '' this.$message.success(res.msg) }).catch(e => { + this.strSelectIds = '' this.checkAll = false this.$message.error(e.msg || '') }) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue index dc1acf5c7f..c13c00d64e 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue @@ -344,6 +344,12 @@ * Close the delete layer */ _closeDelete (i) { + // close batch + if (i < 0) { + this.$refs['poptipDeleteAll'].doClose() + return + } + // close one this.$refs[`poptip-delete-${i}`][0].doClose() }, /** @@ -539,9 +545,11 @@ }).then(res => { this._onUpdate() this.checkAll = false + this.strDelete = '' this.$message.success(res.msg) }).catch(e => { this.checkAll = false + this.strDelete = '' this.$message.error(e.msg || '') }) } From 4ed36387507c50b1042802143676a04fc51e6bcc Mon Sep 17 00:00:00 2001 From: "felix.wang" <59079269+felix-thinkingdata@users.noreply.github.com> Date: Wed, 9 Sep 2020 22:09:49 +0800 Subject: [PATCH 10/45] [Bug][server] fix 3629 pull request (#3706) * fix 3629 pr * fix e2e build --- .github/workflows/ci_e2e.yml | 2 +- .../java/org/apache/dolphinscheduler/server/utils/LogUtils.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml index 7f5fc8a989..8070d7e0c5 100644 --- a/.github/workflows/ci_e2e.yml +++ b/.github/workflows/ci_e2e.yml @@ -59,7 +59,7 @@ jobs: sudo dpkg -i google-chrome*.deb sudo apt-get install -f -y google-chrome -version - googleVersion=`google-chrome -version | awk '{print $3}'` + googleVersion=$(curl -s https://chromedriver.storage.googleapis.com/LATEST_RELEASE) wget -N https://chromedriver.storage.googleapis.com/${googleVersion}/chromedriver_linux64.zip unzip chromedriver_linux64.zip sudo mv -f chromedriver /usr/local/share/chromedriver diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/LogUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/LogUtils.java index 93008b9d64..bb8ddc85de 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/LogUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/LogUtils.java @@ -70,7 +70,7 @@ public class LogUtils { * get task log path by TaskExecutionContext */ public static String getTaskLogPath(TaskExecutionContext taskExecutionContext) { - return getTaskLogPath(taskExecutionContext.getProcessId(), taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getTaskInstanceId()); + return getTaskLogPath(taskExecutionContext.getProcessDefineId(), taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getTaskInstanceId()); } } From 7af20ca3afe858f29abdd9ad9cb5013d8fd33d65 Mon Sep 17 00:00:00 2001 From: BoYiZhang <39816903+BoYiZhang@users.noreply.github.com> Date: Sat, 12 Sep 2020 23:42:40 +0800 Subject: [PATCH 11/45] [Bug-3713][HadoopUtils] catfile method Stream not closed (#3715) * fix bug Delete invalid field: executorcores Modify verification prompt * fix bug Delete invalid field: executorcores Modify verification prompt * fix bug Delete invalid field: executorcores Modify verification prompt * dag add close button * reset last version * reset last version * dag add close buttion dag add close buttion * update CLICK_SAVE_WORKFLOW_BUTTON xpath * updae CLICK_SAVE_WORKFLOW_BUTTON xpath * updae CLICK_SAVE_WORKFLOW_BUTTON xpath * updae CLICK_SAVE_WORKFLOW_BUTTON xpath * Update CreateWorkflowLocator.java modify submit workflow button * Update CreateWorkflowLocator.java * Update CreateWorkflowLocator.java modify CLICK_ADD_BUTTON * Update CreateWorkflowLocator.java delete print * Update CreateWorkflowLocator.java 1 * Update CreateWorkflowLocator.java 1 * Setting '-XX:+DisableExplicitGC ' causes netty memory leaks in addition update '- XX: largepagesizeinbytes = 128M' to '- XX: largepagesizeinbytes = 10M' * Update dag.vue * Update dag.vue * Update dag.vue * Update CreateWorkflowLocator.java * Revert "Setting '-XX:+DisableExplicitGC ' causes netty memory leaks" This reverts commit 3a2cba7a * Setting '-XX:+DisableExplicitGC ' causes netty memory leaks in addition update '- XX: largepagesizeinbytes = 128M' to '- XX: largepagesizeinbytes = 10M' * Update dolphinscheduler-daemon.sh * catfile method Stream not closed * catfile method Stream not closed Co-authored-by: dailidong Co-authored-by: xingchun-chen <55787491+xingchun-chen@users.noreply.github.com> --- .../common/utils/HadoopUtils.java | 45 ++++++++++--------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 68d03506c6..fe9ac932c1 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -14,26 +14,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import org.apache.commons.io.IOUtils; +import static org.apache.dolphinscheduler.common.Constants.RESOURCE_UPLOAD_PATH; + import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.dolphinscheduler.common.enums.ResourceType; + +import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.*; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.client.cli.RMAdminCLI; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.io.*; +import java.io.BufferedReader; +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; import java.nio.file.Files; import java.security.PrivilegedExceptionAction; import java.util.Collections; @@ -43,7 +48,13 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.apache.dolphinscheduler.common.Constants.RESOURCE_UPLOAD_PATH; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; /** * hadoop utils @@ -102,7 +113,6 @@ public class HadoopUtils implements Closeable { } } - /** * init hadoop configuration */ @@ -167,7 +177,6 @@ public class HadoopUtils implements Closeable { fs = FileSystem.get(configuration); } - } catch (Exception e) { logger.error(e.getMessage(), e); } @@ -232,11 +241,11 @@ public class HadoopUtils implements Closeable { return new byte[0]; } - FSDataInputStream fsDataInputStream = fs.open(new Path(hdfsFilePath)); - return IOUtils.toByteArray(fsDataInputStream); + try (FSDataInputStream fsDataInputStream = fs.open(new Path(hdfsFilePath))) { + return IOUtils.toByteArray(fsDataInputStream); + } } - /** * cat file on hdfs * @@ -527,7 +536,6 @@ public class HadoopUtils implements Closeable { return String.format("%s/udfs", getHdfsTenantDir(tenantCode)); } - /** * get hdfs file name * @@ -579,7 +587,6 @@ public class HadoopUtils implements Closeable { return String.format("%s/%s", getHdfsDataBasePath(), tenantCode); } - /** * getAppAddress * @@ -610,7 +617,6 @@ public class HadoopUtils implements Closeable { return start + activeRM + end; } - @Override public void close() throws IOException { if (fs != null) { @@ -623,7 +629,6 @@ public class HadoopUtils implements Closeable { } } - /** * yarn ha admin utils */ @@ -669,7 +674,6 @@ public class HadoopUtils implements Closeable { return null; } - /** * get ResourceManager state * @@ -694,4 +698,5 @@ public class HadoopUtils implements Closeable { } } + } From 94247d84320ef21c9c02ea55edafede112950f75 Mon Sep 17 00:00:00 2001 From: xloya <982052490@qq.com> Date: Sun, 13 Sep 2020 23:52:19 +0800 Subject: [PATCH 12/45] [Bug-3722][ui]fix cannot change password on dev branch. (#3724) #3722 fix cannot change password on dev branch. --- .../js/conf/home/pages/user/pages/password/_source/info.vue | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/password/_source/info.vue b/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/password/_source/info.vue index af296c9f28..c2d97be2f5 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/password/_source/info.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/password/_source/info.vue @@ -85,7 +85,8 @@ userPassword: this.userPassword, tenantId: this.userInfo.tenantId, email: this.userInfo.email, - phone: this.userInfo.phone + phone: this.userInfo.phone, + state: this.userinfo.state } this.spinnerLoading = true this.updateUser(param).then(res => { From 3a2663eeb7579d1574af0c7ed051dbcbd81efa10 Mon Sep 17 00:00:00 2001 From: "felix.wang" <59079269+felix-thinkingdata@users.noreply.github.com> Date: Sun, 13 Sep 2020 23:55:25 +0800 Subject: [PATCH 13/45] =?UTF-8?q?[Improvement][DAO]=20druid=20The=20lower?= =?UTF-8?q?=20version=20is=20not=20compatible=20with=20LocalDateTime(DRUID?= =?UTF-8?q?=20=E4=BD=8E=E7=89=88=E6=9C=AC=E4=B8=8D=E6=94=AF=E6=8C=81LocalD?= =?UTF-8?q?ateTime)=20(#3716)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix mybatis 3.51版本的LocalDateTime映射报SQLFeatureNotSupportedException * fix druid-1.1.14.jar->1.1.22 --- pom.xml | 2 +- tools/dependencies/known-dependencies.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 0321c1a24c..65cbe62bbb 100644 --- a/pom.xml +++ b/pom.xml @@ -69,7 +69,7 @@ 3.2.0 2.0.1 5.0.5 - 1.1.14 + 1.1.22 1.4.200 1.6 1.1.1 diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 0074dcd9fb..4a7f1662f0 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -43,7 +43,7 @@ datanucleus-api-jdo-4.2.1.jar datanucleus-core-4.1.6.jar datanucleus-rdbms-4.1.7.jar derby-10.14.2.0.jar -druid-1.1.14.jar +druid-1.1.22.jar gson-2.8.5.jar guava-20.0.jar guice-3.0.jar From 565b8d3b66cbc035d263ef5695a6c01e8d2eac9b Mon Sep 17 00:00:00 2001 From: "felix.wang" <59079269+felix-thinkingdata@users.noreply.github.com> Date: Mon, 14 Sep 2020 10:39:37 +0800 Subject: [PATCH 14/45] =?UTF-8?q?[Improvement-3690][common]=20Get=20the=20?= =?UTF-8?q?native=20IP=20policy=20problem=20(=E8=8E=B7=E5=8F=96=E6=9C=AC?= =?UTF-8?q?=E6=9C=BAip=E7=AD=96=E7=95=A5=E9=97=AE=E9=A2=98=20)=20(#3695)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Get the Intranet IP * Get the Intranet IP * fix code smell * fix code smell * fix code smell * fix code smell * support ipv6 --x:--x:--x:--x:--x:--x:--x:--x * Update dolphinscheduler-common/src/main/resources/common.properties Co-authored-by: Yichao Yang <1048262223@qq.com> Co-authored-by: Yichao Yang <1048262223@qq.com> --- .../dolphinscheduler/common/Constants.java | 5 + .../common/utils/NetUtils.java | 121 ++++++++++++++---- .../src/main/resources/common.properties | 3 + 3 files changed, 103 insertions(+), 26 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 3b12748888..ba8b0c4921 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -1000,4 +1000,9 @@ public final class Constants { public static final String DATASOURCE_ENCRYPTION_SALT_DEFAULT = "!@#$%^&*"; public static final String DATASOURCE_ENCRYPTION_ENABLE = "datasource.encryption.enable"; public static final String DATASOURCE_ENCRYPTION_SALT = "datasource.encryption.salt"; + + /** + * Network IP gets priority, default inner outer + */ + public static final String NETWORK_PRIORITY_STRATEGY = "dolphin.scheduler.network.priority.strategy"; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java index b001825ce1..ddb29730b7 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java @@ -21,6 +21,8 @@ import static org.apache.dolphinscheduler.common.Constants.DOLPHIN_SCHEDULER_PRE import static java.util.Collections.emptyList; +import org.apache.dolphinscheduler.common.Constants; + import java.io.IOException; import java.net.Inet6Address; import java.net.InetAddress; @@ -42,22 +44,20 @@ import org.slf4j.LoggerFactory; */ public class NetUtils { - private NetUtils() { - throw new UnsupportedOperationException("Construct NetUtils"); - } - - private static Logger logger = LoggerFactory.getLogger(NetUtils.class); - private static final Pattern IP_PATTERN = Pattern.compile("\\d{1,3}(\\.\\d{1,3}){3,5}$"); - - private static String ANY_HOST_VALUE = "0.0.0.0"; - - private static String LOCAL_HOST_VALUE = "127.0.0.1"; - + private static final String NETWORK_PRIORITY_DEFAULT = "default"; + private static final String NETWORK_PRIORITY_INNER = "inner"; + private static final String NETWORK_PRIORITY_OUTER = "outer"; + private static final Logger logger = LoggerFactory.getLogger(NetUtils.class); + private static final String ANY_HOST_VALUE = "0.0.0.0"; + private static final String LOCAL_HOST_VALUE = "127.0.0.1"; private static InetAddress LOCAL_ADDRESS = null; - private static volatile String HOST_ADDRESS; + private NetUtils() { + throw new UnsupportedOperationException("Construct NetUtils"); + } + public static String getHost() { if (HOST_ADDRESS != null) { return HOST_ADDRESS; @@ -87,24 +87,27 @@ public class NetUtils { if (null != LOCAL_ADDRESS) { return LOCAL_ADDRESS; } + InetAddress localAddress = null; - NetworkInterface networkInterface = findNetworkInterface(); - Enumeration addresses = networkInterface.getInetAddresses(); - while (addresses.hasMoreElements()) { - Optional addressOp = toValidAddress(addresses.nextElement()); - if (addressOp.isPresent()) { - try { - if (addressOp.get().isReachable(100)) { - LOCAL_ADDRESS = addressOp.get(); - return LOCAL_ADDRESS; + try { + NetworkInterface networkInterface = findNetworkInterface(); + if (networkInterface != null) { + Enumeration addresses = networkInterface.getInetAddresses(); + while (addresses.hasMoreElements()) { + Optional addressOp = toValidAddress(addresses.nextElement()); + if (addressOp.isPresent()) { + try { + if (addressOp.get().isReachable(100)) { + LOCAL_ADDRESS = addressOp.get(); + return LOCAL_ADDRESS; + } + } catch (IOException e) { + logger.warn("test address id reachable io exception", e); + } } - } catch (IOException e) { - logger.warn("test address id reachable io exception", e); } } - } - try { localAddress = InetAddress.getLocalHost(); } catch (UnknownHostException e) { logger.warn("InetAddress get LocalHost exception", e); @@ -190,7 +193,7 @@ public class NetUtils { if (null != result) { return result; } - return validNetworkInterfaces.get(0); + return findAddress(validNetworkInterfaces); } /** @@ -227,4 +230,70 @@ public class NetUtils { String preferredNetworkInterface = System.getProperty(DOLPHIN_SCHEDULER_PREFERRED_NETWORK_INTERFACE); return Objects.equals(networkInterface.getDisplayName(), preferredNetworkInterface); } + + private static NetworkInterface findAddress(List validNetworkInterfaces) { + if (validNetworkInterfaces.isEmpty()) { + return null; + } + String networkPriority = PropertyUtils.getString(Constants.NETWORK_PRIORITY_STRATEGY, NETWORK_PRIORITY_DEFAULT); + if (NETWORK_PRIORITY_DEFAULT.equalsIgnoreCase(networkPriority)) { + return findAddressByDefaultPolicy(validNetworkInterfaces); + } else if (NETWORK_PRIORITY_INNER.equalsIgnoreCase(networkPriority)) { + return findInnerAddress(validNetworkInterfaces); + } else if (NETWORK_PRIORITY_OUTER.equalsIgnoreCase(networkPriority)) { + return findOuterAddress(validNetworkInterfaces); + } else { + logger.error("There is no matching network card acquisition policy!"); + return null; + } + } + + private static NetworkInterface findAddressByDefaultPolicy(List validNetworkInterfaces) { + NetworkInterface networkInterface; + networkInterface = findInnerAddress(validNetworkInterfaces); + if (networkInterface == null) { + networkInterface = findOuterAddress(validNetworkInterfaces); + if (networkInterface == null) { + networkInterface = validNetworkInterfaces.get(0); + } + } + return networkInterface; + } + + /** + * Get the Intranet IP + * + * @return If no {@link NetworkInterface} is available , return null + */ + private static NetworkInterface findInnerAddress(List validNetworkInterfaces) { + + NetworkInterface networkInterface = null; + for (NetworkInterface ni : validNetworkInterfaces) { + Enumeration address = ni.getInetAddresses(); + while (address.hasMoreElements()) { + InetAddress ip = address.nextElement(); + if (ip.isSiteLocalAddress() + && !ip.isLoopbackAddress()) { + networkInterface = ni; + } + } + } + return networkInterface; + } + + private static NetworkInterface findOuterAddress(List validNetworkInterfaces) { + NetworkInterface networkInterface = null; + for (NetworkInterface ni : validNetworkInterfaces) { + Enumeration address = ni.getInetAddresses(); + while (address.hasMoreElements()) { + InetAddress ip = address.nextElement(); + if (!ip.isSiteLocalAddress() + && !ip.isLoopbackAddress()) { + networkInterface = ni; + } + } + } + return networkInterface; + } + } diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index a75f964fa2..b8c21c853b 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -72,3 +72,6 @@ kerberos.expire.time=2 # datasource encryption salt datasource.encryption.enable=false datasource.encryption.salt=!@#$%^&* + +# Network IP gets priority, default inner outer +#dolphin.scheduler.network.priority.strategy=default From fad28527685453082cc2950da835d477c9ec5a54 Mon Sep 17 00:00:00 2001 From: zhuangchong <37063904+zhuangchong@users.noreply.github.com> Date: Mon, 14 Sep 2020 15:59:56 +0800 Subject: [PATCH 15/45] [Improvement][server] WATERDROP task plug-in optimization in switch case code cleaning. (#3652) * WATERDROP switch case code checkstyle. * add TaskManagerTest. * TaskManagerTest checkstyle. * TaskManagerTest checkstyle. * TaskManagerTest add pom maven-surefire * TaskManagerTest update. --- .../common/utils/TaskParametersUtils.java | 3 +- .../server/worker/task/TaskManager.java | 74 ++++++------- .../server/worker/task/TaskManagerTest.java | 103 ++++++++++++++++++ pom.xml | 1 + 4 files changed, 141 insertions(+), 40 deletions(-) create mode 100644 dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/TaskManagerTest.java diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java index 6099a0d49d..2b40b079c6 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java @@ -59,9 +59,8 @@ public class TaskParametersUtils { switch (EnumUtils.getEnum(TaskType.class, taskType)) { case SUB_PROCESS: return JSONUtils.parseObject(parameter, SubProcessParameters.class); - case WATERDROP: - return JSONUtils.parseObject(parameter, ShellParameters.class); case SHELL: + case WATERDROP: return JSONUtils.parseObject(parameter, ShellParameters.class); case PROCEDURE: return JSONUtils.parseObject(parameter, ProcedureParameters.class); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskManager.java index f98d451350..34eea9dade 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskManager.java @@ -14,8 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.server.worker.task; +package org.apache.dolphinscheduler.server.worker.task; import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.utils.EnumUtils; @@ -30,6 +30,7 @@ import org.apache.dolphinscheduler.server.worker.task.shell.ShellTask; import org.apache.dolphinscheduler.server.worker.task.spark.SparkTask; import org.apache.dolphinscheduler.server.worker.task.sql.SqlTask; import org.apache.dolphinscheduler.server.worker.task.sqoop.SqoopTask; + import org.slf4j.Logger; /** @@ -37,42 +38,39 @@ import org.slf4j.Logger; */ public class TaskManager { - /** - * create new task - * @param taskExecutionContext taskExecutionContext - * @param logger logger - * @return AbstractTask - * @throws IllegalArgumentException illegal argument exception - */ - public static AbstractTask newTask(TaskExecutionContext taskExecutionContext, - Logger logger) - throws IllegalArgumentException { - switch (EnumUtils.getEnum(TaskType.class,taskExecutionContext.getTaskType())) { - case SHELL: - return new ShellTask(taskExecutionContext, logger); - case WATERDROP: - return new ShellTask(taskExecutionContext, logger); - case PROCEDURE: - return new ProcedureTask(taskExecutionContext, logger); - case SQL: - return new SqlTask(taskExecutionContext, logger); - case MR: - return new MapReduceTask(taskExecutionContext, logger); - case SPARK: - return new SparkTask(taskExecutionContext, logger); - case FLINK: - return new FlinkTask(taskExecutionContext, logger); - case PYTHON: - return new PythonTask(taskExecutionContext, logger); - case HTTP: - return new HttpTask(taskExecutionContext, logger); - case DATAX: - return new DataxTask(taskExecutionContext, logger); - case SQOOP: - return new SqoopTask(taskExecutionContext, logger); - default: - logger.error("unsupport task type: {}", taskExecutionContext.getTaskType()); - throw new IllegalArgumentException("not support task type"); + /** + * create new task + * @param taskExecutionContext taskExecutionContext + * @param logger logger + * @return AbstractTask + * @throws IllegalArgumentException illegal argument exception + */ + public static AbstractTask newTask(TaskExecutionContext taskExecutionContext, Logger logger) throws IllegalArgumentException { + switch (EnumUtils.getEnum(TaskType.class,taskExecutionContext.getTaskType())) { + case SHELL: + case WATERDROP: + return new ShellTask(taskExecutionContext, logger); + case PROCEDURE: + return new ProcedureTask(taskExecutionContext, logger); + case SQL: + return new SqlTask(taskExecutionContext, logger); + case MR: + return new MapReduceTask(taskExecutionContext, logger); + case SPARK: + return new SparkTask(taskExecutionContext, logger); + case FLINK: + return new FlinkTask(taskExecutionContext, logger); + case PYTHON: + return new PythonTask(taskExecutionContext, logger); + case HTTP: + return new HttpTask(taskExecutionContext, logger); + case DATAX: + return new DataxTask(taskExecutionContext, logger); + case SQOOP: + return new SqoopTask(taskExecutionContext, logger); + default: + logger.error("unsupport task type: {}", taskExecutionContext.getTaskType()); + throw new IllegalArgumentException("not support task type"); + } } - } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/TaskManagerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/TaskManagerTest.java new file mode 100644 index 0000000000..058270e1e1 --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/TaskManagerTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.task; + +import org.apache.dolphinscheduler.common.utils.LoggerUtils; +import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; +import org.apache.dolphinscheduler.server.worker.cache.impl.TaskExecutionContextCacheManagerImpl; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; + +import java.util.Date; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({SpringApplicationContext.class}) +public class TaskManagerTest { + + private TaskExecutionContext taskExecutionContext; + + private Logger taskLogger; + + private TaskExecutionContextCacheManagerImpl taskExecutionContextCacheManager; + + @Before + public void before() { + // init task execution context, logger + taskExecutionContext = new TaskExecutionContext(); + taskExecutionContext.setProcessId(12345); + taskExecutionContext.setProcessDefineId(1); + taskExecutionContext.setProcessInstanceId(1); + taskExecutionContext.setTaskInstanceId(1); + taskExecutionContext.setTaskType(""); + taskExecutionContext.setFirstSubmitTime(new Date()); + taskExecutionContext.setDelayTime(0); + taskExecutionContext.setLogPath("/tmp/test.log"); + taskExecutionContext.setHost("localhost"); + taskExecutionContext.setExecutePath("/tmp/dolphinscheduler/exec/process/1/2/3/4"); + + taskLogger = LoggerFactory.getLogger(LoggerUtils.buildTaskId( + LoggerUtils.TASK_LOGGER_INFO_PREFIX, + taskExecutionContext.getProcessDefineId(), + taskExecutionContext.getProcessInstanceId(), + taskExecutionContext.getTaskInstanceId() + )); + + taskExecutionContextCacheManager = new TaskExecutionContextCacheManagerImpl(); + taskExecutionContextCacheManager.cacheTaskExecutionContext(taskExecutionContext); + + PowerMockito.mockStatic(SpringApplicationContext.class); + PowerMockito.when(SpringApplicationContext.getBean(TaskExecutionContextCacheManagerImpl.class)) + .thenReturn(taskExecutionContextCacheManager); + } + + @Test + public void testNewTask() { + + taskExecutionContext.setTaskType("SHELL"); + Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); + taskExecutionContext.setTaskType("WATERDROP"); + Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); + taskExecutionContext.setTaskType("HTTP"); + Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); + taskExecutionContext.setTaskType("MR"); + Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); + taskExecutionContext.setTaskType("SPARK"); + Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); + taskExecutionContext.setTaskType("FLINK"); + Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); + taskExecutionContext.setTaskType("PYTHON"); + Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); + taskExecutionContext.setTaskType("DATAX"); + Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); + taskExecutionContext.setTaskType("SQOOP"); + Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); + //taskExecutionContext.setTaskType(null); + //Assert.assertNull(TaskManager.newTask(taskExecutionContext,taskLogger)); + //taskExecutionContext.setTaskType("XXX"); + //Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); + } +} diff --git a/pom.xml b/pom.xml index 65cbe62bbb..207518c936 100644 --- a/pom.xml +++ b/pom.xml @@ -847,6 +847,7 @@ **/server/worker/task/sqoop/SqoopTaskTest.java + **/server/worker/task/TaskManagerTest.java **/server/worker/EnvFileTest.java **/server/worker/runner/TaskExecuteThreadTest.java **/service/quartz/cron/CronUtilsTest.java From 3942740941533721694ceef2177c5363b0d0f200 Mon Sep 17 00:00:00 2001 From: zhuangchong <37063904+zhuangchong@users.noreply.github.com> Date: Mon, 14 Sep 2020 18:56:47 +0800 Subject: [PATCH 16/45] js mailbox verification fix. (#3721) --- .../pages/projects/pages/definition/pages/list/_source/util.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/util.js b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/util.js index 308af45a48..877f349803 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/util.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/util.js @@ -37,7 +37,7 @@ const warningTypeList = [ ] const isEmial = (val) => { - let regEmail = /^([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+@([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+\.[a-zA-Z]{2,3}$/ // eslint-disable-line + let regEmail = /^([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+@([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+\.[a-zA-Z]{2,}$/ // eslint-disable-line return regEmail.test(val) } From 93bc5c0081afade9599f0a9e8bc213e1d426c500 Mon Sep 17 00:00:00 2001 From: xiagw Date: Wed, 16 Sep 2020 13:43:39 +0700 Subject: [PATCH 17/45] fix:Dockerfile user apk --no-cache (#3691) Dockerfile use apk --no-cache Dockerfile use COPY replace ADD --- docker/build/Dockerfile | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile index ceb94ea8c5..7b0f09c539 100644 --- a/docker/build/Dockerfile +++ b/docker/build/Dockerfile @@ -27,13 +27,13 @@ ENV DEBIAN_FRONTEND noninteractive #If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example: #RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories RUN apk update && \ - apk --update add --no-cache dos2unix shadow bash openrc python2 python3 sudo vim wget iputils net-tools openssh-server py-pip tini && \ - apk add --update procps && \ + apk add --update --no-cache dos2unix shadow bash openrc python2 python3 sudo vim wget iputils net-tools openssh-server py-pip tini && \ + apk add --update --no-cache procps && \ openrc boot && \ pip install kazoo #2. install jdk -RUN apk add openjdk8 +RUN apk add --update --no-cache openjdk8 ENV JAVA_HOME /usr/lib/jvm/java-1.8-openjdk ENV PATH $JAVA_HOME/bin:$PATH @@ -43,19 +43,20 @@ RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler #4. install database, if use mysql as your backend database, the `mysql-client` package should be installed -RUN apk add postgresql postgresql-contrib +RUN apk add --update --no-cache postgresql postgresql-contrib + #5. modify nginx RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \ rm -rf /etc/nginx/conf.d/* -ADD ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d +COPY ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d #6. add configuration and modify permissions and set soft links -ADD ./checkpoint.sh /root/checkpoint.sh -ADD ./startup-init-conf.sh /root/startup-init-conf.sh -ADD ./startup.sh /root/startup.sh -ADD ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/ -ADD ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/ -ADD conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/ +COPY ./checkpoint.sh /root/checkpoint.sh +COPY ./startup-init-conf.sh /root/startup-init-conf.sh +COPY ./startup.sh /root/startup.sh +COPY ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/ +COPY ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/ +COPY conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/ RUN chmod +x /root/checkpoint.sh && \ chmod +x /root/startup-init-conf.sh && \ chmod +x /root/startup.sh && \ @@ -70,12 +71,12 @@ RUN chmod +x /root/checkpoint.sh && \ dos2unix /opt/dolphinscheduler/bin/*.sh && \ rm -rf /bin/sh && \ ln -s /bin/bash /bin/sh && \ - mkdir -p /tmp/xls - -#7. remove apk index cache and disable coredup for sudo -RUN rm -rf /var/cache/apk/* && \ + mkdir -p /tmp/xls && \ + #7. remove apk index cache and disable coredup for sudo + rm -rf /var/cache/apk/* && \ echo "Set disable_coredump false" >> /etc/sudo.conf + #8. expose port EXPOSE 2181 2888 3888 5432 5678 1234 12345 50051 8888 From a34f8296cc14e143eea86fb9250524d541b2e4f0 Mon Sep 17 00:00:00 2001 From: elonlo Date: Wed, 16 Sep 2020 18:04:49 +0800 Subject: [PATCH 18/45] [Improvement][ui] Refactored dag formatting logic (#3703) Co-authored-by: dashi --- dolphinscheduler-dist/release-docs/LICENSE | 1 + .../licenses/ui-licenses/LICENSE-dagre | 19 ++ dolphinscheduler-ui/package.json | 1 + .../src/js/conf/home/pages/dag/_source/dag.js | 212 ++---------------- 4 files changed, 45 insertions(+), 188 deletions(-) create mode 100644 dolphinscheduler-dist/release-docs/licenses/ui-licenses/LICENSE-dagre diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index 59da2746bf..707ea5cab1 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -502,6 +502,7 @@ MIT licenses vue-router 2.7.0: https://github.com/vuejs/vue-router MIT vuex 3.0.0: https://github.com/vuejs/vuex MIT vuex-router-sync 4.1.2: https://github.com/vuejs/vuex-router-sync MIT + dagre 0.8.5: https://github.com/dagrejs/dagre MIT ======================================== Apache 2.0 licenses diff --git a/dolphinscheduler-dist/release-docs/licenses/ui-licenses/LICENSE-dagre b/dolphinscheduler-dist/release-docs/licenses/ui-licenses/LICENSE-dagre new file mode 100644 index 0000000000..e3c8f95557 --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/ui-licenses/LICENSE-dagre @@ -0,0 +1,19 @@ +Copyright (c) 2012-2014 Chris Pettitt + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/dolphinscheduler-ui/package.json b/dolphinscheduler-ui/package.json index a5642c8f97..9624fa6212 100644 --- a/dolphinscheduler-ui/package.json +++ b/dolphinscheduler-ui/package.json @@ -20,6 +20,7 @@ "clipboard": "^2.0.1", "codemirror": "^5.43.0", "d3": "^3.5.17", + "dagre": "^0.8.5", "dayjs": "^1.7.8", "echarts": "4.1.0", "html2canvas": "^0.5.0-beta4", diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js index ff8a4528d5..d7f2f78a0c 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js @@ -22,6 +22,7 @@ import { jsPlumb } from 'jsplumb' import JSP from './plugIn/jsPlumbHandle' import DownChart from './plugIn/downChart' import store from '@/conf/home/store' +import dagre from "dagre" /** * Prototype method @@ -115,202 +116,38 @@ Dag.prototype.toolbarEvent = function ({ item, code, is }) { */ Dag.prototype.backfill = function (arg) { if (arg) { - let locationsValue = store.state.dag.locations - const locationsValue1 = store.state.dag.locations - const locationsValue2 = store.state.dag.locations - const arr = [] - for (const i in locationsValue1) { - const objs = {} - objs.id = i - arr.push(Object.assign(objs, locationsValue1[i])) // Attributes - } - const tmp = [] - for (const i in locationsValue2) { - if (locationsValue2[i].targetarr !== '' && locationsValue2[i].targetarr.split(',').length > 1) { - tmp.push(locationsValue2[i]) - } - } - - const copy = function (array) { - const newArray = [] - for (const item of array) { - newArray.push(item) - } - return newArray - } - - const newArr = copy(arr) - const getNewArr = function () { - for (let i = 0; i < newArr.length; i++) { - if (newArr[i].targetarr !== '' && newArr[i].targetarr.split(',').length > 1) { - newArr[i].targetarr = newArr[i].targetarr.split(',').shift() - } - } - return newArr - } - getNewArr() - /** - * @description Transform flat data into a tree structure - * @param {Array} arr Flat data - * @param {String} pidStr targetarr key name - * @param {String} idStr id key name - * @param {String} childrenStr children key name - */ - const fommat = function ({ arrayList, pidStr = 'targetarr', idStr = 'id', childrenStr = 'children' }) { - const listOjb = {} // Used to store objects of the form {key: obj} - const treeList = [] // An array to store the final tree structure data - // Transform the data into {key: obj} format, which is convenient for the following data processing - for (let i = 0; i < arrayList.length; i++) { - listOjb[arrayList[i][idStr]] = arrayList[i] - } - // Format data based on pid - for (let j = 0; j < arrayList.length; j++) { - // Determine if the parent exists - // let haveParent = arrayList[j].targetarr.split(',').length>1?listOjb[arrayList[j].targetarr.split(',')[0]]:listOjb[arrayList[j][pidStr]] - const haveParent = listOjb[arrayList[j][pidStr]] - if (haveParent) { - // If there is no parent children field, create a children field - !haveParent[childrenStr] && (haveParent[childrenStr] = []) - // Insert child in parent - haveParent[childrenStr].push(arrayList[j]) - } else { - // If there is no parent, insert directly into the outermost layer - treeList.push(arrayList[j]) - } - } - return treeList - } - const datas = fommat({ arrayList: newArr, pidStr: 'targetarr' }) - // Count the number of leaf nodes - const getLeafCountTree = function (json) { - if (!json.children) { - json.colspan = 1 - return 1 - } else { - let leafCount = 0 - for (let i = 0; i < json.children.length; i++) { - leafCount = leafCount + getLeafCountTree(json.children[i]) - } - json.colspan = leafCount - return leafCount - } - } - // Number of tree node levels - const countTree = getLeafCountTree(datas[0]) - const getMaxFloor = function (treeData) { - let max = 0 - function each (data, floor) { - data.forEach(e => { - e.floor = floor - e.x = floor * 170 - if (floor > max) { - max = floor - } - if (e.children) { - each(e.children, floor + 1) - } - }) - } - each(treeData, 1) - return max - } - getMaxFloor(datas) - // The last child of each node - let lastchildren = [] - const forxh = function (list) { - for (let i = 0; i < list.length; i++) { - const chlist = list[i] - if (chlist.children) { - forxh(chlist.children) - } else { - lastchildren.push(chlist) - } - } - } - forxh(datas) - // Get all parent nodes above the leaf node - const treeFindPath = function (tree, func, path, n) { - if (!tree) return [] - for (const data of tree) { - path.push(data.name) - if (func(data)) return path - if (data.children) { - const findChildren = treeFindPath(data.children, func, path, n) - if (findChildren.length) return findChildren - } - path.pop() - } - return [] - } - const toLine = function (data) { - return data.reduce((arrData, { id, name, targetarr, x, y, children = [] }) => - arrData.concat([{ id, name, targetarr, x, y }], toLine(children)), []) - } - const listarr = toLine(datas) - const listarrs = toLine(datas) - const dataObject = {} - for (let i = 0; i < listarrs.length; i++) { - delete (listarrs[i].id) - } + const marginX = 100 + const g = new dagre.graphlib.Graph() + g.setGraph({}) + g.setDefaultEdgeLabel(function () { return {} }) - for (let a = 0; a < listarr.length; a++) { - dataObject[listarr[a].id] = listarrs[a] + for (const i in store.state.dag.locations) { + const location = store.state.dag.locations[i] + g.setNode(i, { label: i, width: Math.min(location.name.length * 7, 170), height: 150 }) } - // Comparison function - const createComparisonFunction = function (propertyName) { - return function (object1, object2) { - const value1 = object1[propertyName] - const value2 = object2[propertyName] - if (value1 < value2) { - return -1 - } else if (value1 > value2) { - return 1 - } else { - return 0 - } - } + for (const i in store.state.dag.connects) { + const connect = store.state.dag.connects[i] + g.setEdge(connect['endPointSourceId'], connect['endPointTargetId']) } + dagre.layout(g) - lastchildren = lastchildren.sort(createComparisonFunction('x')) - - // Coordinate value of each leaf node - for (let a = 0; a < lastchildren.length; a++) { - dataObject[lastchildren[a].id].y = (a + 1) * 120 - } - for (let i = 0; i < lastchildren.length; i++) { - const node = treeFindPath(datas, data => data.targetarr === lastchildren[i].targetarr, [], i + 1) - for (let j = 0; j < node.length; j++) { - for (let k = 0; k < listarrs.length; k++) { - if (node[j] === listarrs[k].name) { - listarrs[k].y = (i + 1) * 120 - } - } - } - } - for (let i = 0; i < tmp.length; i++) { - for (const objs in dataObject) { - if (tmp[i].name === dataObject[objs].name) { - dataObject[objs].targetarr = tmp[i].targetarr - } - } - } - for (let a = 0; a < lastchildren.length; a++) { - dataObject[lastchildren[a].id].y = (a + 1) * 120 - } - if (countTree > 1) { - dataObject[Object.keys(locationsValue1)[0]].y = (countTree / 2) * 120 + 50 - } - - locationsValue = dataObject - const self = this + const dataObject = {} + g.nodes().forEach(function (v) { + const node = g.node(v) + const obj = {} + obj.name = node.label + obj.x = node.x + marginX + obj.y = node.y + dataObject[node.label] = obj + }) jsPlumb.ready(() => { JSP.init({ dag: this.dag, instance: this.instance, options: { onRemoveNodes ($id) { - self.dag.removeEventModelById($id) + this.dag.removeEventModelById($id) } } }) @@ -319,20 +156,19 @@ Dag.prototype.backfill = function (arg) { // connects connects: _.cloneDeep(store.state.dag.connects), // Node location information - locations: _.cloneDeep(locationsValue), + locations: _.cloneDeep(dataObject), // Node data largeJson: _.cloneDeep(store.state.dag.tasks) }) }) } else { - const self = this jsPlumb.ready(() => { JSP.init({ dag: this.dag, instance: this.instance, options: { onRemoveNodes ($id) { - self.dag.removeEventModelById($id) + this.dag.removeEventModelById($id) } } }) From 36d534961020f7a44c9e4e2ac7594446597d88c6 Mon Sep 17 00:00:00 2001 From: zhuangchong <37063904+zhuangchong@users.noreply.github.com> Date: Thu, 17 Sep 2020 13:58:43 +0800 Subject: [PATCH 19/45] [fix-3745][server] server get tasktype NPE exception (#3746) --- .../common/utils/TaskParametersUtils.java | 71 ++++++++++--------- .../server/worker/task/TaskManager.java | 10 ++- .../server/worker/task/TaskManagerTest.java | 19 +++-- 3 files changed, 58 insertions(+), 42 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java index 2b40b079c6..af2961e628 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java @@ -55,41 +55,42 @@ public class TaskParametersUtils { * @return task parameters */ public static AbstractParameters getParameters(String taskType, String parameter) { - try { - switch (EnumUtils.getEnum(TaskType.class, taskType)) { - case SUB_PROCESS: - return JSONUtils.parseObject(parameter, SubProcessParameters.class); - case SHELL: - case WATERDROP: - return JSONUtils.parseObject(parameter, ShellParameters.class); - case PROCEDURE: - return JSONUtils.parseObject(parameter, ProcedureParameters.class); - case SQL: - return JSONUtils.parseObject(parameter, SqlParameters.class); - case MR: - return JSONUtils.parseObject(parameter, MapreduceParameters.class); - case SPARK: - return JSONUtils.parseObject(parameter, SparkParameters.class); - case PYTHON: - return JSONUtils.parseObject(parameter, PythonParameters.class); - case DEPENDENT: - return JSONUtils.parseObject(parameter, DependentParameters.class); - case FLINK: - return JSONUtils.parseObject(parameter, FlinkParameters.class); - case HTTP: - return JSONUtils.parseObject(parameter, HttpParameters.class); - case DATAX: - return JSONUtils.parseObject(parameter, DataxParameters.class); - case CONDITIONS: - return JSONUtils.parseObject(parameter, ConditionsParameters.class); - case SQOOP: - return JSONUtils.parseObject(parameter, SqoopParameters.class); - default: - return null; - } - } catch (Exception e) { - logger.error(e.getMessage(), e); + TaskType anEnum = EnumUtils.getEnum(TaskType.class, taskType); + if (anEnum == null) { + logger.error("not support task type: {}", taskType); + return null; } - return null; + switch (anEnum) { + case SUB_PROCESS: + return JSONUtils.parseObject(parameter, SubProcessParameters.class); + case SHELL: + case WATERDROP: + return JSONUtils.parseObject(parameter, ShellParameters.class); + case PROCEDURE: + return JSONUtils.parseObject(parameter, ProcedureParameters.class); + case SQL: + return JSONUtils.parseObject(parameter, SqlParameters.class); + case MR: + return JSONUtils.parseObject(parameter, MapreduceParameters.class); + case SPARK: + return JSONUtils.parseObject(parameter, SparkParameters.class); + case PYTHON: + return JSONUtils.parseObject(parameter, PythonParameters.class); + case DEPENDENT: + return JSONUtils.parseObject(parameter, DependentParameters.class); + case FLINK: + return JSONUtils.parseObject(parameter, FlinkParameters.class); + case HTTP: + return JSONUtils.parseObject(parameter, HttpParameters.class); + case DATAX: + return JSONUtils.parseObject(parameter, DataxParameters.class); + case CONDITIONS: + return JSONUtils.parseObject(parameter, ConditionsParameters.class); + case SQOOP: + return JSONUtils.parseObject(parameter, SqoopParameters.class); + default: + return null; + } + } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskManager.java index 34eea9dade..a9463336b4 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskManager.java @@ -30,7 +30,6 @@ import org.apache.dolphinscheduler.server.worker.task.shell.ShellTask; import org.apache.dolphinscheduler.server.worker.task.spark.SparkTask; import org.apache.dolphinscheduler.server.worker.task.sql.SqlTask; import org.apache.dolphinscheduler.server.worker.task.sqoop.SqoopTask; - import org.slf4j.Logger; /** @@ -46,7 +45,12 @@ public class TaskManager { * @throws IllegalArgumentException illegal argument exception */ public static AbstractTask newTask(TaskExecutionContext taskExecutionContext, Logger logger) throws IllegalArgumentException { - switch (EnumUtils.getEnum(TaskType.class,taskExecutionContext.getTaskType())) { + TaskType anEnum = EnumUtils.getEnum(TaskType.class, taskExecutionContext.getTaskType()); + if (anEnum == null) { + logger.error("not support task type: {}", taskExecutionContext.getTaskType()); + throw new IllegalArgumentException("not support task type"); + } + switch (anEnum) { case SHELL: case WATERDROP: return new ShellTask(taskExecutionContext, logger); @@ -69,7 +73,7 @@ public class TaskManager { case SQOOP: return new SqoopTask(taskExecutionContext, logger); default: - logger.error("unsupport task type: {}", taskExecutionContext.getTaskType()); + logger.error("not support task type: {}", taskExecutionContext.getTaskType()); throw new IllegalArgumentException("not support task type"); } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/TaskManagerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/TaskManagerTest.java index 058270e1e1..eb0383979c 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/TaskManagerTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/TaskManagerTest.java @@ -38,6 +38,8 @@ import org.slf4j.LoggerFactory; @PrepareForTest({SpringApplicationContext.class}) public class TaskManagerTest { + private static Logger logger = LoggerFactory.getLogger(TaskManagerTest.class); + private TaskExecutionContext taskExecutionContext; private Logger taskLogger; @@ -95,9 +97,18 @@ public class TaskManagerTest { Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); taskExecutionContext.setTaskType("SQOOP"); Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); - //taskExecutionContext.setTaskType(null); - //Assert.assertNull(TaskManager.newTask(taskExecutionContext,taskLogger)); - //taskExecutionContext.setTaskType("XXX"); - //Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); + + } + + @Test(expected = IllegalArgumentException.class) + public void testNewTaskIsNull() { + taskExecutionContext.setTaskType(null); + TaskManager.newTask(taskExecutionContext,taskLogger); + } + + @Test(expected = IllegalArgumentException.class) + public void testNewTaskIsNotExists() { + taskExecutionContext.setTaskType("XXX"); + TaskManager.newTask(taskExecutionContext,taskLogger); } } From 2f584f61cbc2b38c34fe53826442e8d998790074 Mon Sep 17 00:00:00 2001 From: xloya <982052490@qq.com> Date: Fri, 18 Sep 2020 19:14:28 +0800 Subject: [PATCH 20/45] [Fix][ui]fix the letter case mistake (#3760) --- .../src/js/conf/home/pages/user/pages/password/_source/info.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/password/_source/info.vue b/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/password/_source/info.vue index c2d97be2f5..132f6d9d24 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/password/_source/info.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/password/_source/info.vue @@ -86,7 +86,7 @@ tenantId: this.userInfo.tenantId, email: this.userInfo.email, phone: this.userInfo.phone, - state: this.userinfo.state + state: this.userInfo.state } this.spinnerLoading = true this.updateUser(param).then(res => { From 42deff0dc7252d6bd6b2e965cfbcdcf22aee0632 Mon Sep 17 00:00:00 2001 From: xingchun-chen <55787491+xingchun-chen@users.noreply.github.com> Date: Sat, 19 Sep 2020 23:07:23 +0800 Subject: [PATCH 21/45] =?UTF-8?q?[test-2559][e2e]=20test=20data=20configur?= =?UTF-8?q?e=20isolation=C2=A0=20(#3712)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add task connection  * Optimize test cases   * Modify variable format     * Optimize test cases   * Update BrowserCommon.java * Update BrowserCommon.java * Update WorkflowDefineLocator.java * Optimize waiting time   * Optimize wait time   * Optimize wait time   * Optimize wait time   * Optimize wait time   * Optimize wait time   * Optimize wait time   * Optimize wait time   * Optimize wait time   * Optimize wait time   * Optimize project wait time   * Optimize wait time   * Optimize wait time   * open timing testcase  * Optimize wait time   * Optimize wait time   * Optimize wait time   * Optimize wait time   * Optimize wait time   * Optimize wait time   * Optimize alert to save workflow   * Optimize alert to save workflow   * add yaml reader * add yaml file  * [e2e] test data configure isolation  * [e2e] test data configure isolation  * [e2e] test data configure isolation  * [e2e] modify check code style  * [e2e] modify check code style  * [e2e] modify check code style  * [e2e] modify style AvoidStarImport  * [e2e] modify check code style  * [e2e] modify check code style  * [e2e] modify check code style  * [e2e] modify check code style  * [e2e] modify check code style  * [e2e] modify check code style  * [e2e] modify check code style  * [e2e] modify check code style  * [e2e] modify check code style  * [e2e] modify check code style  Co-authored-by: chenxingchun <438044805@qq.com> --- e2e/pom.xml | 10 +++ .../dolphinscheduler/util/YmlReader.java | 45 +++++++++++ .../dolphinscheduler/base/BaseDriver.java | 1 - .../dolphinscheduler/data/LoginData.java | 3 - .../data/project/ProcessInstanceData.java | 12 +-- .../data/project/ProjectData.java | 16 ++-- .../data/project/RunWorkflowData.java | 19 ++--- .../data/project/TimingData.java | 25 ++---- .../data/project/WorkflowDefineData.java | 76 ++----------------- .../data/security/AlertManageData.java | 18 ++--- .../data/security/QueueManageData.java | 31 ++------ .../data/security/TenantManageData.java | 30 ++------ .../data/security/TokenManageData.java | 15 ++-- .../data/security/UserManageData.java | 29 ++----- .../project/WorkflowDefineLocator.java | 2 + .../locator/security/TokenManageLocator.java | 6 +- .../locator/security/UserManageLocator.java | 2 + .../dolphinscheduler/page/LoginPage.java | 8 +- .../page/project/ProcessInstancePage.java | 4 +- .../page/project/ProjectPage.java | 15 ++-- .../page/project/RunWorkflowPage.java | 11 ++- .../page/project/TimingPage.java | 31 ++++---- .../page/project/WorkflowDefinePage.java | 57 +++++++------- .../page/security/AlertManagePage.java | 14 ++-- .../page/security/QueueManagePage.java | 18 +++-- .../page/security/TenantManagePage.java | 16 ++-- .../page/security/TokenManagePage.java | 14 ++-- .../page/security/UserManagePage.java | 33 ++++---- .../resources/testData/security_zh_cn.yml | 55 ++++++++++++++ .../resources/testData/workflow_zh_cn.yml | 63 +++++++++++++++ e2e/testng.xml | 2 +- 31 files changed, 369 insertions(+), 312 deletions(-) create mode 100644 e2e/src/main/java/org/apache/dolphinscheduler/util/YmlReader.java create mode 100644 e2e/src/test/resources/testData/security_zh_cn.yml create mode 100644 e2e/src/test/resources/testData/workflow_zh_cn.yml diff --git a/e2e/pom.xml b/e2e/pom.xml index a5bdf69b41..fffbf8974e 100644 --- a/e2e/pom.xml +++ b/e2e/pom.xml @@ -87,6 +87,16 @@ commons-pool2 ${commons-pool2.version} + + org.jyaml + jyaml + 1.3 + + + org.springframework + spring-core + 5.1.5.RELEASE + diff --git a/e2e/src/main/java/org/apache/dolphinscheduler/util/YmlReader.java b/e2e/src/main/java/org/apache/dolphinscheduler/util/YmlReader.java new file mode 100644 index 0000000000..e2784eb0ff --- /dev/null +++ b/e2e/src/main/java/org/apache/dolphinscheduler/util/YmlReader.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.util; + +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; + +import org.ho.yaml.Yaml; +import org.springframework.core.io.DefaultResourceLoader; +import org.springframework.core.io.Resource; + +/** + * read yml file + */ +public class YmlReader { + public static HashMap> map; + public String getDataYml(String filePath, String key1, String key2) { + Yaml yaml = new Yaml(); + Resource resource = new DefaultResourceLoader().getResource("classpath:" + filePath + ".yml"); + try { + InputStream inputStream = resource.getInputStream(); + map = yaml.loadType(inputStream, HashMap.class); + } catch (IOException e) { + e.printStackTrace(); + } + String data = map.get(key1).get(key2); + return data; + } +} diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseDriver.java b/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseDriver.java index 23c98b4e46..73a8da2d62 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseDriver.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseDriver.java @@ -23,7 +23,6 @@ import org.openqa.selenium.PageLoadStrategy; import org.openqa.selenium.WebDriver; import org.openqa.selenium.chrome.ChromeDriver; import org.openqa.selenium.chrome.ChromeOptions; -import org.openqa.selenium.remote.CapabilityType; import java.io.IOException; import java.util.concurrent.TimeUnit; diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/LoginData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/LoginData.java index e56df5a448..332b8000b6 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/data/LoginData.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/LoginData.java @@ -28,7 +28,6 @@ public class LoginData { */ public static final String URL = PropertiesReader.getKey("LOGIN_URL"); - /** * Login username */ @@ -38,6 +37,4 @@ public class LoginData { * Login password */ public static final String PASSWORD = PropertiesReader.getKey("PASSWORD"); - - public static final String TENANT = "租户管理 - DolphinScheduler"; } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/ProcessInstanceData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/ProcessInstanceData.java index 7114851d25..fa827f6707 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/ProcessInstanceData.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/ProcessInstanceData.java @@ -16,10 +16,12 @@ */ package org.apache.dolphinscheduler.data.project; -public class ProcessInstanceData { - //Process Instance page title - public static final String PROCESS_INSTANCE_TITLE = "工作流实例 - DolphinScheduler"; - public static final String RERUN_TYPE= "重跑"; - +import org.apache.dolphinscheduler.util.YmlReader; +public class ProcessInstanceData { + public String getProcessInstanceData(String param) { + YmlReader ymlReader = new YmlReader(); + String processInstanceData = ymlReader.getDataYml("testData/workflow_zh_cn", "processInstance", param); + return processInstanceData; + } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/ProjectData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/ProjectData.java index 036a6771bd..22b44684bb 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/ProjectData.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/ProjectData.java @@ -16,14 +16,12 @@ */ package org.apache.dolphinscheduler.data.project; -public class ProjectData { - - public static final String CREATE_PROJECT_BUTTON = "创建项目"; +import org.apache.dolphinscheduler.util.YmlReader; - // create project name - public static final String PROJECT_NAME = "selenium_project_1"; - // create project description - public static final String DESCRIPTION = "test create project description"; - // project page title - public static final String PROJECT_TITLE = "项目 - DolphinScheduler"; +public class ProjectData { + public String getProjectData(String param) { + YmlReader ymlReader = new YmlReader(); + String projectData = ymlReader.getDataYml("testData/workflow_zh_cn", "projectManage", param); + return projectData; + } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/RunWorkflowData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/RunWorkflowData.java index 70112edd7a..7653f5a09d 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/RunWorkflowData.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/RunWorkflowData.java @@ -16,17 +16,12 @@ */ package org.apache.dolphinscheduler.data.project; -public class RunWorkflowData { - /** - * run workflow data - */ - //input shell task name - public static final String RECIPIENT = "123456789@qq.com"; - - //input shell task description - public static final String Cc = "qwe12312sds@qq.com"; - - public static final String RUN_WORKFLOW_TITLE = "工作流定义 - DolphinScheduler"; - +import org.apache.dolphinscheduler.util.YmlReader; +public class RunWorkflowData { + public String getRunWorkflowData(String param) { + YmlReader ymlReader = new YmlReader(); + String runWorkflowData = ymlReader.getDataYml("testData/workflow_zh_cn", "runWorkflow", param); + return runWorkflowData; + } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/TimingData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/TimingData.java index e76852dd0c..87fbbd8c03 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/TimingData.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/TimingData.java @@ -16,23 +16,12 @@ */ package org.apache.dolphinscheduler.data.project; -public class TimingData { - /** - * timing data - */ - //input shell task name - public static final String RECIPIENT = "test123456@qq.com"; - - //input shell task description - public static final String Cc = "test.123qwe@qq.com"; - - public static final String EDIT_RECIPIENT = "test.edit123456@qq.com"; - - public static final String EDIT_Cc = "test.edit123qwe@qq.com"; +import org.apache.dolphinscheduler.util.YmlReader; - public static final String TIMING_OFFLINE_STATE = "下线"; - - public static final String TIMING_ONLINE_STATE = "上线"; - - public static final String TIMING_TITLE = "定时任务列表 - DolphinScheduler"; +public class TimingData { + public String getTimingData(String param) { + YmlReader ymlReader = new YmlReader(); + String timingData = ymlReader.getDataYml("testData/workflow_zh_cn", "timing", param); + return timingData; + } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/WorkflowDefineData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/WorkflowDefineData.java index c9eb32e948..41b33e76fc 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/WorkflowDefineData.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/WorkflowDefineData.java @@ -16,74 +16,12 @@ */ package org.apache.dolphinscheduler.data.project; -public class WorkflowDefineData { - /** - * create workflow data - */ - - public static final String workflow_define = "工作流定义"; - - //input shell task name - public static final String SHELL_TASK_NAME = "shell_task_selenium_1"; - - //input shell task description - public static final String SHELL_TASK_DESCRIPTION = "shell task description test"; - - //input timeout - public static final String INPUT_TIMEOUT = "60"; - - //input shell script - public static final String SHELL_SCRIPT = "echo 1111111"; - - //input custom parameters - public static final String INPUT_CUSTOM_PARAMETERS = "selenium_parameter"; - - //input custom parameters value - public static final String INPUT_CUSTOM_PARAMETERS_VALUE = "selenium_parameter_123"; - - //input add custom parameters - public static final String INPUT_ADD_CUSTOM_PARAMETERS = "selenium_parameter_delete"; - - //input add custom parameters value - public static final String INPUT_ADD_CUSTOM_PARAMETERS_VALUE = "selenium_parameter_delete_456"; - - //workflow define title - public static final String WORKFLOW_TITLE = "工作流定义 - DolphinScheduler"; - - //create workflow title - public static final String CREATE_WORKFLOW_TITLE = "创建流程定义 - DolphinScheduler"; - - - /** - * save workflow data - */ - //input workflow name - public static final String INPUT_WORKFLOW_NAME = "selenium_shell_1"; - - //input workflow description - public static final String INPUT_WORKFLOW_DESCRIPTION = "test selenium_shell_1 description"; - - //input workflow timeout - public static final String INPUT_WORKFLOW_TIMEOUT = "30"; - - //input workflow global parameters - public static final String INPUT_WORKFLOW_GLOBAL_PARAMETERS = "selenium_global_parameters_1"; - - //input workflow global parameters value - public static final String INPUT_WORKFLOW_GLOBAL_PARAMETERS_VALUES = "selenium_global_parameters_value_1"; - - //input to add workflow global parameters - public static final String INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS = "selenium_global_parameters_2"; - - //input to add workflow global parameters value - public static final String INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS_VALUES = "selenium_global_parameters_value_2"; - - /** - * workflowDefine list - */ - public static final String WORKFLOW_ONLINE_STATE = "上线"; - - public static final String WORKFLOW_OFFLINE_STATE = "下线"; - +import org.apache.dolphinscheduler.util.YmlReader; +public class WorkflowDefineData { + public String getWorkflowDefineData(String param) { + YmlReader ymlReader = new YmlReader(); + String workflowDefineData = ymlReader.getDataYml("testData/workflow_zh_cn", "workflowDefine", param); + return workflowDefineData; + } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/security/AlertManageData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/security/AlertManageData.java index ae996a209d..6bb2db4528 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/data/security/AlertManageData.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/security/AlertManageData.java @@ -16,17 +16,13 @@ */ package org.apache.dolphinscheduler.data.security; -public class AlertManageData { - //Alert Name - public static final String ALERT_NAME = "selenium_alert_Name"; - public static final String CREATE_ALERT = "创建告警组"; - - // Alert Type - public static final String ALERT_TYPE = "邮件"; +import org.apache.dolphinscheduler.util.YmlReader; - //Alert Description - public static final String DESCRIPTION = "create alert test"; - - public static final String ALERT_MANAGE = "告警组管理 - DolphinScheduler"; +public class AlertManageData { + public String getAlertData(String param) { + YmlReader ymlReader = new YmlReader(); + String alertData = ymlReader.getDataYml("testData/security_zh_cn", "alertManage", param); + return alertData; + } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/security/QueueManageData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/security/QueueManageData.java index 46f0d834d1..4b721602be 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/data/security/QueueManageData.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/security/QueueManageData.java @@ -16,29 +16,12 @@ */ package org.apache.dolphinscheduler.data.security; -public class QueueManageData { - /** - * Create Queue Name - */ - public static final String QUEUE_NAME = "selenium_queue_name"; - - public static final String CREATE_QUEUE = "创建队列"; - - /** - * Create Queue Value - */ - public static final String QUEUE_VALUE = "selenium_queue_value"; - - /** - * Edit Queue Name - */ - public static final String EDIT_QUEUE_NAME = "selenium_queue_value_edit"; - - /** - * Edit Queue Value - */ - public static final String EDIT_QUEUE_VALUE = "selenium_queue_value_edit"; - - public static final String QUEUE_MANAGE = "队列管理 - DolphinScheduler"; +import org.apache.dolphinscheduler.util.YmlReader; +public class QueueManageData { + public String getQueueData(String param) { + YmlReader ymlReader = new YmlReader(); + String queueData = ymlReader.getDataYml("testData/security_zh_cn", "queueManage", param); + return queueData; + } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/security/TenantManageData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/security/TenantManageData.java index 71e50e7150..0c9727799d 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/data/security/TenantManageData.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/security/TenantManageData.java @@ -20,30 +20,12 @@ */ package org.apache.dolphinscheduler.data.security; +import org.apache.dolphinscheduler.util.YmlReader; public class TenantManageData { - /** - * Tenant Code - */ - public static final String TENANT_CODE = "selenium_tenant_code_1"; - - /** - * Tenant Name - */ - public static final String TENANT_NAME = "selenium_tenant_Name"; - - /** - * Queue - */ - public static final String QUEUE = "default"; - - /** - * Description - */ - public static final String DESCRIPTION = "create tenant test"; - - public static final String TENANT_MANAGE = "租户管理 - DolphinScheduler"; - - - + public String getTenantData(String param) { + YmlReader ymlReader = new YmlReader(); + String tenantData = ymlReader.getDataYml("testData/security_zh_cn", "tenantManage", param); + return tenantData; + } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/security/TokenManageData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/security/TokenManageData.java index 9dd625ec25..10fdc25795 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/data/security/TokenManageData.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/security/TokenManageData.java @@ -16,13 +16,12 @@ */ package org.apache.dolphinscheduler.data.security; -public class TokenManageData { - public static final String TOKEN_MANAGE = "令牌管理 - DolphinScheduler"; - public static final String CREATE_TOKEN = "创建令牌"; - - public static final String DATE = "2038-06-10 00:00:00"; - public static final String NAME = "admin"; - - +import org.apache.dolphinscheduler.util.YmlReader; +public class TokenManageData { + public String getTokenData(String param) { + YmlReader ymlReader = new YmlReader(); + String tokenData = ymlReader.getDataYml("testData/security_zh_cn", "tokenManage", param); + return tokenData; + } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/security/UserManageData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/security/UserManageData.java index ee7fc60bc4..699cce4568 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/data/security/UserManageData.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/security/UserManageData.java @@ -16,29 +16,12 @@ */ package org.apache.dolphinscheduler.data.security; +import org.apache.dolphinscheduler.util.YmlReader; public class UserManageData { - /** - * create user - */ - public static final String USERNAME = "selenium_user_1"; - - public static final String PASSWORD = "123456qwe"; - - public static final String EMAIL = "123456789@qq.com"; - - public static final String PHONE = "15811112222"; - - public static final String USER_MANAGE = "用户管理 - DolphinScheduler"; - - public static final String CREATE_USER_BUTTON = "创建用户"; - - - /** - * edit user - */ - public static final String EDIT_USERNAME = "selenium_user_edit"; - public static final String EDIT_PASSWORD = "123456qwe"; - public static final String EDIT_EMAIL = "123456_edit@qq.com"; - public static final String EDIT_PHONE = "15800001111"; + public String getUserData(String param) { + YmlReader ymlReader = new YmlReader(); + String userData = ymlReader.getDataYml("testData/security_zh_cn", "userManage", param); + return userData; + } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/WorkflowDefineLocator.java b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/WorkflowDefineLocator.java index 01fdb39e4a..c851be3d0d 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/WorkflowDefineLocator.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/WorkflowDefineLocator.java @@ -179,6 +179,8 @@ public class WorkflowDefineLocator { //scroll to element bottom public static final By SCROLL_BOTTOM = By.xpath("//span/a/em"); + public static final By WORKFLOW_NAME = By.xpath("//table/tr[2]/td[3]/span/a"); + /** * online workflow */ diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/TokenManageLocator.java b/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/TokenManageLocator.java index d935dd54a0..e85807fa68 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/TokenManageLocator.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/TokenManageLocator.java @@ -24,18 +24,18 @@ public class TokenManageLocator { public static final By CLICK_CREATE_TOKEN = By.xpath("//div[2]/div/div[2]/div[2]/div/div[1]/button/span"); + public static final By CREATE_TOKEN_POPUP = By.xpath("//div[5]/div/div[2]/div/div[1]/span"); + public static final By SELECT_USER = By.xpath("//div[2]/div[2]/div/div/div/span/i"); public static final By CLICK_GENERATE_TOKEN_BUTTON = By.xpath("//div[3]/div[2]/button/span"); public static final By CLICK_SUBMIT_BUTTON = By.xpath("//div[3]/button[2]/span"); - public static final By EDIT_TOKEN_BUTTON = By.xpath("//table/tr[2]/td[7]/button"); - //edit token public static final By TOKEN = By.xpath("//table/tr[2]/td[1]/span"); + public static final By EDIT_TOKEN_BUTTON = By.xpath("//table/tr[2]/td[7]/button"); - public static final By CLICK_EDIT_BUTTON = By.xpath("//div[3]/div[1]/div/table/tr[2]/td[7]/button/i"); //delete token public static final By CLICK_DELETE_BUTTON = By.xpath("//div[3]/div[1]/div/table/tr[2]/td[7]/span/button"); diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/UserManageLocator.java b/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/UserManageLocator.java index ecfd285d60..80f9df20c0 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/UserManageLocator.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/UserManageLocator.java @@ -49,6 +49,8 @@ public class UserManageLocator { public static final By SUBMIT = By.xpath("//div[3]/button[2]/span"); + public static final By USERNAME = By.xpath("//table/tr[2]/td[2]/span"); + /** * edit user */ diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/LoginPage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/LoginPage.java index 39bd152d0a..468938d342 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/LoginPage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/LoginPage.java @@ -18,13 +18,14 @@ package org.apache.dolphinscheduler.page; import org.apache.dolphinscheduler.common.PageCommon; import org.apache.dolphinscheduler.data.LoginData; +import org.apache.dolphinscheduler.data.security.TenantManageData; import org.apache.dolphinscheduler.locator.LoginLocator; import org.openqa.selenium.Cookie; import org.openqa.selenium.WebDriver; - - public class LoginPage extends PageCommon { + TenantManageData tenantManageData = new TenantManageData(); + /** * Unique constructor * @param driver driver @@ -33,7 +34,6 @@ public class LoginPage extends PageCommon { super(driver); } - /** * jump page */ @@ -65,6 +65,6 @@ public class LoginPage extends PageCommon { moveToElement(LoginLocator.LOGIN_BUTTON_MOVE); // Whether to enter the specified page after login - return ifTitleContains(LoginData.TENANT); + return ifTitleContains(tenantManageData.getTenantData("tenantTitle")); } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/ProcessInstancePage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/ProcessInstancePage.java index 8c251afc84..04124b467f 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/ProcessInstancePage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/ProcessInstancePage.java @@ -23,6 +23,8 @@ import org.apache.dolphinscheduler.data.project.ProcessInstanceData; public class ProcessInstancePage extends PageCommon { + ProcessInstanceData processInstanceData = new ProcessInstanceData(); + public ProcessInstancePage(WebDriver driver) { super(driver); } @@ -36,7 +38,7 @@ public class ProcessInstancePage extends PageCommon { clickTopElement(ProcessInstanceLocator.CLICK_PROCESS_INSTANCE_NAME); locateElement(ProcessInstanceLocator.PROCESS_INSTANCE_SUCCESS_STATE); clickTopElement(ProcessInstanceLocator.CLICK_RERUN_BUTTON); - return ifTitleContains(ProcessInstanceData.PROCESS_INSTANCE_TITLE); + return ifTitleContains(processInstanceData.getProcessInstanceData("processInstanceTitle")); } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/ProjectPage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/ProjectPage.java index 6a93094340..a450ec7121 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/ProjectPage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/ProjectPage.java @@ -17,12 +17,13 @@ package org.apache.dolphinscheduler.page.project; import org.apache.dolphinscheduler.common.PageCommon; -import org.apache.dolphinscheduler.constant.TestConstant; import org.apache.dolphinscheduler.data.project.ProjectData; import org.apache.dolphinscheduler.locator.project.ProjectLocator; import org.openqa.selenium.WebDriver; public class ProjectPage extends PageCommon { + ProjectData projectData = new ProjectData(); + public ProjectPage(WebDriver driver) { super(driver); } @@ -32,7 +33,7 @@ public class ProjectPage extends PageCommon { */ public boolean jumpProjectManagePage() throws InterruptedException { clickTopElement(ProjectLocator.PROJECT_MANAGE); - return ifTitleContains(ProjectData.PROJECT_TITLE); + return ifTitleContains(projectData.getProjectData("projectTitle")); } /** @@ -41,18 +42,18 @@ public class ProjectPage extends PageCommon { * @return Whether to enter the specified page after create project */ public boolean createProject() throws InterruptedException { - ifTextExists(ProjectLocator.CREATE_PROJECT_BUTTON,ProjectData.CREATE_PROJECT_BUTTON); + ifTextExists(ProjectLocator.CREATE_PROJECT_BUTTON, projectData.getProjectData("createProjectButton")); clickElement(ProjectLocator.CREATE_PROJECT_BUTTON); // input create project data - sendInput(ProjectLocator.PROJECT_NAME, ProjectData.PROJECT_NAME); - sendInput(ProjectLocator.PROJECT_DESCRIPTION, ProjectData.DESCRIPTION); + sendInput(ProjectLocator.PROJECT_NAME, projectData.getProjectData("projectName")); + sendInput(ProjectLocator.PROJECT_DESCRIPTION, projectData.getProjectData("description")); // click submit button clickButton(ProjectLocator.SUBMIT_BUTTON); // Whether to enter the specified page after submit - return ifTextExists(ProjectLocator.LIST_PROJECT_NAME,ProjectData.PROJECT_NAME); + return ifTextExists(ProjectLocator.LIST_PROJECT_NAME, projectData.getProjectData("projectName")); } /** @@ -68,6 +69,6 @@ public class ProjectPage extends PageCommon { clickElement(ProjectLocator.CONFIRM_DELETE_PROJECT_BUTTON); // Whether to enter the specified page after submit - return ifTitleContains(ProjectData.PROJECT_TITLE); + return ifTitleContains(projectData.getProjectData("projectTitle")); } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/RunWorkflowPage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/RunWorkflowPage.java index 921e593e44..c4b8a68dee 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/RunWorkflowPage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/RunWorkflowPage.java @@ -24,13 +24,16 @@ import org.apache.dolphinscheduler.locator.project.WorkflowDefineLocator; import org.openqa.selenium.WebDriver; public class RunWorkflowPage extends PageCommon { + RunWorkflowData runWorkflowData = new RunWorkflowData(); + WorkflowDefineData workflowDefineData = new WorkflowDefineData(); + public RunWorkflowPage(WebDriver driver) { super(driver); } public boolean runWorkflow() throws InterruptedException { // Determine whether the workflow status is online - ifTextExists(WorkflowDefineLocator.WORKFLOW_STATE, WorkflowDefineData.WORKFLOW_ONLINE_STATE); + ifTextExists(WorkflowDefineLocator.WORKFLOW_STATE, runWorkflowData.getRunWorkflowData("online")); // click run workflow button System.out.println("Click run workflow button"); @@ -46,10 +49,10 @@ public class RunWorkflowPage extends PageCommon { clickElement(RunWorkflowLocator.SELECT_WORKER_GROUP); clickElement(RunWorkflowLocator.CLICK_NOTICE_GROUP); clickElement(RunWorkflowLocator.SELECT_NOTICE_GROUP); - sendInput(RunWorkflowLocator.INPUT_RECIPIENT, RunWorkflowData.RECIPIENT); - sendInput(RunWorkflowLocator.INPUT_Cc,RunWorkflowData.Cc); + sendInput(RunWorkflowLocator.INPUT_RECIPIENT, runWorkflowData.getRunWorkflowData("recipient")); + sendInput(RunWorkflowLocator.INPUT_Cc, runWorkflowData.getRunWorkflowData("Cc")); clickButton(RunWorkflowLocator.CLICK_RUNNING_BUTTON); - return ifTitleContains(RunWorkflowData.RUN_WORKFLOW_TITLE); + return ifTitleContains(workflowDefineData.getWorkflowDefineData("workflowDefineTitle")); } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/TimingPage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/TimingPage.java index 3febc545ff..37f3fdab2a 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/TimingPage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/TimingPage.java @@ -18,23 +18,24 @@ package org.apache.dolphinscheduler.page.project; import org.apache.dolphinscheduler.common.PageCommon; import org.apache.dolphinscheduler.data.project.TimingData; -import org.apache.dolphinscheduler.data.project.WorkflowDefineData; import org.apache.dolphinscheduler.locator.project.TimingLocator; import org.apache.dolphinscheduler.locator.project.WorkflowDefineLocator; import org.openqa.selenium.WebDriver; public class TimingPage extends PageCommon { + TimingData timingData = new TimingData(); + public TimingPage(WebDriver driver) { super(driver); } - /** * create timing */ public boolean createTiming() throws InterruptedException { + flushPage(); // Determine whether the workflow status is online - ifTextExists(WorkflowDefineLocator.WORKFLOW_STATE, WorkflowDefineData.WORKFLOW_ONLINE_STATE); + ifTextExists(WorkflowDefineLocator.WORKFLOW_STATE, timingData.getTimingData("online")); // click timing button System.out.println("Click timing button"); @@ -52,11 +53,11 @@ public class TimingPage extends PageCommon { clickElement(TimingLocator.SELECT_WORKER_GROUP); clickElement(TimingLocator.CLICK_NOTICE_GROUP); clickElement(TimingLocator.SELECT_NOTICE_GROUP); - sendInput(TimingLocator.INPUT_RECIPIENT, TimingData.RECIPIENT); - sendInput(TimingLocator.INPUT_Cc,TimingData.Cc); + sendInput(TimingLocator.INPUT_RECIPIENT, timingData.getTimingData("recipient")); + sendInput(TimingLocator.INPUT_Cc, timingData.getTimingData("Cc")); clickButton(TimingLocator.CLICK_CREATE_BUTTON); - return ifTextExists(TimingLocator.TIMING_STATE, TimingData.TIMING_OFFLINE_STATE); + return ifTextExists(TimingLocator.TIMING_STATE, timingData.getTimingData("offline")); } /** @@ -68,7 +69,7 @@ public class TimingPage extends PageCommon { clickButton(TimingLocator.CLICK_TIMING_MANAGEMENT_BUTTON); // Determine whether the workflow name exists - ifTextExists(TimingLocator.WORKFLOW_NAME, WorkflowDefineData.INPUT_WORKFLOW_NAME); + ifTextExists(TimingLocator.WORKFLOW_NAME, timingData.getTimingData("offline")); System.out.println("Click edit timing button"); clickButton(TimingLocator.CLICK_EDIT_TIMING_BUTTON); @@ -85,11 +86,11 @@ public class TimingPage extends PageCommon { clickElement(TimingLocator.SELECT_WORKER_GROUP); clickElement(TimingLocator.CLICK_NOTICE_GROUP); clickElement(TimingLocator.SELECT_NOTICE_GROUP); - sendInput(TimingLocator.INPUT_RECIPIENT, TimingData.EDIT_RECIPIENT); - sendInput(TimingLocator.INPUT_Cc,TimingData.EDIT_Cc); + sendInput(TimingLocator.INPUT_RECIPIENT, timingData.getTimingData("editRecipient")); + sendInput(TimingLocator.INPUT_Cc, timingData.getTimingData("editCc")); clickButton(TimingLocator.CLICK_CREATE_BUTTON); - return ifTitleContains(TimingData.TIMING_TITLE ); + return ifTitleContains(timingData.getTimingData("timingTitle")); } @@ -99,13 +100,13 @@ public class TimingPage extends PageCommon { public boolean onlineTiming() throws InterruptedException { flushPage(); // Determine whether the timing is offline - ifTextExists(TimingLocator.TIMING_MANAGEMENT_TIMING_STATE, TimingData.TIMING_OFFLINE_STATE); + ifTextExists(TimingLocator.TIMING_MANAGEMENT_TIMING_STATE, timingData.getTimingData("offline")); // click online timing button System.out.println("Click online timing button"); clickElement(TimingLocator.CLICK_ONLINE_TIMING_BUTTON); - return ifTextExists(TimingLocator.TIMING_MANAGEMENT_TIMING_STATE, TimingData.TIMING_ONLINE_STATE); + return ifTextExists(TimingLocator.TIMING_MANAGEMENT_TIMING_STATE, timingData.getTimingData("online")); } @@ -115,13 +116,13 @@ public class TimingPage extends PageCommon { public boolean offlineTiming() throws InterruptedException { flushPage(); // Determine whether the timing is online - ifTextExists(TimingLocator.TIMING_MANAGEMENT_TIMING_STATE, TimingData.TIMING_ONLINE_STATE); + ifTextExists(TimingLocator.TIMING_MANAGEMENT_TIMING_STATE, timingData.getTimingData("online")); // click offline timing button System.out.println("Click offline timing button"); clickElement(TimingLocator.CLICK_OFFLINE_TIMING_BUTTON); - return ifTextExists(TimingLocator.TIMING_MANAGEMENT_TIMING_STATE, TimingData.TIMING_OFFLINE_STATE); + return ifTextExists(TimingLocator.TIMING_MANAGEMENT_TIMING_STATE, timingData.getTimingData("offline")); } @@ -131,7 +132,7 @@ public class TimingPage extends PageCommon { */ public boolean deleteTiming() throws InterruptedException { // Determine whether the timing is offline - ifTextExists(TimingLocator.TIMING_MANAGEMENT_TIMING_STATE, TimingData.TIMING_OFFLINE_STATE); + ifTextExists(TimingLocator.TIMING_MANAGEMENT_TIMING_STATE, timingData.getTimingData("offline")); // click offline timing button System.out.println("Click delete timing button"); diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/WorkflowDefinePage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/WorkflowDefinePage.java index 8826222189..72722d8f10 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/WorkflowDefinePage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/WorkflowDefinePage.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.page.project; import org.apache.dolphinscheduler.common.PageCommon; -import org.apache.dolphinscheduler.constant.TestConstant; import org.apache.dolphinscheduler.data.project.ProjectData; import org.apache.dolphinscheduler.data.project.WorkflowDefineData; import org.apache.dolphinscheduler.locator.project.ProjectLocator; @@ -25,6 +24,9 @@ import org.apache.dolphinscheduler.locator.project.WorkflowDefineLocator; import org.openqa.selenium.WebDriver; public class WorkflowDefinePage extends PageCommon { + WorkflowDefineData workflowDefineData = new WorkflowDefineData(); + ProjectData projectData = new ProjectData(); + public WorkflowDefinePage(WebDriver driver) { super(driver); } @@ -34,18 +36,18 @@ public class WorkflowDefinePage extends PageCommon { */ public boolean jumpWorkflowPage() throws InterruptedException { - ifTextExists(ProjectLocator.LIST_PROJECT_NAME, ProjectData.PROJECT_NAME); + ifTextExists(ProjectLocator.LIST_PROJECT_NAME, projectData.getProjectData("projectName")); // click project name clickElement(WorkflowDefineLocator.CLICK_PROJECT_NAME); - ifTextExists(WorkflowDefineLocator.CLICK_WORKFLOW_DEFINE,WorkflowDefineData.workflow_define); + ifTextExists(WorkflowDefineLocator.CLICK_WORKFLOW_DEFINE,workflowDefineData.getWorkflowDefineData("workflowDefine")); System.out.println("Click on workflow define to jump to workflow define page"); // click workflow define clickElement(WorkflowDefineLocator.CLICK_WORKFLOW_DEFINE); - return ifTitleContains(WorkflowDefineData.WORKFLOW_TITLE); + return ifTitleContains(workflowDefineData.getWorkflowDefineData("workflowDefineTitle")); } public boolean createWorkflow() throws InterruptedException { @@ -58,8 +60,7 @@ public class WorkflowDefinePage extends PageCommon { dragAndDrop(WorkflowDefineLocator.MOUSE_DOWN_AT_SHELL, WorkflowDefineLocator.MOUSE_MOVE_SHELL_AT_DAG); //input shell task _name - sendInput(WorkflowDefineLocator.INPUT_SHELL_TASK_NAME , WorkflowDefineData.SHELL_TASK_NAME); - + sendInput(WorkflowDefineLocator.INPUT_SHELL_TASK_NAME, workflowDefineData.getWorkflowDefineData("shellTaskName")); //click stop run type clickElement(WorkflowDefineLocator.CLICK_STOP_RUN_TYPE); @@ -67,7 +68,7 @@ public class WorkflowDefinePage extends PageCommon { clickElement(WorkflowDefineLocator.CLICK_NORMAL_RUN_TYPE); //input shell task description - sendInput(WorkflowDefineLocator.INPUT_SHELL_TASK_DESCRIPTION , WorkflowDefineData.SHELL_TASK_DESCRIPTION); + sendInput(WorkflowDefineLocator.INPUT_SHELL_TASK_DESCRIPTION, workflowDefineData.getWorkflowDefineData("shellTaskDescription")); //select task priority clickElement(WorkflowDefineLocator.CLICK_TASK_PRIORITY); @@ -100,20 +101,20 @@ public class WorkflowDefinePage extends PageCommon { clearInput(WorkflowDefineLocator.SELECT_TIMEOUT); //input timeout - sendInput(WorkflowDefineLocator.SELECT_TIMEOUT, WorkflowDefineData.INPUT_TIMEOUT); + sendInput(WorkflowDefineLocator.SELECT_TIMEOUT, workflowDefineData.getWorkflowDefineData("taskTimeout")); //click codeMirror and input script - inputCodeMirror(WorkflowDefineLocator.CLICK_CODE_MIRROR, WorkflowDefineLocator.INPUT_SCRIPT, WorkflowDefineData.SHELL_SCRIPT); + inputCodeMirror(WorkflowDefineLocator.CLICK_CODE_MIRROR, WorkflowDefineLocator.INPUT_SCRIPT, workflowDefineData.getWorkflowDefineData("shellScript")); scrollToElementBottom(WorkflowDefineLocator.SCROLL_BOTTOM); //click custom parameters clickElement(WorkflowDefineLocator.CLICK_CUSTOM_PARAMETERS); //input custom parameters - sendInput(WorkflowDefineLocator.INPUT_CUSTOM_PARAMETERS, WorkflowDefineData.INPUT_CUSTOM_PARAMETERS); + sendInput(WorkflowDefineLocator.INPUT_CUSTOM_PARAMETERS, workflowDefineData.getWorkflowDefineData("customParameter1")); //input custom parameters value - sendInput(WorkflowDefineLocator.INPUT_CUSTOM_PARAMETERS_VALUE, WorkflowDefineData.INPUT_CUSTOM_PARAMETERS_VALUE); + sendInput(WorkflowDefineLocator.INPUT_CUSTOM_PARAMETERS_VALUE, workflowDefineData.getWorkflowDefineData("customParameterValue1")); //click add custom parameters clickElement(WorkflowDefineLocator.CLICK_ADD_CUSTOM_PARAMETERS); @@ -121,10 +122,10 @@ public class WorkflowDefinePage extends PageCommon { scrollToElementBottom(WorkflowDefineLocator.SCROLL_BOTTOM); //input add custom parameters - sendInput(WorkflowDefineLocator.INPUT_ADD_CUSTOM_PARAMETERS, WorkflowDefineData.INPUT_ADD_CUSTOM_PARAMETERS); + sendInput(WorkflowDefineLocator.INPUT_ADD_CUSTOM_PARAMETERS, workflowDefineData.getWorkflowDefineData("customParameter2")); //input add custom parameters value - sendInput(WorkflowDefineLocator.INPUT_ADD_CUSTOM_PARAMETERS_VALUE, WorkflowDefineData.INPUT_ADD_CUSTOM_PARAMETERS_VALUE); + sendInput(WorkflowDefineLocator.INPUT_ADD_CUSTOM_PARAMETERS_VALUE, workflowDefineData.getWorkflowDefineData("customParameterValue2")); //click delete custom parameters clickElement(WorkflowDefineLocator.CLICK_DELETE_CUSTOM_PARAMETERS); @@ -140,7 +141,7 @@ public class WorkflowDefinePage extends PageCommon { clickButton(WorkflowDefineLocator.COPY_TASK); clickButton(WorkflowDefineLocator.CLICK_LINE); mouseMovePosition(WorkflowDefineLocator.LINE_SOURCES_TASK,WorkflowDefineLocator.LINE_TARGET_TASK); - return ifTitleContains(WorkflowDefineData.CREATE_WORKFLOW_TITLE); + return ifTitleContains(workflowDefineData.getWorkflowDefineData("createWorkflowTitle")); } /** @@ -153,10 +154,10 @@ public class WorkflowDefinePage extends PageCommon { clickElement(WorkflowDefineLocator.CLICK_SAVE_WORKFLOW_BUTTON); //input workflow name - sendInput(WorkflowDefineLocator.INPUT_WORKFLOW_NAME, WorkflowDefineData.INPUT_WORKFLOW_NAME); + sendInput(WorkflowDefineLocator.INPUT_WORKFLOW_NAME, workflowDefineData.getWorkflowDefineData("workflowName")); //input workflow description - sendInput(WorkflowDefineLocator.INPUT_WORKFLOW_DESCRIPTION, WorkflowDefineData.INPUT_WORKFLOW_DESCRIPTION); + sendInput(WorkflowDefineLocator.INPUT_WORKFLOW_DESCRIPTION, workflowDefineData.getWorkflowDefineData("workflowDescription")); //select tenant clickElement(WorkflowDefineLocator.CLICK_TENANT); @@ -167,25 +168,25 @@ public class WorkflowDefinePage extends PageCommon { clearInput(WorkflowDefineLocator.INPUT_WORKFLOW_TIMEOUT); //input workflow timeout - sendInput(WorkflowDefineLocator.INPUT_WORKFLOW_TIMEOUT, WorkflowDefineData.INPUT_WORKFLOW_TIMEOUT); + sendInput(WorkflowDefineLocator.INPUT_WORKFLOW_TIMEOUT, workflowDefineData.getWorkflowDefineData("workflowTimeout")); //click workflow global parameters clickElement(WorkflowDefineLocator.CLICK_WORKFLOW_GLOBAL_PARAMETERS); //input workflow global parameters - sendInput(WorkflowDefineLocator.INPUT_WORKFLOW_GLOBAL_PARAMETERS, WorkflowDefineData.INPUT_WORKFLOW_GLOBAL_PARAMETERS); + sendInput(WorkflowDefineLocator.INPUT_WORKFLOW_GLOBAL_PARAMETERS, workflowDefineData.getWorkflowDefineData("globalParameter1")); //input workflow global parameters value - sendInput(WorkflowDefineLocator.INPUT_WORKFLOW_GLOBAL_PARAMETERS_VALUES, WorkflowDefineData.INPUT_WORKFLOW_GLOBAL_PARAMETERS_VALUES); + sendInput(WorkflowDefineLocator.INPUT_WORKFLOW_GLOBAL_PARAMETERS_VALUES, workflowDefineData.getWorkflowDefineData("globalParameterValue1")); //click to add workflow global parameters clickElement(WorkflowDefineLocator.CLICK_ADD_WORKFLOW_GLOBAL_PARAMETERS); //input to add workflow global parameters - sendInput(WorkflowDefineLocator.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS, WorkflowDefineData.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS); + sendInput(WorkflowDefineLocator.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS, workflowDefineData.getWorkflowDefineData("globalParameter2")); //input to add workflow global parameters value - sendInput(WorkflowDefineLocator.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS_VALUES, WorkflowDefineData.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS_VALUES); + sendInput(WorkflowDefineLocator.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS_VALUES, workflowDefineData.getWorkflowDefineData("globalParameterValue2")); //delete workflow global parameters value clickElement(WorkflowDefineLocator.CLICK_DELETE_WORKFLOW_GLOBAL_PARAMETERS); @@ -194,33 +195,33 @@ public class WorkflowDefinePage extends PageCommon { System.out.println("submit workflow"); clickButton(WorkflowDefineLocator.CLICK_ADD_BUTTON); - return ifTitleContains(WorkflowDefineData.CREATE_WORKFLOW_TITLE); + return ifTitleContains(workflowDefineData.getWorkflowDefineData("createWorkflowTitle")); } public boolean onlineWorkflow() throws InterruptedException { clickElement(WorkflowDefineLocator.CLICK_WORKFLOW_DEFINE); // Determine whether the workflow status is offline - ifTextExists(WorkflowDefineLocator.WORKFLOW_STATE,WorkflowDefineData.WORKFLOW_OFFLINE_STATE); + ifTextExists(WorkflowDefineLocator.WORKFLOW_STATE, workflowDefineData.getWorkflowDefineData("offline")); // click online button System.out.println("Click online workflow button"); clickButton(WorkflowDefineLocator.CLICK_ONLINE_WORKFLOW_BUTTON); - return ifTitleContains(WorkflowDefineData.WORKFLOW_TITLE); + return ifTextExists(WorkflowDefineLocator.WORKFLOW_STATE, workflowDefineData.getWorkflowDefineData("online")); } public boolean offlineWorkflow() throws InterruptedException { clickElement(WorkflowDefineLocator.CLICK_WORKFLOW_DEFINE); // Determine whether the workflow status is online - ifTextExists(WorkflowDefineLocator.WORKFLOW_STATE,WorkflowDefineData.WORKFLOW_ONLINE_STATE); + ifTextExists(WorkflowDefineLocator.WORKFLOW_STATE, workflowDefineData.getWorkflowDefineData("online")); // click offline button System.out.println("offline workflow"); clickButton(WorkflowDefineLocator.CLICK_OFFLINE_WORKFLOW_BUTTON); - return ifTitleContains(WorkflowDefineData.WORKFLOW_TITLE); + return ifTextExists(WorkflowDefineLocator.WORKFLOW_STATE, workflowDefineData.getWorkflowDefineData("offline")); } @@ -229,7 +230,7 @@ public class WorkflowDefinePage extends PageCommon { clickElement(WorkflowDefineLocator.CLICK_WORKFLOW_DEFINE); // Determine whether the workflow status is offline - ifTextExists(WorkflowDefineLocator.WORKFLOW_STATE,WorkflowDefineData.WORKFLOW_OFFLINE_STATE); + ifTextExists(WorkflowDefineLocator.WORKFLOW_STATE, workflowDefineData.getWorkflowDefineData("offline")); clickButton(WorkflowDefineLocator.DELETE_WORKFLOW_BOTTOM); @@ -237,6 +238,6 @@ public class WorkflowDefinePage extends PageCommon { clickButton(WorkflowDefineLocator.CONFIRM_DELETE_WORKFLOW_BOTTOM); // Whether to enter the specified page after submit - return ifTitleContains(WorkflowDefineData.WORKFLOW_TITLE); + return ifTitleContains(workflowDefineData.getWorkflowDefineData("workflowDefineTitle")); } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/security/AlertManagePage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/security/AlertManagePage.java index 541251e0db..97e167ec29 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/security/AlertManagePage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/security/AlertManagePage.java @@ -22,6 +22,8 @@ import org.apache.dolphinscheduler.locator.security.AlertManageLocator; import org.openqa.selenium.WebDriver; public class AlertManagePage extends PageCommon { + AlertManageData alertManageData = new AlertManageData(); + /** * Unique constructor * @param driver driver @@ -41,26 +43,26 @@ public class AlertManagePage extends PageCommon { clickElement(AlertManageLocator.CLICK_ALERT_MANAGE); //determine whether the create alert button exists - ifTextExists(AlertManageLocator.CLICK_CREATE_ALERT,AlertManageData.CREATE_ALERT); + ifTextExists(AlertManageLocator.CLICK_CREATE_ALERT,alertManageData.getAlertData("createAlert")); // click create alert button System.out.println("start click create alert button"); clickElement(AlertManageLocator.CLICK_CREATE_ALERT); // input alert data System.out.println("start input alert "); - sendInput(AlertManageLocator.INPUT_ALERT_NAME, AlertManageData.ALERT_NAME); + sendInput(AlertManageLocator.INPUT_ALERT_NAME, alertManageData.getAlertData("alertName")); clickElement(AlertManageLocator.CLICK_ALERT_TYPE); clickElement(AlertManageLocator.SELECT_ALERT_EMAIL); - sendInput(AlertManageLocator.INPUT_ALERT_DESCRIPTION, AlertManageData.DESCRIPTION); + sendInput(AlertManageLocator.INPUT_ALERT_DESCRIPTION, alertManageData.getAlertData("description")); // click button clickButton(AlertManageLocator.SUBMIT_ALERT); // Whether to enter the specified page after submit - return ifTextExists(AlertManageLocator.ALERT_NAME, AlertManageData.ALERT_NAME); + return ifTextExists(AlertManageLocator.ALERT_NAME, alertManageData.getAlertData("alertName")); } public boolean deleteAlert() throws InterruptedException { @@ -68,7 +70,7 @@ public class AlertManagePage extends PageCommon { // click alert manage clickElement(AlertManageLocator.CLICK_ALERT_MANAGE); - ifTextExists(AlertManageLocator.ALERT_NAME, AlertManageData.ALERT_NAME); + ifTextExists(AlertManageLocator.ALERT_NAME, alertManageData.getAlertData("alertName")); // click delete alert button clickButton(AlertManageLocator.DELETE_ALERT_BUTTON); @@ -77,6 +79,6 @@ public class AlertManagePage extends PageCommon { clickButton(AlertManageLocator.CONFIRM_DELETE_ALERT_BUTTON); // Whether to enter the specified page after submit - return ifTitleContains(AlertManageData.ALERT_MANAGE); + return ifTitleContains(alertManageData.getAlertData("alertTitle")); } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/security/QueueManagePage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/security/QueueManagePage.java index 368202e921..f6a55bb73b 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/security/QueueManagePage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/security/QueueManagePage.java @@ -22,6 +22,8 @@ import org.apache.dolphinscheduler.locator.security.QueueManageLocator; import org.openqa.selenium.WebDriver; public class QueueManagePage extends PageCommon { + QueueManageData queueManageData = new QueueManageData(); + /** * Unique constructor * @param driver driver @@ -41,7 +43,7 @@ public class QueueManagePage extends PageCommon { clickElement(QueueManageLocator.CLICK_QUEUE_MANAGE); //determine whether the create queue button exists - ifTextExists(QueueManageLocator.CLICK_CREATE_QUEUE,QueueManageData.CREATE_QUEUE); + ifTextExists(QueueManageLocator.CLICK_CREATE_QUEUE, queueManageData.getQueueData("createQueueButton")); // click create queue button System.out.println("start click create queue button"); @@ -49,15 +51,15 @@ public class QueueManagePage extends PageCommon { // input queue data System.out.println("start input queue"); - sendInput(QueueManageLocator.INPUT_QUEUE_NAME, QueueManageData.QUEUE_NAME); + sendInput(QueueManageLocator.INPUT_QUEUE_NAME, queueManageData.getQueueData("queueName")); - sendInput(QueueManageLocator.INPUT_QUEUE_VALUE, QueueManageData.QUEUE_VALUE); + sendInput(QueueManageLocator.INPUT_QUEUE_VALUE, queueManageData.getQueueData("queueValue")); // click button clickButton(QueueManageLocator.SUBMIT_QUEUE); // Whether to enter the specified page after submit - return ifTextExists(QueueManageLocator.LIST_QUEUE_NAME, QueueManageData.QUEUE_NAME); + return ifTextExists(QueueManageLocator.LIST_QUEUE_NAME, queueManageData.getQueueData("queueName")); } @@ -70,19 +72,19 @@ public class QueueManagePage extends PageCommon { // click queue manage clickElement(QueueManageLocator.CLICK_QUEUE_MANAGE); - ifTextExists(QueueManageLocator.LIST_QUEUE_NAME, QueueManageData.QUEUE_NAME); + ifTextExists(QueueManageLocator.LIST_QUEUE_NAME, queueManageData.getQueueData("queueName")); // click edit queue button clickButton(QueueManageLocator.CLICK_EDIT_QUEUE); // input queue data - clearSendInput(QueueManageLocator.INPUT_QUEUE_NAME, QueueManageData.EDIT_QUEUE_NAME); - clearSendInput(QueueManageLocator.INPUT_QUEUE_VALUE, QueueManageData.EDIT_QUEUE_VALUE); + clearSendInput(QueueManageLocator.INPUT_QUEUE_NAME, queueManageData.getQueueData("editQueueName")); + clearSendInput(QueueManageLocator.INPUT_QUEUE_VALUE, queueManageData.getQueueData("editQueueValue")); // click button clickButton(QueueManageLocator.SUBMIT_QUEUE); // Whether to enter the specified page after submit - return ifTextExists(QueueManageLocator.LIST_QUEUE_NAME, QueueManageData.EDIT_QUEUE_NAME); + return ifTextExists(QueueManageLocator.LIST_QUEUE_NAME, queueManageData.getQueueData("editQueueName")); } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/security/TenantManagePage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/security/TenantManagePage.java index 40d964fa33..44f45f1d2a 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/security/TenantManagePage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/security/TenantManagePage.java @@ -22,6 +22,8 @@ import org.apache.dolphinscheduler.locator.security.TenantManageLocator; import org.openqa.selenium.WebDriver; public class TenantManagePage extends PageCommon { + TenantManageData tenantManageData = new TenantManageData(); + /** * Unique constructor * @param driver driver @@ -37,7 +39,7 @@ public class TenantManagePage extends PageCommon { */ public boolean jumpSecurity() throws InterruptedException { clickTopElement(TenantManageLocator.SECURITY_CENTER); - return ifTitleContains(TenantManageData.TENANT_MANAGE); + return ifTitleContains(tenantManageData.getTenantData("tenantTitle")); } /** @@ -52,16 +54,16 @@ public class TenantManagePage extends PageCommon { clickButton(TenantManageLocator.CREATE_TENANT_BUTTON); // tenant data - sendInput(TenantManageLocator.TENANT_INPUT_CODE, TenantManageData.TENANT_CODE); - sendInput(TenantManageLocator.TENANT_INPUT_NAME, TenantManageData.TENANT_NAME); - sendInput(TenantManageLocator.QUEUE, TenantManageData.QUEUE); - sendInput(TenantManageLocator.DESCRIPTION, TenantManageData.DESCRIPTION); + sendInput(TenantManageLocator.TENANT_INPUT_CODE, tenantManageData.getTenantData("tenantCode")); + sendInput(TenantManageLocator.TENANT_INPUT_NAME, tenantManageData.getTenantData("tenantName")); + sendInput(TenantManageLocator.QUEUE, tenantManageData.getTenantData("queue")); + sendInput(TenantManageLocator.DESCRIPTION, tenantManageData.getTenantData("description")); // click button clickButton(TenantManageLocator.SUBMIT_BUTTON); // Whether to enter the specified page after submit - return ifTextExists(TenantManageLocator.TENANT_CODE_FIRST, TenantManageData.TENANT_CODE); + return ifTextExists(TenantManageLocator.TENANT_CODE_FIRST, tenantManageData.getTenantData("tenantCode")); } public boolean deleteTenant() throws InterruptedException { @@ -74,6 +76,6 @@ public class TenantManagePage extends PageCommon { clickButton(TenantManageLocator.CONFIRM_DELETE_TENANT_BUTTON); // Whether to enter the specified page after submit - return ifTitleContains(TenantManageData.TENANT_MANAGE); + return ifTitleContains(tenantManageData.getTenantData("tenantTitle")); } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/security/TokenManagePage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/security/TokenManagePage.java index 61fb93c570..35a9377578 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/security/TokenManagePage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/security/TokenManagePage.java @@ -22,6 +22,8 @@ import org.apache.dolphinscheduler.locator.security.TokenManageLocator; import org.openqa.selenium.WebDriver; public class TokenManagePage extends PageCommon { + TokenManageData tokenManageData = new TokenManageData(); + public TokenManagePage(WebDriver driver) { super(driver); } @@ -36,11 +38,13 @@ public class TokenManagePage extends PageCommon { clickElement(TokenManageLocator.CLICK_TOKEN_MANAGE); //determine whether the create token button exists - ifTextExists(TokenManageLocator.CLICK_CREATE_TOKEN,TokenManageData.CREATE_TOKEN); + ifTextExists(TokenManageLocator.CLICK_CREATE_TOKEN, tokenManageData.getTokenData("createTokenText")); // click create token button clickButton(TokenManageLocator.CLICK_CREATE_TOKEN); + ifTextExists(TokenManageLocator.CREATE_TOKEN_POPUP,tokenManageData.getTokenData("createTokenText")); + clickButton(TokenManageLocator.SELECT_USER); clickButton(TokenManageLocator.CLICK_GENERATE_TOKEN_BUTTON); @@ -50,7 +54,7 @@ public class TokenManagePage extends PageCommon { clickButton(TokenManageLocator.CLICK_SUBMIT_BUTTON); // Whether to enter the specified page after submit - return ifTitleContains(TokenManageData.TOKEN_MANAGE); + return ifTitleContains(tokenManageData.getTokenData("tokenTitle")); } /** @@ -66,7 +70,7 @@ public class TokenManagePage extends PageCommon { locateElement(TokenManageLocator.EDIT_TOKEN_BUTTON); // click edit token button - clickButton(TokenManageLocator.CLICK_EDIT_BUTTON); + clickButton(TokenManageLocator.EDIT_TOKEN_BUTTON); clickButton(TokenManageLocator.SELECT_USER); @@ -77,7 +81,7 @@ public class TokenManagePage extends PageCommon { clickButton(TokenManageLocator.CLICK_SUBMIT_BUTTON); // Whether to enter the specified page after submit - return ifTitleContains(TokenManageData.TOKEN_MANAGE); + return ifTitleContains(tokenManageData.getTokenData("tokenTitle")); } @@ -89,7 +93,7 @@ public class TokenManagePage extends PageCommon { clickButton(TokenManageLocator.CLICK_DELETE_BUTTON); clickButton(TokenManageLocator.CLICK_CONFIRM_DELETE_BUTTON); - return ifTitleContains(TokenManageData.TOKEN_MANAGE); + return ifTitleContains(tokenManageData.getTokenData("tokenTitle")); } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/security/UserManagePage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/security/UserManagePage.java index 367896a3cf..b489d12621 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/security/UserManagePage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/security/UserManagePage.java @@ -22,6 +22,8 @@ import org.apache.dolphinscheduler.locator.security.UserManageLocator; import org.openqa.selenium.WebDriver; public class UserManagePage extends PageCommon { + UserManageData userManageData = new UserManageData(); + public UserManagePage(WebDriver driver) { super(driver); } @@ -34,22 +36,21 @@ public class UserManagePage extends PageCommon { public boolean createUser() throws InterruptedException { // click user manage clickElement(UserManageLocator.CLICK_USER_MANAGE); - //determine whether the create user button exists - ifTextExists(UserManageLocator.CLICK_CREATE_USER_BUTTON,UserManageData.CREATE_USER_BUTTON); + ifTextExists(UserManageLocator.CLICK_CREATE_USER_BUTTON, userManageData.getUserData("createUserButton")); // click create user button clickButton(UserManageLocator.CLICK_CREATE_USER_BUTTON); // input user data - sendInput(UserManageLocator.INPUT_USERNAME, UserManageData.USERNAME); - sendInput(UserManageLocator.INPUT_PASSWORD, UserManageData.PASSWORD); + sendInput(UserManageLocator.INPUT_USERNAME, userManageData.getUserData("userName")); + sendInput(UserManageLocator.INPUT_PASSWORD, userManageData.getUserData("password")); clickButton(UserManageLocator.CLICK_TENANT); clickButton(UserManageLocator.SELECT_TENANT); clickButton(UserManageLocator.CLICK_QUEUE); clickButton(UserManageLocator.SELECT_QUEUE); - sendInput(UserManageLocator.INPUT_EMAIL, UserManageData.EMAIL); - sendInput(UserManageLocator.INPUT_PHONE, UserManageData.PHONE); + sendInput(UserManageLocator.INPUT_EMAIL, userManageData.getUserData("email")); + sendInput(UserManageLocator.INPUT_PHONE, userManageData.getUserData("phone")); clickElement(UserManageLocator.SELECT_STOP_STATE); clickElement(UserManageLocator.SELECT_ENABLE_STATE); @@ -57,7 +58,7 @@ public class UserManagePage extends PageCommon { clickButton(UserManageLocator.SUBMIT); // Whether to enter the specified page after submit - return ifTitleContains(UserManageData.USER_MANAGE); + return ifTextExists(UserManageLocator.USERNAME, userManageData.getUserData("userName")); } /** @@ -70,14 +71,14 @@ public class UserManagePage extends PageCommon { clickElement(UserManageLocator.CLICK_USER_MANAGE); //determine whether the general user exists - ifTextExists(UserManageLocator.USER_NAME,UserManageData.USERNAME); + ifTextExists(UserManageLocator.USERNAME, userManageData.getUserData("userName")); // click edit user button clickButton(UserManageLocator.EDIT_GENERAL_USER_BUTTON ); // input user data - clearSendInput(UserManageLocator.INPUT_USERNAME, UserManageData.EDIT_USERNAME); - clearSendInput(UserManageLocator.INPUT_PASSWORD, UserManageData.EDIT_PASSWORD); + clearSendInput(UserManageLocator.INPUT_USERNAME, userManageData.getUserData("editUserName")); + clearSendInput(UserManageLocator.INPUT_PASSWORD, userManageData.getUserData("editPassword")); clickButton(UserManageLocator.CLICK_TENANT); clickButton(UserManageLocator.SELECT_TENANT); @@ -85,8 +86,8 @@ public class UserManagePage extends PageCommon { clickButton(UserManageLocator.CLICK_QUEUE); clickButton(UserManageLocator.SELECT_QUEUE); - clearSendInput(UserManageLocator.INPUT_EMAIL, UserManageData.EDIT_EMAIL); - clearSendInput(UserManageLocator.INPUT_PHONE, UserManageData.EDIT_PHONE); + clearSendInput(UserManageLocator.INPUT_EMAIL, userManageData.getUserData("editEmail")); + clearSendInput(UserManageLocator.INPUT_PHONE, userManageData.getUserData("editPhone")); clickElement(UserManageLocator.SELECT_STOP_STATE); clickElement(UserManageLocator.SELECT_ENABLE_STATE); @@ -96,7 +97,7 @@ public class UserManagePage extends PageCommon { System.out.println("end edit general user"); // Whether to enter the specified page after submit - return ifTitleContains(UserManageData.USER_MANAGE); + return ifTitleContains(userManageData.getUserData("userTitle")); } /** @@ -109,7 +110,7 @@ public class UserManagePage extends PageCommon { clickElement(UserManageLocator.CLICK_USER_MANAGE); //determine whether the general user edit success - ifTextExists(UserManageLocator.USER_NAME,UserManageData.EDIT_USERNAME); + ifTextExists(UserManageLocator.USER_NAME,userManageData.getUserData("editUserName")); // click edit user button clickButton(UserManageLocator.EDIT_ADMIN_USER_BUTTON ); @@ -123,7 +124,7 @@ public class UserManagePage extends PageCommon { clickButton(UserManageLocator.SUBMIT); System.out.println("end edit admin user"); // Whether to enter the specified page after submit - return ifTitleContains(UserManageData.USER_MANAGE); + return ifTitleContains(userManageData.getUserData("userTitle")); } @@ -144,6 +145,6 @@ public class UserManagePage extends PageCommon { clickButton(UserManageLocator.CONFIRM_DELETE_USER_BUTTON); // Whether to enter the specified page after submit - return ifTitleContains(UserManageData.USER_MANAGE); + return ifTitleContains(userManageData.getUserData("userTitle")); } } diff --git a/e2e/src/test/resources/testData/security_zh_cn.yml b/e2e/src/test/resources/testData/security_zh_cn.yml new file mode 100644 index 0000000000..89be444df5 --- /dev/null +++ b/e2e/src/test/resources/testData/security_zh_cn.yml @@ -0,0 +1,55 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +userManage: + userName: selenium_user_11111 + password: 123456qwe + email: 123456789@qq.com + phone: '15811112222' + createUserButton: 创建用户 + editUserName: selenium_user_edit + editPassword: 123456qwe@asd + editEmail: 123456_edit@qq.com + editPhone: '15800001111' + userTitle: 用户管理 - DolphinScheduler + +tenantManage: + tenantCode: selenium_tenant_code_1 + tenantName: selenium_tenant_Name + queue: default + description: create tenant test + tenantTitle: 租户管理 - DolphinScheduler + +alertManage: + alertName: selenium_alert_Name + createAlert: 创建告警组 + alertType: 邮件 + description: create alert test + alertTitle: 告警组管理 - DolphinScheduler + +queueManage: + queueName: selenium_queue_name + queueValue: selenium_queue_value + createQueueButton: 创建队列 + editQueueName: selenium_queue_value_edit + editQueueValue: selenium_queue_value_edit + queueTitle: 队列管理 - DolphinScheduler + +tokenManage: + tokenTitle: 令牌管理 - DolphinScheduler + createTokenText: 创建令牌 + userName: admin diff --git a/e2e/src/test/resources/testData/workflow_zh_cn.yml b/e2e/src/test/resources/testData/workflow_zh_cn.yml new file mode 100644 index 0000000000..bb6dea7851 --- /dev/null +++ b/e2e/src/test/resources/testData/workflow_zh_cn.yml @@ -0,0 +1,63 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +projectManage: + createProjectButton: 创建项目 + projectName: selenium_project_1 + description: test create project description + projectTitle: 项目 - DolphinScheduler + +workflowDefine: + workflowDefine: 工作流定义 + shellTaskName: shell_task_selenium_1 + shellTaskDescription: shell task description test + taskTimeout: '60' + shellScript: echo "shell task test" + customParameter1: selenium_parameter + customParameterValue1: selenium_parameter_123 + customParameter2: selenium_parameter_delete + customParameterValue2: selenium_parameter_delete_456 + workflowDefineTitle: 工作流定义 - DolphinScheduler + createWorkflowTitle: 创建流程定义 - DolphinScheduler + workflowName: selenium_shell_1 + workflowDescription: test selenium_shell_1 description + workflowTimeout: '30' + globalParameter1: selenium_global_parameters_1 + globalParameterValue1: selenium_global_parameters_value_1 + globalParameter2: selenium_global_parameters_2 + globalParameterValue2: selenium_global_parameters_value_2 + online: 上线 + offline: 下线 + +runWorkflow: + recipient: 123456789@qq.com + Cc: qwe12312sds@qq.com + online: 上线 + offline: 下线 + +timing: + recipient: test123456@qq.com + Cc: test.123qwe@qq.com + editRecipient: test.edit123456@qq.com + editCc: test.edit123qwe@qq.com + timingTitle: 定时任务列表 - DolphinScheduler + online: 上线 + offline: 下线 + +processInstance: + processInstanceTitle: 工作流实例 - DolphinScheduler + rerun: 重跑 diff --git a/e2e/testng.xml b/e2e/testng.xml index e08c4da917..8a4791fea6 100644 --- a/e2e/testng.xml +++ b/e2e/testng.xml @@ -56,9 +56,9 @@ + - From 8ef043151c5f1a5cc0f87edd08d1614f12b8a421 Mon Sep 17 00:00:00 2001 From: Hsu Pu Date: Mon, 21 Sep 2020 16:49:25 +0800 Subject: [PATCH 22/45] done (#3752) --- tools/dependencies/check-LICENSE.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/dependencies/check-LICENSE.sh b/tools/dependencies/check-LICENSE.sh index d414bd40c6..0072554e29 100755 --- a/tools/dependencies/check-LICENSE.sh +++ b/tools/dependencies/check-LICENSE.sh @@ -25,7 +25,9 @@ tar -zxf dolphinscheduler-dist/target/apache-dolphinscheduler*-bin.tar.gz --stri # licenses echo '=== Self modules: ' && ./mvnw --batch-mode --quiet -Dexec.executable='echo' -Dexec.args='${project.artifactId}-${project.version}.jar' exec:exec | tee self-modules.txt -echo '=== Distributed dependencies: ' && ls dist/lib | tee all-dependencies.txt +echo '=== Distributed dependencies: ' && find dist/lib -name "*.jar" | tee all-dependencies.txt +# The prefix "dist/lib/" (9 chars) should be stripped to be ready to compare +sed -i 's/.\{9\}//' all-dependencies.txt # Exclude all self modules(jars) to generate all third-party dependencies echo '=== Third party dependencies: ' && grep -vf self-modules.txt all-dependencies.txt | tee third-party-dependencies.txt From 375ec125c5cc50a9634fac120a9936ccfc2807c7 Mon Sep 17 00:00:00 2001 From: BoYiZhang <39816903+BoYiZhang@users.noreply.github.com> Date: Tue, 22 Sep 2020 17:38:17 +0800 Subject: [PATCH 23/45] [Feature-3773][processInstance, taskInstance] Change title name : Replacement System.currentTimeMillis () (#3774) * fix bug Delete invalid field: executorcores Modify verification prompt * fix bug Delete invalid field: executorcores Modify verification prompt * fix bug Delete invalid field: executorcores Modify verification prompt * dag add close button * reset last version * reset last version * dag add close buttion dag add close buttion * update CLICK_SAVE_WORKFLOW_BUTTON xpath * updae CLICK_SAVE_WORKFLOW_BUTTON xpath * updae CLICK_SAVE_WORKFLOW_BUTTON xpath * updae CLICK_SAVE_WORKFLOW_BUTTON xpath * Update CreateWorkflowLocator.java modify submit workflow button * Update CreateWorkflowLocator.java * Update CreateWorkflowLocator.java modify CLICK_ADD_BUTTON * Update CreateWorkflowLocator.java delete print * Update CreateWorkflowLocator.java 1 * Update CreateWorkflowLocator.java 1 * Setting '-XX:+DisableExplicitGC ' causes netty memory leaks in addition update '- XX: largepagesizeinbytes = 128M' to '- XX: largepagesizeinbytes = 10M' * Update dag.vue * Update dag.vue * Update dag.vue * Update CreateWorkflowLocator.java * Revert "Setting '-XX:+DisableExplicitGC ' causes netty memory leaks" This reverts commit 3a2cba7a * Setting '-XX:+DisableExplicitGC ' causes netty memory leaks in addition update '- XX: largepagesizeinbytes = 128M' to '- XX: largepagesizeinbytes = 10M' * Update dolphinscheduler-daemon.sh * Replacement System.currentTimeMillis () to DateUtils.getCurrentTimeStamp() * update checkstyle * update checkstyle Co-authored-by: dailidong Co-authored-by: xingchun-chen <55787491+xingchun-chen@users.noreply.github.com> --- .../impl/ProcessDefinitionServiceImpl.java | 4 +- .../dolphinscheduler/common/Constants.java | 4 + .../common/utils/DateUtils.java | 10 + .../common/utils/DateUtilsTest.java | 19 +- .../dao/entity/ProcessInstance.java | 200 +++++++++++------- 5 files changed, 155 insertions(+), 82 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index a5e297072c..f4ea1bf065 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -845,7 +845,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements try { createProcessResult = createProcessDefinition(loginUser , currentProjectName, - processDefinitionName + "_import_" + System.currentTimeMillis(), + processDefinitionName + "_import_" + DateUtils.getCurrentTimeStamp(), importProcessParam, processMeta.getProcessDefinitionDescription(), processMeta.getProcessDefinitionLocations(), @@ -1433,7 +1433,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements return createProcessDefinition( loginUser, targetProject.getName(), - processDefinition.getName() + "_copy_" + System.currentTimeMillis(), + processDefinition.getName() + "_copy_" + DateUtils.getCurrentTimeStamp(), processDefinition.getProcessDefinitionJson(), processDefinition.getDescription(), processDefinition.getLocations(), diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index ba8b0c4921..85066cc55a 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -266,6 +266,10 @@ public final class Constants { */ public static final String YYYYMMDDHHMMSS = "yyyyMMddHHmmss"; + /** + * date format of yyyyMMddHHmmssSSS + */ + public static final String YYYYMMDDHHMMSSSSS = "yyyyMMddHHmmssSSS"; /** * http connect time out */ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java index 283b4e7f80..80c0ed411c 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java @@ -444,4 +444,14 @@ public class DateUtils { long usedTime = (System.currentTimeMillis() - baseTime.getTime()) / 1000; return intervalSeconds - usedTime; } + + /** + * get current time stamp : yyyyMMddHHmmssSSS + * + * @return date string + */ + public static String getCurrentTimeStamp() { + return getCurrentTime(Constants.YYYYMMDDHHMMSSSSS); + } + } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java index 6800f6b542..fa16446cd8 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java @@ -14,14 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; -import org.junit.Assert; -import org.junit.Test; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; +import org.junit.Assert; +import org.junit.Test; + public class DateUtilsTest { @Test public void format2Readable() throws ParseException { @@ -38,10 +40,8 @@ public class DateUtilsTest { Assert.assertEquals("01 09:23:08", readableDate); } - @Test - public void testWeek(){ - + public void testWeek() { Date curr = DateUtils.stringToDate("2019-02-01 00:00:00"); Date monday1 = DateUtils.stringToDate("2019-01-28 00:00:00"); Date sunday1 = DateUtils.stringToDate("2019-02-03 00:00:00"); @@ -54,7 +54,7 @@ public class DateUtilsTest { } @Test - public void diffHours(){ + public void diffHours() { Date d1 = DateUtils.stringToDate("2019-01-28 00:00:00"); Date d2 = DateUtils.stringToDate("2019-01-28 20:00:00"); Assert.assertEquals(DateUtils.diffHours(d1, d2), 20); @@ -150,4 +150,11 @@ public class DateUtilsTest { Date curr = DateUtils.getEndOfHour(d1); Assert.assertEquals(DateUtils.dateToString(curr), "2019-01-31 11:59:59"); } + + @Test + public void getCurrentTimeStamp() { + String timeStamp = DateUtils.getCurrentTimeStamp(); + Assert.assertNotNull(timeStamp); + } + } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java index 28f1eba975..3d1a756d25 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java @@ -14,18 +14,27 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.dao.entity; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.enums.FailureStrategy; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.common.enums.Priority; +import org.apache.dolphinscheduler.common.enums.TaskDependType; +import org.apache.dolphinscheduler.common.enums.WarningType; +import org.apache.dolphinscheduler.common.utils.DateUtils; + +import java.util.Date; +import java.util.Objects; + import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import com.baomidou.mybatisplus.core.toolkit.StringUtils; import com.fasterxml.jackson.annotation.JsonFormat; -import org.apache.dolphinscheduler.common.enums.*; - -import java.util.Date; -import java.util.Objects; /** * process instance @@ -36,7 +45,7 @@ public class ProcessInstance { /** * id */ - @TableId(value="id", type=IdType.AUTO) + @TableId(value = "id", type = IdType.AUTO) private int id; /** * process definition id @@ -53,13 +62,13 @@ public class ProcessInstance { /** * start time */ - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") private Date startTime; /** * end time */ - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") private Date endTime; /** @@ -108,7 +117,7 @@ public class ProcessInstance { private FailureStrategy failureStrategy; /** - * warning type + * warning type */ private WarningType warningType; @@ -120,13 +129,13 @@ public class ProcessInstance { /** * schedule time */ - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") private Date scheduleTime; /** * command start time */ - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") private Date commandStartTime; /** @@ -189,6 +198,7 @@ public class ProcessInstance { /** * process duration + * * @return */ @TableField(exist = false) @@ -226,19 +236,24 @@ public class ProcessInstance { @TableField(exist = false) private String receiversCc; - public ProcessInstance(){ + public ProcessInstance() { } /** * set the process name with process define version and timestamp + * * @param processDefinition processDefinition */ - public ProcessInstance(ProcessDefinition processDefinition){ + public ProcessInstance(ProcessDefinition processDefinition) { this.processDefinition = processDefinition; - this.name = processDefinition.getName() + "-" + - processDefinition.getVersion() + "-" + - System.currentTimeMillis(); + this.name = processDefinition.getName() + + "-" + + + processDefinition.getVersion() + + "-" + + + DateUtils.getCurrentTimeStamp(); } public ProcessDefinition getProcessDefinition() { @@ -313,7 +328,6 @@ public class ProcessInstance { this.name = name; } - public String getHost() { return host; } @@ -322,7 +336,6 @@ public class ProcessInstance { this.host = host; } - public CommandType getCommandType() { return commandType; } @@ -347,7 +360,6 @@ public class ProcessInstance { this.taskDependType = taskDependType; } - public int getMaxTryTimes() { return maxTryTimes; } @@ -364,8 +376,7 @@ public class ProcessInstance { this.failureStrategy = failureStrategy; } - - public boolean isProcessInstanceStop(){ + public boolean isProcessInstanceStop() { return this.state.typeIsFinished(); } @@ -441,7 +452,6 @@ public class ProcessInstance { this.executorId = executorId; } - public Flag getIsSubProcess() { return isSubProcess; } @@ -457,6 +467,7 @@ public class ProcessInstance { public void setProcessInstancePriority(Priority processInstancePriority) { this.processInstancePriority = processInstancePriority; } + public String getLocations() { return locations; } @@ -477,6 +488,10 @@ public class ProcessInstance { return historyCmd; } + public void setHistoryCmd(String historyCmd) { + this.historyCmd = historyCmd; + } + public String getExecutorName() { return executorName; } @@ -485,28 +500,26 @@ public class ProcessInstance { this.executorName = executorName; } - public void setHistoryCmd(String historyCmd) { - this.historyCmd = historyCmd; - } - /** * add command to history + * * @param cmd cmd */ - public void addHistoryCmd(CommandType cmd){ - if(StringUtils.isNotEmpty(this.historyCmd)){ + public void addHistoryCmd(CommandType cmd) { + if (StringUtils.isNotEmpty(this.historyCmd)) { this.historyCmd = String.format("%s,%s", this.historyCmd, cmd.toString()); - }else{ + } else { this.historyCmd = cmd.toString(); } } /** * check this process is start complement data + * * @return whether complement data */ - public boolean isComplementData(){ - if(StringUtils.isEmpty(this.historyCmd)){ + public boolean isComplementData() { + if (StringUtils.isEmpty(this.historyCmd)) { return false; } return historyCmd.startsWith(CommandType.COMPLEMENT_DATA.toString()); @@ -515,10 +528,11 @@ public class ProcessInstance { /** * get current command type, * if start with complement data,return complement + * * @return CommandType */ - public CommandType getCmdTypeIfComplement(){ - if(isComplementData()){ + public CommandType getCmdTypeIfComplement() { + if (isComplementData()) { return CommandType.COMPLEMENT_DATA; } return commandType; @@ -556,15 +570,14 @@ public class ProcessInstance { this.timeout = timeout; } + public int getTenantId() { + return this.tenantId; + } public void setTenantId(int tenantId) { this.tenantId = tenantId; } - public int getTenantId() { - return this.tenantId ; - } - public String getReceivers() { return receivers; } @@ -583,44 +596,83 @@ public class ProcessInstance { @Override public String toString() { - return "ProcessInstance{" + - "id=" + id + - ", processDefinitionId=" + processDefinitionId + - ", state=" + state + - ", recovery=" + recovery + - ", startTime=" + startTime + - ", endTime=" + endTime + - ", runTimes=" + runTimes + - ", name='" + name + '\'' + - ", host='" + host + '\'' + - ", processDefinition=" + processDefinition + - ", commandType=" + commandType + - ", commandParam='" + commandParam + '\'' + - ", taskDependType=" + taskDependType + - ", maxTryTimes=" + maxTryTimes + - ", failureStrategy=" + failureStrategy + - ", warningType=" + warningType + - ", warningGroupId=" + warningGroupId + - ", scheduleTime=" + scheduleTime + - ", commandStartTime=" + commandStartTime + - ", globalParams='" + globalParams + '\'' + - ", processInstanceJson='" + processInstanceJson + '\'' + - ", executorId=" + executorId + - ", tenantCode='" + tenantCode + '\'' + - ", queue='" + queue + '\'' + - ", isSubProcess=" + isSubProcess + - ", locations='" + locations + '\'' + - ", connects='" + connects + '\'' + - ", historyCmd='" + historyCmd + '\'' + - ", dependenceScheduleTimes='" + dependenceScheduleTimes + '\'' + - ", duration=" + duration + - ", processInstancePriority=" + processInstancePriority + - ", workerGroup='" + workerGroup + '\'' + - ", timeout=" + timeout + - ", tenantId=" + tenantId + - ", receivers='" + receivers + '\'' + - ", receiversCc='" + receiversCc + '\'' + - '}'; + return "ProcessInstance{" + + "id=" + id + + ", processDefinitionId=" + processDefinitionId + + ", state=" + state + + ", recovery=" + recovery + + ", startTime=" + startTime + + ", endTime=" + endTime + + ", runTimes=" + runTimes + + ", name='" + name + '\'' + + ", host='" + host + '\'' + + ", processDefinition=" + + processDefinition + + ", commandType=" + + commandType + + ", commandParam='" + + commandParam + + '\'' + + ", taskDependType=" + + taskDependType + + ", maxTryTimes=" + + maxTryTimes + + ", failureStrategy=" + + failureStrategy + + ", warningType=" + + warningType + + ", warningGroupId=" + + warningGroupId + + ", scheduleTime=" + + scheduleTime + + ", commandStartTime=" + + commandStartTime + + ", globalParams='" + + globalParams + + '\'' + + ", processInstanceJson='" + + processInstanceJson + + '\'' + + ", executorId=" + + executorId + + ", tenantCode='" + + tenantCode + + '\'' + + ", queue='" + + queue + + '\'' + + ", isSubProcess=" + + isSubProcess + + ", locations='" + + locations + + '\'' + + ", connects='" + + connects + + '\'' + + ", historyCmd='" + + historyCmd + + '\'' + + ", dependenceScheduleTimes='" + + dependenceScheduleTimes + + '\'' + + ", duration=" + + duration + + ", processInstancePriority=" + + processInstancePriority + + ", workerGroup='" + + workerGroup + + '\'' + + ", timeout=" + + timeout + + ", tenantId=" + + tenantId + + ", receivers='" + + receivers + + '\'' + + ", receiversCc='" + + receiversCc + + '\'' + + '}'; } @Override From db663a13a37c6bc4e47286ebbc8efb2f20efa57b Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Tue, 22 Sep 2020 17:39:39 +0800 Subject: [PATCH 24/45] [Improvement][remote]load balance warm up (#3770) * [Improvement][remote]load balance warm up * reformat code * reformat code * code smell * code smell * add test * add test * add test * add test * fix bug * fix bug * add docs * add host test * add host test * add host test * add docs * code reformat * code reformat --- .../remote/utils/Constants.java | 5 ++ .../dolphinscheduler/remote/utils/Host.java | 48 ++++++++++--- .../dispatch/host/assign/HostWeight.java | 35 +++++++--- .../worker/registry/WorkerRegistry.java | 44 +++++++----- .../assign/LowerWeightRoundRobinTest.java | 51 +++++++++++--- .../host/assign/RandomSelectorTest.java | 16 ++--- .../host/assign/RoundRobinSelectorTest.java | 70 ++++++++++++------- .../server/utils/HostTest.java | 43 ++++++++++++ pom.xml | 1 + 9 files changed, 232 insertions(+), 81 deletions(-) create mode 100644 dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/HostTest.java diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java index 370467f6ca..91d4ac245e 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java @@ -53,4 +53,9 @@ public class Constants { */ public static final String OS_NAME = System.getProperty("os.name"); + /** + * warm up time + */ + public static final int WARM_UP_TIME = 10 * 60 * 1000; + } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java index b905a9fea8..c18d02f09a 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.remote.utils; import java.io.Serializable; @@ -44,6 +45,11 @@ public class Host implements Serializable { */ private int weight; + /** + * startTime + */ + private long startTime; + /** * workGroup */ @@ -58,19 +64,21 @@ public class Host implements Serializable { this.address = ip + ":" + port; } - public Host(String ip, int port, int weight) { + public Host(String ip, int port, int weight, long startTime) { this.ip = ip; this.port = port; this.address = ip + ":" + port; - this.weight = weight; + this.weight = getWarmUpWeight(weight, startTime); + this.startTime = startTime; } - public Host(String ip, int port, int weight,String workGroup) { + public Host(String ip, int port, int weight, long startTime, String workGroup) { this.ip = ip; this.port = port; this.address = ip + ":" + port; - this.weight = weight; - this.workGroup=workGroup; + this.weight = getWarmUpWeight(weight, startTime); + this.workGroup = workGroup; + this.startTime = startTime; } public String getAddress() { @@ -98,6 +106,14 @@ public class Host implements Serializable { this.weight = weight; } + public long getStartTime() { + return startTime; + } + + public void setStartTime(long startTime) { + this.startTime = startTime; + } + public int getPort() { return port; } @@ -133,8 +149,8 @@ public class Host implements Serializable { if (parts.length == 2) { host = new Host(parts[0], Integer.parseInt(parts[1])); } - if (parts.length == 3) { - host = new Host(parts[0], Integer.parseInt(parts[1]), Integer.parseInt(parts[2])); + if (parts.length == 4) { + host = new Host(parts[0], Integer.parseInt(parts[1]), Integer.parseInt(parts[2]), Long.parseLong(parts[3])); } return host; } @@ -169,8 +185,20 @@ public class Host implements Serializable { @Override public String toString() { - return "Host{" + - "address='" + address + '\'' + - '}'; + return "Host{" + + "address='" + address + '\'' + + '}'; + } + + /** + * warm up + */ + private int getWarmUpWeight(int weight, long startTime) { + long uptime = System.currentTimeMillis() - startTime; + //If the warm-up is not over, reduce the weight + if (uptime > 0 && uptime < Constants.WARM_UP_TIME) { + return (int) (weight * ((float) uptime / Constants.WARM_UP_TIME)); + } + return weight; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java index ebceea7b13..298a62a6d9 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.master.dispatch.host.assign; +import org.apache.dolphinscheduler.remote.utils.Constants; import org.apache.dolphinscheduler.remote.utils.Host; /** @@ -37,9 +38,9 @@ public class HostWeight { private int currentWeight; public HostWeight(Host host, double cpu, double memory, double loadAverage) { - this.weight = calculateWeight(cpu, memory, loadAverage); - this.host = host ; - this.currentWeight = weight ; + this.weight = getWeight(cpu, memory, loadAverage, host); + this.host = host; + this.currentWeight = weight; } public int getCurrentWeight() { @@ -60,14 +61,28 @@ public class HostWeight { @Override public String toString() { - return "HostWeight{" + - "host=" + host + - ", weight=" + weight + - ", currentWeight=" + currentWeight + - '}'; + return "HostWeight{" + + "host=" + host + + ", weight=" + weight + + ", currentWeight=" + currentWeight + + '}'; } - private int calculateWeight(double cpu, double memory, double loadAverage){ - return (int)(cpu * CPU_FACTOR + memory * MEMORY_FACTOR + loadAverage * LOAD_AVERAGE_FACTOR); + private int getWeight(double cpu, double memory, double loadAverage, Host host) { + int calculateWeight = (int) (cpu * CPU_FACTOR + memory * MEMORY_FACTOR + loadAverage * LOAD_AVERAGE_FACTOR); + return getWarmUpWeight(host, calculateWeight); + + } + + /** + * If the warm-up is not over, add the weight + */ + private int getWarmUpWeight(Host host, int weight) { + long startTime = host.getStartTime(); + long uptime = System.currentTimeMillis() - startTime; + if (uptime > 0 && uptime < Constants.WARM_UP_TIME) { + return (int) ((weight * Constants.WARM_UP_TIME) / uptime); + } + return weight; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java index 36998fad63..904ea3a807 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java @@ -14,19 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.server.worker.registry; -import java.util.Date; -import java.util.Set; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; +import static org.apache.dolphinscheduler.common.Constants.COLON; +import static org.apache.dolphinscheduler.common.Constants.DEFAULT_WORKER_GROUP; +import static org.apache.dolphinscheduler.common.Constants.SLASH; -import javax.annotation.PostConstruct; - -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.state.ConnectionState; -import org.apache.curator.framework.state.ConnectionStateListener; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; @@ -34,6 +28,19 @@ import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory; import org.apache.dolphinscheduler.server.registry.HeartBeatTask; import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; + +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.state.ConnectionState; +import org.apache.curator.framework.state.ConnectionStateListener; + +import java.util.Date; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import javax.annotation.PostConstruct; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -41,8 +48,6 @@ import org.springframework.stereotype.Service; import com.google.common.collect.Sets; -import static org.apache.dolphinscheduler.common.Constants.*; - /** * worker registry @@ -111,10 +116,10 @@ public class WorkerRegistry { } HeartBeatTask heartBeatTask = new HeartBeatTask(this.startTime, - this.workerConfig.getWorkerReservedMemory(), - this.workerConfig.getWorkerMaxCpuloadAvg(), - workerZkPaths, - this.zookeeperRegistryCenter); + this.workerConfig.getWorkerReservedMemory(), + this.workerConfig.getWorkerMaxCpuloadAvg(), + workerZkPaths, + this.zookeeperRegistryCenter); this.heartBeatExecutor.scheduleAtFixedRate(heartBeatTask, workerHeartbeatInterval, workerHeartbeatInterval, TimeUnit.SECONDS); logger.info("worker node : {} heartbeat interval {} s", address, workerHeartbeatInterval); @@ -142,6 +147,7 @@ public class WorkerRegistry { String address = getLocalAddress(); String workerZkPathPrefix = this.zookeeperRegistryCenter.getWorkerPath(); String weight = getWorkerWeight(); + String workerStartTime = COLON + System.currentTimeMillis(); for (String workGroup : this.workerGroups) { StringBuilder workerZkPathBuilder = new StringBuilder(100); @@ -153,6 +159,7 @@ public class WorkerRegistry { workerZkPathBuilder.append(workGroup.trim().toLowerCase()).append(SLASH); workerZkPathBuilder.append(address); workerZkPathBuilder.append(weight); + workerZkPathBuilder.append(workerStartTime); workerZkPaths.add(workerZkPathBuilder.toString()); } return workerZkPaths; @@ -162,13 +169,14 @@ public class WorkerRegistry { * get local address */ private String getLocalAddress() { - return NetUtils.getHost() + ":" + workerConfig.getListenPort(); + return NetUtils.getHost() + COLON + workerConfig.getListenPort(); } /** * get Worker Weight */ private String getWorkerWeight() { - return ":" + workerConfig.getWeight(); + return COLON + workerConfig.getWeight(); } + } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java index fadaa84a69..fd5dda0873 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java @@ -14,9 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.server.master.dispatch.host.assign; import org.apache.dolphinscheduler.remote.utils.Host; + +import org.junit.Assert; import org.junit.Test; import java.util.ArrayList; @@ -27,15 +30,47 @@ public class LowerWeightRoundRobinTest { @Test - public void testSelect(){ + public void testSelect() { Collection sources = new ArrayList<>(); - sources.add(new HostWeight(Host.of("192.158.2.1:11"), 0.06, 0.44, 3.84)); - sources.add(new HostWeight(Host.of("192.158.2.1:22"), 0.06, 0.56, 3.24)); - sources.add(new HostWeight(Host.of("192.158.2.1:33"), 0.06, 0.80, 3.15)); - System.out.println(sources); + sources.add(new HostWeight(Host.of("192.158.2.1:11:100:" + (System.currentTimeMillis() - 60 * 8 * 1000)), 0.06, 0.44, 3.84)); + sources.add(new HostWeight(Host.of("192.158.2.2:22:100:" + (System.currentTimeMillis() - 60 * 5 * 1000)), 0.06, 0.56, 3.24)); + sources.add(new HostWeight(Host.of("192.158.2.3:33:100:" + (System.currentTimeMillis() - 60 * 2 * 1000)), 0.06, 0.80, 3.15)); + + LowerWeightRoundRobin roundRobin = new LowerWeightRoundRobin(); + HostWeight result; + result = roundRobin.select(sources); + Assert.assertEquals("192.158.2.1", result.getHost().getIp()); + result = roundRobin.select(sources); + Assert.assertEquals("192.158.2.2", result.getHost().getIp()); + result = roundRobin.select(sources); + Assert.assertEquals("192.158.2.1", result.getHost().getIp()); + result = roundRobin.select(sources); + Assert.assertEquals("192.158.2.2", result.getHost().getIp()); + result = roundRobin.select(sources); + Assert.assertEquals("192.158.2.1", result.getHost().getIp()); + result = roundRobin.select(sources); + Assert.assertEquals("192.158.2.2", result.getHost().getIp()); + } + + @Test + public void testWarmUpSelect() { + Collection sources = new ArrayList<>(); + sources.add(new HostWeight(Host.of("192.158.2.1:11:100:" + (System.currentTimeMillis() - 60 * 8 * 1000)), 0.06, 0.44, 3.84)); + sources.add(new HostWeight(Host.of("192.158.2.2:22:100:" + (System.currentTimeMillis() - 60 * 5 * 1000)), 0.06, 0.44, 3.84)); + sources.add(new HostWeight(Host.of("192.158.2.3:33:100:" + (System.currentTimeMillis() - 60 * 3 * 1000)), 0.06, 0.44, 3.84)); + sources.add(new HostWeight(Host.of("192.158.2.4:33:100:" + (System.currentTimeMillis() - 60 * 11 * 1000)), 0.06, 0.44, 3.84)); + LowerWeightRoundRobin roundRobin = new LowerWeightRoundRobin(); - for(int i = 0; i < 100; i ++){ - System.out.println(roundRobin.select(sources)); - } + HostWeight result; + result = roundRobin.select(sources); + Assert.assertEquals("192.158.2.4", result.getHost().getIp()); + result = roundRobin.select(sources); + Assert.assertEquals("192.158.2.1", result.getHost().getIp()); + result = roundRobin.select(sources); + Assert.assertEquals("192.158.2.2", result.getHost().getIp()); + result = roundRobin.select(sources); + Assert.assertEquals("192.158.2.4", result.getHost().getIp()); + result = roundRobin.select(sources); + Assert.assertEquals("192.158.2.1", result.getHost().getIp()); } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java index f25a227947..14aa7b8f1f 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.server.master.dispatch.host.assign; -import org.apache.commons.lang.ObjectUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.remote.utils.Host; + import org.junit.Assert; import org.junit.Test; @@ -31,22 +31,22 @@ import java.util.Collections; public class RandomSelectorTest { @Test(expected = IllegalArgumentException.class) - public void testSelectWithIllegalArgumentException(){ + public void testSelectWithIllegalArgumentException() { RandomSelector selector = new RandomSelector(); - selector.select(Collections.EMPTY_LIST); + selector.select(null); } @Test - public void testSelect1(){ + public void testSelect1() { RandomSelector selector = new RandomSelector(); - Host result = selector.select(Arrays.asList(new Host("192.168.1.1",80,100),new Host("192.168.1.2",80,20))); + Host result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 100, System.currentTimeMillis()), new Host("192.168.1.2", 80, 20, System.currentTimeMillis()))); Assert.assertNotNull(result); } @Test - public void testSelect(){ + public void testSelect() { RandomSelector selector = new RandomSelector(); - Host result = selector.select(Arrays.asList(new Host("192.168.1.1",80,100),new Host("192.168.1.1",80,20))); + Host result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 100, System.currentTimeMillis()), new Host("192.168.1.1", 80, 20, System.currentTimeMillis()))); Assert.assertNotNull(result); } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java index ed62caaa2c..9e41cd68bf 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java @@ -14,16 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.server.master.dispatch.host.assign; -import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.remote.utils.Host; + import org.junit.Assert; import org.junit.Test; import java.util.Arrays; import java.util.Collections; -import java.util.List; /** * round robin selector @@ -33,43 +33,59 @@ public class RoundRobinSelectorTest { @Test(expected = IllegalArgumentException.class) public void testSelectWithIllegalArgumentException() { RoundRobinSelector selector = new RoundRobinSelector(); - selector.select(Collections.EMPTY_LIST); + selector.select(null); } @Test public void testSelect1() { RoundRobinSelector selector = new RoundRobinSelector(); - Host result = null; - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"))); + Host result; + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); Assert.assertEquals("192.168.1.1", result.getIp()); - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"))); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); Assert.assertEquals("192.168.1.2", result.getIp()); - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"))); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); Assert.assertEquals("192.168.1.1", result.getIp()); // add new host - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"))); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); + Assert.assertEquals("192.168.1.1", result.getIp()); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); + Assert.assertEquals("192.168.1.2", result.getIp()); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"), + new Host("192.168.1.3", 80, 10, System.currentTimeMillis(), "kris"))); Assert.assertEquals("192.168.1.1", result.getIp()); - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"))); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"), + new Host("192.168.1.3", 80, 10, System.currentTimeMillis(), "kris"))); + Assert.assertEquals("192.168.1.3", result.getIp()); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"), + new Host("192.168.1.3", 80, 10, System.currentTimeMillis(), "kris"))); + Assert.assertEquals("192.168.1.1", result.getIp()); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"), + new Host("192.168.1.3", 80, 10, System.currentTimeMillis(), "kris"))); Assert.assertEquals("192.168.1.2", result.getIp()); - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"), new Host("192.168.1.3", 80, 10, "kris"))); - Assert.assertEquals("192.168.1.1",result.getIp()); - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"), new Host("192.168.1.3", 80, 10, "kris"))); - Assert.assertEquals("192.168.1.3",result.getIp()); - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"), new Host("192.168.1.3", 80, 10, "kris"))); - Assert.assertEquals("192.168.1.1",result.getIp()); - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"), new Host("192.168.1.3", 80, 10, "kris"))); - Assert.assertEquals("192.168.1.2",result.getIp()); - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"), new Host("192.168.1.3", 80, 10, "kris"))); - Assert.assertEquals("192.168.1.1",result.getIp()); - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"), new Host("192.168.1.3", 80, 10, "kris"))); - Assert.assertEquals("192.168.1.3",result.getIp()); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"), + new Host("192.168.1.3", 80, 10, System.currentTimeMillis(), "kris"))); + Assert.assertEquals("192.168.1.1", result.getIp()); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"), + new Host("192.168.1.3", 80, 10, System.currentTimeMillis(), "kris"))); + Assert.assertEquals("192.168.1.3", result.getIp()); // remove host3 - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"))); - Assert.assertEquals("192.168.1.1",result.getIp()); - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"))); - Assert.assertEquals("192.168.1.2",result.getIp()); - result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"))); - Assert.assertEquals("192.168.1.1",result.getIp()); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); + Assert.assertEquals("192.168.1.1", result.getIp()); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); + Assert.assertEquals("192.168.1.2", result.getIp()); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis(), "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis(), "kris"))); + Assert.assertEquals("192.168.1.1", result.getIp()); + + } + + @Test + public void testWarmUpRoundRobinSelector() { + RoundRobinSelector selector = new RoundRobinSelector(); + Host result; + result = selector.select( + Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis() - 60 * 1000 * 2, "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis() - 60 * 1000 * 10, "kris"))); + Assert.assertEquals("192.168.1.2", result.getIp()); } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/HostTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/HostTest.java new file mode 100644 index 0000000000..6273569485 --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/HostTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.utils; + +import org.apache.dolphinscheduler.remote.utils.Host; + +import org.junit.Assert; +import org.junit.Test; + +/** + * host test + */ +public class HostTest { + + @Test + public void testHostWarmUp() { + Host host = Host.of(("192.158.2.2:22:100:" + (System.currentTimeMillis() - 60 * 5 * 1000))); + Assert.assertEquals(50, host.getWeight()); + host = Host.of(("192.158.2.2:22:100:" + (System.currentTimeMillis() - 60 * 10 * 1000))); + Assert.assertEquals(100, host.getWeight()); + } + + @Test + public void testHost() { + Host host = Host.of("192.158.2.2:22"); + Assert.assertEquals(22, host.getPort()); + } +} diff --git a/pom.xml b/pom.xml index 207518c936..c8cb5a9dea 100644 --- a/pom.xml +++ b/pom.xml @@ -832,6 +832,7 @@ **/server/register/ZookeeperNodeManagerTest.java **/server/utils/DataxUtilsTest.java **/server/utils/ExecutionContextTestUtils.java + **/server/utils/HostTest.java **/server/utils/LogUtilsTest.java **/server/utils/ParamUtilsTest.java From 3b581455fc572c01c93a9d5ecfb955a966a20263 Mon Sep 17 00:00:00 2001 From: guyinyou <36399867+guyinyou@users.noreply.github.com> Date: Fri, 25 Sep 2020 13:20:42 +0800 Subject: [PATCH 25/45] [Feature]Add Python task "task variable / result transfer" implementation (#3659) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 增加Python Task的“任务变量/结果传递”实现 Signed-off-by: 古崟佑 * add two files license Signed-off-by: 古崟佑 * fix 'server/worker/task/AbstractCommandExecutor.java' code style Signed-off-by: 1941815847Cy4 <1941815847cy4@kuaishou.com> * update DB Signed-off-by: 古崟佑 * update DB -- 2 Signed-off-by: 古崟佑 * fix codeStyle Signed-off-by: 古崟佑 * fix codestyle Signed-off-by: 古崟佑 * fix codeStyle Signed-off-by: 古崟佑 * fix codeStyle Signed-off-by: 古崟佑 * fix codeStyle Signed-off-by: 古崟佑 * add VarPoolUtils Test Signed-off-by: 古崟佑 * fix VarPoolUtilsTest codeStyle Signed-off-by: 古崟佑 * fix VarPoolUtilsTest codeStyle Signed-off-by: 古崟佑 * fix VarPoolUtilsTest codeStyle Signed-off-by: 古崟佑 * fix VarPoolUtilsTest codeStyle Signed-off-by: 古崟佑 * fix VarPoolUtilsTest codeStyle Signed-off-by: 古崟佑 * add test config for VarPoolUtilsTest Signed-off-by: 古崟佑 * fix unit test Signed-off-by: 古崟佑 * fix codeStyle Signed-off-by: 古崟佑 * fix VarPoolUtilsTest.java Signed-off-by: 古崟佑 * fix Signed-off-by: 古崟佑 * change the test class path Signed-off-by: 古崟佑 * fix Signed-off-by: 古崟佑 * fix "print the error message" Signed-off-by: 古崟佑 * fix bug Signed-off-by: 古崟佑 * fix Signed-off-by: 古崟佑 Co-authored-by: 1941815847Cy4 <1941815847cy4@kuaishou.com> --- .../common/task/TaskParams.java | 78 ++++++++ .../common/utils/VarPoolUtils.java | 124 ++++++++++++ .../common/utils/VarPoolUtilsTest.java | 73 +++++++ .../dao/entity/ProcessInstance.java | 13 ++ .../dao/entity/TaskInstance.java | 12 ++ .../command/TaskExecuteResponseCommand.java | 12 ++ .../processor/TaskResponseProcessor.java | 3 +- .../processor/queue/TaskResponseEvent.java | 18 +- .../processor/queue/TaskResponseService.java | 3 +- .../master/runner/MasterExecThread.java | 15 +- .../worker/runner/TaskExecuteThread.java | 1 + .../worker/task/AbstractCommandExecutor.java | 15 +- .../server/worker/task/AbstractTask.java | 13 ++ .../server/worker/task/python/PythonTask.java | 189 +++++++++--------- .../service/process/ProcessService.java | 5 +- pom.xml | 1 + sql/dolphinscheduler-postgre.sql | 2 + sql/dolphinscheduler_mysql.sql | 2 + .../mysql/dolphinscheduler_ddl.sql | 40 ++++ .../postgresql/dolphinscheduler_ddl.sql | 36 ++++ 20 files changed, 555 insertions(+), 100 deletions(-) create mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/TaskParams.java create mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/VarPoolUtils.java create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/VarPoolUtilsTest.java diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/TaskParams.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/TaskParams.java new file mode 100644 index 0000000000..abea2d95b0 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/TaskParams.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.task; + +import java.util.Map; + +public class TaskParams { + + private String rawScript; + private Map[] localParams; + + public void setRawScript(String rawScript) { + this.rawScript = rawScript; + } + + public void setLocalParams(Map[] localParams) { + this.localParams = localParams; + } + + public String getRawScript() { + return rawScript; + } + + public void setLocalParamValue(String prop, Object value) { + if (localParams == null || value == null) { + return; + } + for (int i = 0; i < localParams.length; i++) { + if (localParams[i].get("prop").equals(prop)) { + localParams[i].put("value", (String)value); + } + } + } + + public void setLocalParamValue(Map propToValue) { + if (localParams == null || propToValue == null) { + return; + } + for (int i = 0; i < localParams.length; i++) { + String prop = localParams[i].get("prop"); + if (propToValue.containsKey(prop)) { + localParams[i].put("value",(String)propToValue.get(prop)); + } + } + } + + public String getLocalParamValue(String prop) { + if (localParams == null) { + return null; + } + for (int i = 0; i < localParams.length; i++) { + String tmpProp = localParams[i].get("prop"); + if (tmpProp.equals(prop)) { + return localParams[i].get("value"); + } + } + return null; + } + + public Map[] getLocalParams() { + return localParams; + } +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/VarPoolUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/VarPoolUtils.java new file mode 100644 index 0000000000..837e96f55f --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/VarPoolUtils.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.utils; + +import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.task.TaskParams; + +import java.text.ParseException; +import java.util.Map; + +public class VarPoolUtils { + /** + * getTaskNodeLocalParam + * @param taskNode taskNode + * @param prop prop + * @return localParamForProp + */ + public static Object getTaskNodeLocalParam(TaskNode taskNode, String prop) { + String taskParamsJson = taskNode.getParams(); + TaskParams taskParams = JSONUtils.parseObject(taskParamsJson, TaskParams.class); + if (taskParams == null) { + return null; + } + return taskParams.getLocalParamValue(prop); + } + + /** + * setTaskNodeLocalParams + * @param taskNode taskNode + * @param prop LocalParamName + * @param value LocalParamValue + */ + public static void setTaskNodeLocalParams(TaskNode taskNode, String prop, Object value) { + String taskParamsJson = taskNode.getParams(); + TaskParams taskParams = JSONUtils.parseObject(taskParamsJson, TaskParams.class); + if (taskParams == null) { + return; + } + taskParams.setLocalParamValue(prop, value); + taskNode.setParams(JSONUtils.toJsonString(taskParams)); + } + + /** + * setTaskNodeLocalParams + * @param taskNode taskNode + * @param propToValue propToValue + */ + public static void setTaskNodeLocalParams(TaskNode taskNode, Map propToValue) { + String taskParamsJson = taskNode.getParams(); + TaskParams taskParams = JSONUtils.parseObject(taskParamsJson, TaskParams.class); + if (taskParams == null) { + return; + } + taskParams.setLocalParamValue(propToValue); + taskNode.setParams(JSONUtils.toJsonString(taskParams)); + } + + /** + * convertVarPoolToMap + * @param propToValue propToValue + * @param varPool varPool + * @throws ParseException ParseException + */ + public static void convertVarPoolToMap(Map propToValue, String varPool) throws ParseException { + if (varPool == null || propToValue == null) { + return; + } + String[] splits = varPool.split("\\$VarPool\\$"); + for (String kv : splits) { + String[] kvs = kv.split(","); + if (kvs.length == 2) { + propToValue.put(kvs[0], kvs[1]); + } else { + throw new ParseException(kv, 2); + } + } + } + + /** + * convertPythonScriptPlaceholders + * @param rawScript rawScript + * @return String + * @throws StringIndexOutOfBoundsException StringIndexOutOfBoundsException + */ + public static String convertPythonScriptPlaceholders(String rawScript) throws StringIndexOutOfBoundsException { + int len = "${setShareVar(${".length(); + int scriptStart = 0; + while ((scriptStart = rawScript.indexOf("${setShareVar(${", scriptStart)) != -1) { + int start = -1; + int end = rawScript.indexOf('}', scriptStart + len); + String prop = rawScript.substring(scriptStart + len, end); + + start = rawScript.indexOf(',', end); + end = rawScript.indexOf(')', start); + + String value = rawScript.substring(start + 1, end); + + start = rawScript.indexOf('}', start) + 1; + end = rawScript.length(); + + String replaceScript = String.format("print(\"${{setValue({},{})}}\".format(\"%s\",%s))", prop, value); + + rawScript = rawScript.substring(0, scriptStart) + replaceScript + rawScript.substring(start, end); + + scriptStart += replaceScript.length(); + } + return rawScript; + } +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/VarPoolUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/VarPoolUtilsTest.java new file mode 100644 index 0000000000..e47203c225 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/VarPoolUtilsTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.utils; + +import org.apache.dolphinscheduler.common.model.TaskNode; + +import java.util.concurrent.ConcurrentHashMap; + +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class VarPoolUtilsTest { + + private static final Logger logger = LoggerFactory.getLogger(VarPoolUtilsTest.class); + + @Test + public void testSetTaskNodeLocalParams() { + String taskJson = "{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\"," + + "\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"tasks-75298\",\"maxRetryTimes\":0,\"name\":\"a1\"," + + "\"params\":\"{\\\"rawScript\\\":\\\"print(\\\\\\\"this is python task \\\\\\\",${p0})\\\"," + + "\\\"localParams\\\":[{\\\"prop\\\":\\\"p1\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"1\\\"}]," + + "\\\"resourceList\\\":[]}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\"," + + "\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\"," + + "\"type\":\"PYTHON\",\"workerGroup\":\"default\"}"; + TaskNode taskNode = JSONUtils.parseObject(taskJson, TaskNode.class); + + VarPoolUtils.setTaskNodeLocalParams(taskNode, "p1", "test1"); + Assert.assertEquals(VarPoolUtils.getTaskNodeLocalParam(taskNode, "p1"), "test1"); + + ConcurrentHashMap propToValue = new ConcurrentHashMap(); + propToValue.put("p1", "test2"); + + VarPoolUtils.setTaskNodeLocalParams(taskNode, propToValue); + Assert.assertEquals(VarPoolUtils.getTaskNodeLocalParam(taskNode, "p1"), "test2"); + } + + @Test + public void testConvertVarPoolToMap() throws Exception { + String varPool = "p1,66$VarPool$p2,69$VarPool$"; + ConcurrentHashMap propToValue = new ConcurrentHashMap(); + VarPoolUtils.convertVarPoolToMap(propToValue, varPool); + Assert.assertEquals((String)propToValue.get("p1"), "66"); + Assert.assertEquals((String)propToValue.get("p2"), "69"); + logger.info(propToValue.toString()); + } + + @Test + public void testConvertPythonScriptPlaceholders() throws Exception { + String rawScript = "print(${p1});\n${setShareVar(${p1},3)};\n${setShareVar(${p2},4)};"; + rawScript = VarPoolUtils.convertPythonScriptPlaceholders(rawScript); + Assert.assertEquals(rawScript, "print(${p1});\n" + + "print(\"${{setValue({},{})}}\".format(\"p1\",3));\n" + + "print(\"${{setValue({},{})}}\".format(\"p2\",4));"); + logger.info(rawScript); + } +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java index 3d1a756d25..e3a3f11386 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java @@ -224,6 +224,11 @@ public class ProcessInstance { */ private int tenantId; + /** + * varPool string + */ + private String varPool; + /** * receivers for api */ @@ -256,6 +261,14 @@ public class ProcessInstance { DateUtils.getCurrentTimeStamp(); } + public String getVarPool() { + return varPool; + } + + public void setVarPool(String varPool) { + this.varPool = varPool; + } + public ProcessDefinition getProcessDefinition() { return processDefinition; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java index 9688200b2c..b13ca87e38 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java @@ -211,6 +211,11 @@ public class TaskInstance implements Serializable { */ private int executorId; + /** + * varPool string + */ + private String varPool; + /** * executor name */ @@ -232,7 +237,14 @@ public class TaskInstance implements Serializable { this.executePath = executePath; } + public String getVarPool() { + return varPool; + } + public void setVarPool(String varPool) { + this.varPool = varPool; + } + public ProcessInstance getProcessInstance() { return processInstance; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java index e559334f48..7f6ee668a8 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java @@ -63,7 +63,19 @@ public class TaskExecuteResponseCommand implements Serializable { */ private String appIds; + /** + * varPool string + */ + private String varPool; + public void setVarPool(String varPool) { + this.varPool = varPool; + } + + public String getVarPool() { + return varPool; + } + public int getTaskInstanceId() { return taskInstanceId; } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java index b04b930fd4..2633ccd634 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java @@ -90,7 +90,8 @@ public class TaskResponseProcessor implements NettyRequestProcessor { responseCommand.getEndTime(), responseCommand.getProcessId(), responseCommand.getAppIds(), - responseCommand.getTaskInstanceId()); + responseCommand.getTaskInstanceId(), + responseCommand.getVarPool()); taskResponseService.addResponse(taskResponseEvent); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseEvent.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseEvent.java index 051cc388bf..ba07be50f3 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseEvent.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseEvent.java @@ -79,7 +79,12 @@ public class TaskResponseEvent { */ private Event event; - public static TaskResponseEvent newAck(ExecutionStatus state, Date startTime, String workerAddress, String executePath, String logPath, int taskInstanceId){ + /** + * varPool + */ + private String varPool; + + public static TaskResponseEvent newAck(ExecutionStatus state, Date startTime, String workerAddress, String executePath, String logPath, int taskInstanceId) { TaskResponseEvent event = new TaskResponseEvent(); event.setState(state); event.setStartTime(startTime); @@ -91,7 +96,7 @@ public class TaskResponseEvent { return event; } - public static TaskResponseEvent newResult(ExecutionStatus state, Date endTime, int processId, String appIds, int taskInstanceId){ + public static TaskResponseEvent newResult(ExecutionStatus state, Date endTime, int processId, String appIds, int taskInstanceId, String varPool) { TaskResponseEvent event = new TaskResponseEvent(); event.setState(state); event.setEndTime(endTime); @@ -99,9 +104,18 @@ public class TaskResponseEvent { event.setAppIds(appIds); event.setTaskInstanceId(taskInstanceId); event.setEvent(Event.RESULT); + event.setVarPool(varPool); return event; } + public String getVarPool() { + return varPool; + } + + public void setVarPool(String varPool) { + this.varPool = varPool; + } + public int getTaskInstanceId() { return taskInstanceId; } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java index ba07313a9a..6434db70e5 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java @@ -139,7 +139,8 @@ public class TaskResponseService { taskResponseEvent.getEndTime(), taskResponseEvent.getProcessId(), taskResponseEvent.getAppIds(), - taskResponseEvent.getTaskInstanceId()); + taskResponseEvent.getTaskInstanceId(), + taskResponseEvent.getVarPool()); break; default: throw new IllegalArgumentException("invalid event type : " + event); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 788b30638e..3c28e16651 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -45,6 +45,7 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.utils.VarPoolUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.Schedule; import org.apache.dolphinscheduler.dao.entity.TaskInstance; @@ -59,6 +60,7 @@ import org.apache.commons.io.FileUtils; import java.io.File; import java.io.IOException; +import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -651,14 +653,23 @@ public class MasterExecThread implements Runnable { * submit post node * @param parentNodeName parent node name */ + private Map propToValue = new ConcurrentHashMap(); private void submitPostNode(String parentNodeName){ List submitTaskNodeList = parsePostNodeList(parentNodeName); List taskInstances = new ArrayList<>(); for(String taskNode : submitTaskNodeList){ + try { + VarPoolUtils.convertVarPoolToMap(propToValue, processInstance.getVarPool()); + } catch (ParseException e) { + logger.error("parse {} exception", processInstance.getVarPool(), e); + throw new RuntimeException(); + } + TaskNode taskNodeObject = dag.getNode(taskNode); + VarPoolUtils.setTaskNodeLocalParams(taskNodeObject, propToValue); taskInstances.add(createTaskInstance(processInstance, taskNode, - dag.getNode(taskNode))); + taskNodeObject)); } // if previous node success , post node submit @@ -999,6 +1010,8 @@ public class MasterExecThread implements Runnable { task.getName(), task.getId(), task.getState()); // node success , post node submit if(task.getState() == ExecutionStatus.SUCCESS){ + processInstance.setVarPool(task.getVarPool()); + processService.updateProcessInstance(processInstance); completeTaskList.put(task.getName(), task); submitPostNode(task.getName()); continue; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java index 3ba49451c7..58f743303c 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java @@ -155,6 +155,7 @@ public class TaskExecuteThread implements Runnable { responseCommand.setEndTime(new Date()); responseCommand.setProcessId(task.getProcessId()); responseCommand.setAppIds(task.getAppIds()); + responseCommand.setVarPool(task.getVarPool()); logger.info("task instance id : {},task final status : {}", taskExecutionContext.getTaskInstanceId(), task.getExitStatus()); } catch (Exception e) { logger.error("task scheduler failure", e); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index 3dedeced06..dddd1a64b7 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -66,6 +66,7 @@ public abstract class AbstractCommandExecutor { */ protected static final Pattern APPLICATION_REGEX = Pattern.compile(Constants.APPLICATION_REGEX); + protected StringBuilder varPool = new StringBuilder(); /** * process */ @@ -234,7 +235,10 @@ public abstract class AbstractCommandExecutor { return result; } - + public String getVarPool() { + return varPool.toString(); + } + /** * cancel application * @throws Exception exception @@ -347,8 +351,13 @@ public abstract class AbstractCommandExecutor { long lastFlushTime = System.currentTimeMillis(); while ((line = inReader.readLine()) != null) { - logBuffer.add(line); - lastFlushTime = flush(lastFlushTime); + if (line.startsWith("${setValue(")) { + varPool.append(line.substring("${setValue(".length(), line.length() - 2)); + varPool.append("$VarPool$"); + } else { + logBuffer.add(line); + lastFlushTime = flush(lastFlushTime); + } } } catch (Exception e) { logger.error(e.getMessage(),e); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java index ae03932a52..1a66349817 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractTask.java @@ -47,6 +47,11 @@ import java.util.Map; */ public abstract class AbstractTask { + /** + * varPool string + */ + protected String varPool; + /** * taskExecutionContext **/ @@ -121,6 +126,14 @@ public abstract class AbstractTask { logger.info(" -> {}", String.join("\n\t", logs)); } + public void setVarPool(String varPool) { + this.varPool = varPool; + } + + public String getVarPool() { + return varPool; + } + /** * get exit status code * @return exit status code diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java index 367da80a0c..6e561c1cab 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java @@ -38,103 +38,110 @@ import java.util.Map; */ public class PythonTask extends AbstractTask { - /** - * python parameters - */ - private PythonParameters pythonParameters; - - /** - * task dir - */ - private String taskDir; - - /** - * python command executor - */ - private PythonCommandExecutor pythonCommandExecutor; - - /** - * taskExecutionContext - */ - private TaskExecutionContext taskExecutionContext; - - /** - * constructor - * @param taskExecutionContext taskExecutionContext - * @param logger logger - */ - public PythonTask(TaskExecutionContext taskExecutionContext, Logger logger) { - super(taskExecutionContext, logger); - this.taskExecutionContext = taskExecutionContext; - - this.pythonCommandExecutor = new PythonCommandExecutor(this::logHandle, - taskExecutionContext, - logger); - } - - @Override - public void init() { - logger.info("python task params {}", taskExecutionContext.getTaskParams()); - - pythonParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), PythonParameters.class); - - if (!pythonParameters.checkParameters()) { - throw new RuntimeException("python task params is not valid"); + /** + * python parameters + */ + private PythonParameters pythonParameters; + + /** + * task dir + */ + private String taskDir; + + /** + * python command executor + */ + private PythonCommandExecutor pythonCommandExecutor; + + /** + * taskExecutionContext + */ + private TaskExecutionContext taskExecutionContext; + + /** + * constructor + * @param taskExecutionContext taskExecutionContext + * @param logger logger + */ + public PythonTask(TaskExecutionContext taskExecutionContext, Logger logger) { + super(taskExecutionContext, logger); + this.taskExecutionContext = taskExecutionContext; + + this.pythonCommandExecutor = new PythonCommandExecutor(this::logHandle, + taskExecutionContext, + logger); } - } - @Override - public void handle() throws Exception { - try { - // construct process - CommandExecuteResult commandExecuteResult = pythonCommandExecutor.run(buildCommand()); + @Override + public void init() { + logger.info("python task params {}", taskExecutionContext.getTaskParams()); - setExitStatusCode(commandExecuteResult.getExitStatusCode()); - setAppIds(commandExecuteResult.getAppIds()); - setProcessId(commandExecuteResult.getProcessId()); - } - catch (Exception e) { - logger.error("python task failure", e); - setExitStatusCode(Constants.EXIT_CODE_FAILURE); - throw e; - } - } - - @Override - public void cancelApplication(boolean cancelApplication) throws Exception { - // cancel process - pythonCommandExecutor.cancelApplication(); - } - - /** - * build command - * @return raw python script - * @throws Exception exception - */ - private String buildCommand() throws Exception { - String rawPythonScript = pythonParameters.getRawScript().replaceAll("\\r\\n", "\n"); - - // replace placeholder - Map paramsMap = ParamUtils.convert(ParamUtils.getUserDefParamsMap(taskExecutionContext.getDefinedParams()), - taskExecutionContext.getDefinedParams(), - pythonParameters.getLocalParametersMap(), - CommandType.of(taskExecutionContext.getCmdTypeIfComplement()), - taskExecutionContext.getScheduleTime()); - if (paramsMap != null){ - rawPythonScript = ParameterUtils.convertParameterPlaceholders(rawPythonScript, ParamUtils.convert(paramsMap)); - } + pythonParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), PythonParameters.class); - logger.info("raw python script : {}", pythonParameters.getRawScript()); - logger.info("task dir : {}", taskDir); - - return rawPythonScript; - } + if (!pythonParameters.checkParameters()) { + throw new RuntimeException("python task params is not valid"); + } + } - @Override - public AbstractParameters getParameters() { - return pythonParameters; - } + @Override + public void handle() throws Exception { + try { + // construct process + CommandExecuteResult commandExecuteResult = pythonCommandExecutor.run(buildCommand()); + + setExitStatusCode(commandExecuteResult.getExitStatusCode()); + setAppIds(commandExecuteResult.getAppIds()); + setProcessId(commandExecuteResult.getProcessId()); + setVarPool(pythonCommandExecutor.getVarPool()); + } + catch (Exception e) { + logger.error("python task failure", e); + setExitStatusCode(Constants.EXIT_CODE_FAILURE); + throw e; + } + } + @Override + public void cancelApplication(boolean cancelApplication) throws Exception { + // cancel process + pythonCommandExecutor.cancelApplication(); + } + /** + * build command + * @return raw python script + * @throws Exception exception + */ + private String buildCommand() throws Exception { + String rawPythonScript = pythonParameters.getRawScript().replaceAll("\\r\\n", "\n"); + + // replace placeholder + Map paramsMap = ParamUtils.convert(ParamUtils.getUserDefParamsMap(taskExecutionContext.getDefinedParams()), + taskExecutionContext.getDefinedParams(), + pythonParameters.getLocalParametersMap(), + CommandType.of(taskExecutionContext.getCmdTypeIfComplement()), + taskExecutionContext.getScheduleTime()); + + try { + rawPythonScript = VarPoolUtils.convertPythonScriptPlaceholders(rawPythonScript); + } + catch (StringIndexOutOfBoundsException e) { + logger.error("setShareVar field format error, raw python script : {}", rawPythonScript); + } + + if (paramsMap != null) { + rawPythonScript = ParameterUtils.convertParameterPlaceholders(rawPythonScript, ParamUtils.convert(paramsMap)); + } + + logger.info("raw python script : {}", pythonParameters.getRawScript()); + logger.info("task dir : {}", taskDir); + + return rawPythonScript; + } + @Override + public AbstractParameters getParameters() { + return pythonParameters; + } + } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index 6f642672cd..7344cf13e5 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -1464,17 +1464,20 @@ public class ProcessService { * @param state state * @param endTime endTime * @param taskInstId taskInstId + * @param varPool varPool */ public void changeTaskState(ExecutionStatus state, Date endTime, int processId, String appIds, - int taskInstId) { + int taskInstId, + String varPool) { TaskInstance taskInstance = taskInstanceMapper.selectById(taskInstId); taskInstance.setPid(processId); taskInstance.setAppLink(appIds); taskInstance.setState(state); taskInstance.setEndTime(endTime); + taskInstance.setVarPool(varPool); saveTaskInstance(taskInstance); } diff --git a/pom.xml b/pom.xml index c8cb5a9dea..e895b01d89 100644 --- a/pom.xml +++ b/pom.xml @@ -785,6 +785,7 @@ **/common/utils/StringTest.java **/common/utils/StringUtilsTest.java **/common/utils/TaskParametersUtilsTest.java + **/common/utils/VarPoolUtilsTest.java **/common/utils/HadoopUtilsTest.java **/common/utils/HttpUtilsTest.java **/common/utils/KerberosHttpClientTest.java diff --git a/sql/dolphinscheduler-postgre.sql b/sql/dolphinscheduler-postgre.sql index 5ae37e1be8..e2f5ebd91f 100644 --- a/sql/dolphinscheduler-postgre.sql +++ b/sql/dolphinscheduler-postgre.sql @@ -377,6 +377,7 @@ CREATE TABLE t_ds_process_instance ( worker_group varchar(64) , timeout int DEFAULT '0' , tenant_id int NOT NULL DEFAULT '-1' , + var_pool text , PRIMARY KEY (id) ) ; create index process_instance_index on t_ds_process_instance (process_definition_id,id); @@ -595,6 +596,7 @@ CREATE TABLE t_ds_task_instance ( executor_id int DEFAULT NULL , first_submit_time timestamp DEFAULT NULL , delay_time int DEFAULT '0' , + var_pool text , PRIMARY KEY (id) ) ; diff --git a/sql/dolphinscheduler_mysql.sql b/sql/dolphinscheduler_mysql.sql index 61e697568a..9039a19084 100644 --- a/sql/dolphinscheduler_mysql.sql +++ b/sql/dolphinscheduler_mysql.sql @@ -487,6 +487,7 @@ CREATE TABLE `t_ds_process_instance` ( `worker_group` varchar(64) DEFAULT NULL COMMENT 'worker group id', `timeout` int(11) DEFAULT '0' COMMENT 'time out', `tenant_id` int(11) NOT NULL DEFAULT '-1' COMMENT 'tenant id', + `var_pool` longtext COMMENT 'var_pool', PRIMARY KEY (`id`), KEY `process_instance_index` (`process_definition_id`,`id`) USING BTREE, KEY `start_time_index` (`start_time`) USING BTREE @@ -737,6 +738,7 @@ CREATE TABLE `t_ds_task_instance` ( `executor_id` int(11) DEFAULT NULL, `first_submit_time` datetime DEFAULT NULL COMMENT 'task first submit time', `delay_time` int(4) DEFAULT '0' COMMENT 'task delay execution time', + `var_pool` longtext COMMENT 'var_pool', PRIMARY KEY (`id`), KEY `process_instance_id` (`process_instance_id`) USING BTREE, KEY `task_instance_index` (`process_definition_id`,`process_instance_id`) USING BTREE, diff --git a/sql/upgrade/1.3.3_schema/mysql/dolphinscheduler_ddl.sql b/sql/upgrade/1.3.3_schema/mysql/dolphinscheduler_ddl.sql index 43488272e2..ae66da914f 100644 --- a/sql/upgrade/1.3.3_schema/mysql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.3.3_schema/mysql/dolphinscheduler_ddl.sql @@ -56,6 +56,46 @@ delimiter ; CALL uc_dolphin_T_t_ds_task_instance_A_delay_time(); DROP PROCEDURE uc_dolphin_T_t_ds_task_instance_A_delay_time; +-- uc_dolphin_T_t_ds_task_instance_A_var_pool +drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_task_instance_A_var_pool; +delimiter d// +CREATE PROCEDURE uc_dolphin_T_t_ds_task_instance_A_var_pool() + BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_task_instance' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='var_pool') + THEN + ALTER TABLE t_ds_task_instance ADD `var_pool` longtext NULL; + END IF; + END; + +d// + +delimiter ; +CALL uc_dolphin_T_t_ds_task_instance_A_var_pool(); +DROP PROCEDURE uc_dolphin_T_t_ds_task_instance_A_var_pool; + +-- uc_dolphin_T_t_ds_process_instance_A_var_pool +drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_process_instance_A_var_pool; +delimiter d// +CREATE PROCEDURE uc_dolphin_T_t_ds_process_instance_A_var_pool() + BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_process_instance' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='var_pool') + THEN + ALTER TABLE t_ds_process_instance ADD `var_pool` longtext NULL; + END IF; + END; + +d// + +delimiter ; +CALL uc_dolphin_T_t_ds_process_instance_A_var_pool(); +DROP PROCEDURE uc_dolphin_T_t_ds_process_instance_A_var_pool; + -- uc_dolphin_T_t_ds_process_definition_A_modify_by drop PROCEDURE if EXISTS ct_dolphin_T_t_ds_process_definition_version; delimiter d// diff --git a/sql/upgrade/1.3.3_schema/postgresql/dolphinscheduler_ddl.sql b/sql/upgrade/1.3.3_schema/postgresql/dolphinscheduler_ddl.sql index e2767617df..3351cac88c 100644 --- a/sql/upgrade/1.3.3_schema/postgresql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.3.3_schema/postgresql/dolphinscheduler_ddl.sql @@ -51,6 +51,42 @@ delimiter ; SELECT uc_dolphin_T_t_ds_task_instance_A_delay_time(); DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_task_instance_A_delay_time(); +-- uc_dolphin_T_t_ds_process_instance_A_var_pool +delimiter d// +CREATE OR REPLACE FUNCTION uc_dolphin_T_t_ds_process_instance_A_var_pool() RETURNS void AS $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_process_instance' + AND COLUMN_NAME ='var_pool') + THEN + ALTER TABLE t_ds_process_instance ADD COLUMN var_pool text; + END IF; +END; +$$ LANGUAGE plpgsql; +d// + +delimiter ; +SELECT uc_dolphin_T_t_ds_process_instance_A_var_pool(); +DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_process_instance_A_var_pool(); + +-- uc_dolphin_T_t_ds_task_instance_A_var_pool +delimiter d// +CREATE OR REPLACE FUNCTION uc_dolphin_T_t_ds_task_instance_A_var_pool() RETURNS void AS $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_task_instance' + AND COLUMN_NAME ='var_pool') + THEN + ALTER TABLE t_ds_task_instance ADD COLUMN var_pool text; + END IF; +END; +$$ LANGUAGE plpgsql; +d// + +delimiter ; +SELECT uc_dolphin_T_t_ds_task_instance_A_var_pool(); +DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_task_instance_A_var_pool(); + -- uc_dolphin_T_t_ds_process_definition_A_modify_by delimiter d// CREATE OR REPLACE FUNCTION ct_dolphin_T_t_ds_process_definition_version() RETURNS void AS $$ From 6564e96315b1a9011ea9c97145450df5c59f318f Mon Sep 17 00:00:00 2001 From: dashi Date: Fri, 25 Sep 2020 17:51:28 +0800 Subject: [PATCH 26/45] [Fix][ui] Fix the problem that all pre-tasks are lost after dag reformatting #3823 --- dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js index d7f2f78a0c..74671cca57 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js @@ -135,10 +135,12 @@ Dag.prototype.backfill = function (arg) { const dataObject = {} g.nodes().forEach(function (v) { const node = g.node(v) + const location = store.state.dag.locations[node.label] const obj = {} - obj.name = node.label + obj.name = location.name obj.x = node.x + marginX obj.y = node.y + obj.targetarr = location.targetarr dataObject[node.label] = obj }) jsPlumb.ready(() => { From eb597e67e35179ee627d8ba664c50e4f42854479 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E7=BB=A7=E5=B3=B0?= Date: Sat, 26 Sep 2020 15:45:24 +0800 Subject: [PATCH 27/45] [Improvement][API] simplify with stream (#3764) * simplify with stream * add distinct * compatible tasks is null * add unit test --- .../impl/ProcessDefinitionServiceImpl.java | 33 +++----- .../service/ProcessDefinitionServiceTest.java | 84 +++++++++++++++++++ 2 files changed, 97 insertions(+), 20 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index f4ea1bf065..cade36a1d6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -48,7 +48,7 @@ import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.process.ProcessDag; import org.apache.dolphinscheduler.common.process.Property; -import org.apache.dolphinscheduler.common.task.AbstractParameters; +import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; @@ -77,6 +77,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -84,6 +85,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -232,25 +234,16 @@ public class ProcessDefinitionServiceImpl extends BaseService implements * @return resource ids */ private String getResourceIds(ProcessData processData) { - List tasks = processData.getTasks(); - Set resourceIds = new HashSet<>(); - for (TaskNode taskNode : tasks) { - String taskParameter = taskNode.getParams(); - AbstractParameters params = TaskParametersUtils.getParameters(taskNode.getType(), taskParameter); - if (CollectionUtils.isNotEmpty(params.getResourceFilesList())) { - Set tempSet = params.getResourceFilesList().stream().map(t -> t.getId()).collect(Collectors.toSet()); - resourceIds.addAll(tempSet); - } - } - - StringBuilder sb = new StringBuilder(); - for (int i : resourceIds) { - if (sb.length() > 0) { - sb.append(","); - } - sb.append(i); - } - return sb.toString(); + return Optional.ofNullable(processData.getTasks()) + .orElse(Collections.emptyList()) + .stream() + .map(taskNode -> TaskParametersUtils.getParameters(taskNode.getType(), taskNode.getParams())) + .filter(Objects::nonNull) + .flatMap(parameters -> parameters.getResourceFilesList().stream()) + .map(ResourceInfo::getId) + .distinct() + .map(Objects::toString) + .collect(Collectors.joining(",")); } /** diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java index 33032f54e4..f35ff9509c 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.api.service; +import static org.assertj.core.api.Assertions.assertThat; + import org.apache.dolphinscheduler.api.dto.ProcessMeta; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.impl.ProcessDefinitionServiceImpl; @@ -28,6 +30,9 @@ import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.WarningType; +import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.process.ResourceInfo; +import org.apache.dolphinscheduler.common.task.shell.ShellParameters; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; @@ -51,8 +56,11 @@ import org.apache.http.entity.ContentType; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.lang.reflect.Method; import java.text.MessageFormat; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -66,6 +74,7 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import org.springframework.mock.web.MockMultipartFile; +import org.springframework.util.ReflectionUtils; import org.springframework.web.multipart.MultipartFile; import com.baomidou.mybatisplus.core.metadata.IPage; @@ -984,6 +993,81 @@ public class ProcessDefinitionServiceTest { loginUser, projectName, "1", null); } + @Test + public void testGetResourceIds() throws Exception { + // set up + Method testMethod = ReflectionUtils.findMethod(ProcessDefinitionServiceImpl.class, "getResourceIds", ProcessData.class); + assertThat(testMethod).isNotNull(); + testMethod.setAccessible(true); + + // when processData has empty task, then return empty string + ProcessData input1 = new ProcessData(); + input1.setTasks(Collections.emptyList()); + String output1 = (String) testMethod.invoke(processDefinitionService, input1); + assertThat(output1).isEmpty(); + + // when task is null, then return empty string + ProcessData input2 = new ProcessData(); + input2.setTasks(null); + String output2 = (String) testMethod.invoke(processDefinitionService, input2); + assertThat(output2).isEmpty(); + + // when task type is incorrect mapping, then return empty string + ProcessData input3 = new ProcessData(); + TaskNode taskNode3 = new TaskNode(); + taskNode3.setType("notExistType"); + input3.setTasks(Collections.singletonList(taskNode3)); + String output3 = (String) testMethod.invoke(processDefinitionService, input3); + assertThat(output3).isEmpty(); + + // when task parameter list is null, then return empty string + ProcessData input4 = new ProcessData(); + TaskNode taskNode4 = new TaskNode(); + taskNode4.setType("SHELL"); + taskNode4.setParams(null); + input4.setTasks(Collections.singletonList(taskNode4)); + String output4 = (String) testMethod.invoke(processDefinitionService, input4); + assertThat(output4).isEmpty(); + + // when resource id list is 0 1, then return 0,1 + ProcessData input5 = new ProcessData(); + TaskNode taskNode5 = new TaskNode(); + taskNode5.setType("SHELL"); + ShellParameters shellParameters5 = new ShellParameters(); + ResourceInfo resourceInfo5A = new ResourceInfo(); + resourceInfo5A.setId(0); + ResourceInfo resourceInfo5B = new ResourceInfo(); + resourceInfo5B.setId(1); + shellParameters5.setResourceList(Arrays.asList(resourceInfo5A, resourceInfo5B)); + taskNode5.setParams(JSONUtils.toJsonString(shellParameters5)); + input5.setTasks(Collections.singletonList(taskNode5)); + String output5 = (String) testMethod.invoke(processDefinitionService, input5); + assertThat(output5.split(",")).hasSize(2) + .containsExactlyInAnyOrder("0", "1"); + + // when resource id list is 0 1 1 2, then return 0,1,2 + ProcessData input6 = new ProcessData(); + TaskNode taskNode6 = new TaskNode(); + taskNode6.setType("SHELL"); + ShellParameters shellParameters6 = new ShellParameters(); + ResourceInfo resourceInfo6A = new ResourceInfo(); + resourceInfo6A.setId(0); + ResourceInfo resourceInfo6B = new ResourceInfo(); + resourceInfo6B.setId(1); + ResourceInfo resourceInfo6C = new ResourceInfo(); + resourceInfo6C.setId(1); + ResourceInfo resourceInfo6D = new ResourceInfo(); + resourceInfo6D.setId(2); + shellParameters6.setResourceList(Arrays.asList(resourceInfo6A, resourceInfo6B, resourceInfo6C, resourceInfo6D)); + taskNode6.setParams(JSONUtils.toJsonString(shellParameters6)); + input6.setTasks(Collections.singletonList(taskNode6)); + + String output6 = (String) testMethod.invoke(processDefinitionService, input6); + + assertThat(output6.split(",")).hasSize(3) + .containsExactlyInAnyOrder("0", "1", "2"); + } + /** * get mock datasource * From 11d3c2cdaf95be9ea9ef8c9e7838320dd76312ba Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Mon, 28 Sep 2020 09:56:46 +0800 Subject: [PATCH 28/45] [Improvement][server]lower weight round robin support weight (#3768) * [Improvement][server]lower weight round robin support weight * delete code * resolve conflicts --- .../dispatch/host/assign/HostWeight.java | 19 +++++++++---------- .../host/assign/LowerWeightRoundRobin.java | 13 +++++++------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java index 298a62a6d9..839ebc85c7 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java @@ -33,9 +33,9 @@ public class HostWeight { private final Host host; - private final int weight; + private final double weight; - private int currentWeight; + private double currentWeight; public HostWeight(Host host, double cpu, double memory, double loadAverage) { this.weight = getWeight(cpu, memory, loadAverage, host); @@ -43,15 +43,15 @@ public class HostWeight { this.currentWeight = weight; } - public int getCurrentWeight() { + public double getCurrentWeight() { return currentWeight; } - public int getWeight() { + public double getWeight() { return weight; } - public void setCurrentWeight(int currentWeight) { + public void setCurrentWeight(double currentWeight) { this.currentWeight = currentWeight; } @@ -68,20 +68,19 @@ public class HostWeight { + '}'; } - private int getWeight(double cpu, double memory, double loadAverage, Host host) { - int calculateWeight = (int) (cpu * CPU_FACTOR + memory * MEMORY_FACTOR + loadAverage * LOAD_AVERAGE_FACTOR); + private double getWeight(double cpu, double memory, double loadAverage, Host host) { + double calculateWeight = cpu * CPU_FACTOR + memory * MEMORY_FACTOR + loadAverage * LOAD_AVERAGE_FACTOR; return getWarmUpWeight(host, calculateWeight); - } /** * If the warm-up is not over, add the weight */ - private int getWarmUpWeight(Host host, int weight) { + private double getWarmUpWeight(Host host, double weight) { long startTime = host.getStartTime(); long uptime = System.currentTimeMillis() - startTime; if (uptime > 0 && uptime < Constants.WARM_UP_TIME) { - return (int) ((weight * Constants.WARM_UP_TIME) / uptime); + return weight * Constants.WARM_UP_TIME / uptime; } return weight; } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobin.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobin.java index 843397e20c..ea55785182 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobin.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobin.java @@ -20,24 +20,25 @@ package org.apache.dolphinscheduler.server.master.dispatch.host.assign; import java.util.Collection; /** - * lower weight round robin + * lower weight round robin */ -public class LowerWeightRoundRobin extends AbstractSelector{ +public class LowerWeightRoundRobin extends AbstractSelector { /** * select + * * @param sources sources * @return HostWeight */ @Override - public HostWeight doSelect(Collection sources){ - int totalWeight = 0; - int lowWeight = 0; + public HostWeight doSelect(Collection sources) { + double totalWeight = 0; + double lowWeight = 0; HostWeight lowerNode = null; for (HostWeight hostWeight : sources) { totalWeight += hostWeight.getWeight(); hostWeight.setCurrentWeight(hostWeight.getCurrentWeight() + hostWeight.getWeight()); - if (lowerNode == null || lowWeight > hostWeight.getCurrentWeight() ) { + if (lowerNode == null || lowWeight > hostWeight.getCurrentWeight()) { lowerNode = hostWeight; lowWeight = hostWeight.getCurrentWeight(); } From 42c88a7c68a89af7e3d7f060fd65d329a13c33d0 Mon Sep 17 00:00:00 2001 From: yangruochen Date: Mon, 28 Sep 2020 18:18:52 +0800 Subject: [PATCH 29/45] [FIX-Bug #3845][Ambari Plugin] Start Ambari report an error: Table 't_ds_process_definition_version' already exists (#3846) * [Bug][Ambari Plugin] Start Ambari report an error: Table 't_ds_process_definition_version' already exists #3845 * Revert "[Bug][Ambari Plugin] Start Ambari report an error: Table 't_ds_process_definition_version' already exists #3845" This reverts commit 56e030e6 * [Bug][Ambari Plugin] Start Ambari report an error: Table 't_ds_process_definition_version' already exists #3845 --- sql/upgrade/1.3.3_schema/mysql/dolphinscheduler_ddl.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/upgrade/1.3.3_schema/mysql/dolphinscheduler_ddl.sql b/sql/upgrade/1.3.3_schema/mysql/dolphinscheduler_ddl.sql index ae66da914f..f7b3bbcabb 100644 --- a/sql/upgrade/1.3.3_schema/mysql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.3.3_schema/mysql/dolphinscheduler_ddl.sql @@ -101,7 +101,7 @@ drop PROCEDURE if EXISTS ct_dolphin_T_t_ds_process_definition_version; delimiter d// CREATE PROCEDURE ct_dolphin_T_t_ds_process_definition_version() BEGIN - CREATE TABLE `t_ds_process_definition_version` ( + CREATE TABLE IF NOT EXISTS `t_ds_process_definition_version` ( `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'key', `process_definition_id` int(11) NOT NULL COMMENT 'process definition id', `version` int(11) DEFAULT NULL COMMENT 'process definition version', From 2a927bcad049efd2a187af5070f9b6307d7162e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9B=B9=E8=81=9A=E9=98=B3?= Date: Tue, 29 Sep 2020 10:58:14 +0800 Subject: [PATCH 30/45] Dependent Timeout --- .../formModel/_source/dependentTimeout.vue | 203 ++++++++++++++++++ .../pages/dag/_source/formModel/formModel.vue | 33 ++- .../src/js/module/i18n/locale/en_US.js | 2 + .../src/js/module/i18n/locale/zh_CN.js | 2 + 4 files changed, 238 insertions(+), 2 deletions(-) create mode 100644 dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/_source/dependentTimeout.vue diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/_source/dependentTimeout.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/_source/dependentTimeout.vue new file mode 100644 index 0000000000..076a536b44 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/_source/dependentTimeout.vue @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + \ No newline at end of file diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index 7cd63c07db..b6c48fe5e8 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -162,10 +162,18 @@ + + + Date: Tue, 29 Sep 2020 14:22:56 +0800 Subject: [PATCH 31/45] add check interval --- .../_source/formModel/_source/dependentTimeout.vue | 14 ++++++++++++-- .../src/js/module/i18n/locale/en_US.js | 1 + .../src/js/module/i18n/locale/zh_CN.js | 1 + 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/_source/dependentTimeout.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/_source/dependentTimeout.vue index 076a536b44..235fb08412 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/_source/dependentTimeout.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/_source/dependentTimeout.vue @@ -49,6 +49,12 @@ {{$t('Minute')}} + + {{$t('Check interval')}} + + + {{$t('Minute')}} + {{$t('Timeout strategy')}} @@ -110,7 +116,8 @@ // Timeout strategy strategy: ['FAILED'], // Timeout period - interval: null + interval: null, + checkInterval: null }, waitCompleteTimeout: { enable: false, @@ -131,6 +138,7 @@ // p = 0 for timeout switch; p = 1 for wait start timeout switch; p = 2 for wait complete timeout switch. if (p === 1 || p === 0) { this.waitStartTimeout.interval = is ? 30 : null + this.waitStartTimeout.checkInterval = is ? 1 : null } if (p === 2 || p === 0) { this.waitCompleteTimeout.strategy = is ? ['WARN'] : [] @@ -149,7 +157,7 @@ const reg = /^[1-9]\d*$/ if (this.enable && (this.waitCompleteTimeout.enable && !reg.test(this.waitCompleteTimeout.interval)) - || (this.waitStartTimeout.enable && !reg.test(this.waitStartTimeout.interval))) { + || (this.waitStartTimeout.enable && (!reg.test(this.waitStartTimeout.interval || !reg.test(this.waitStartTimeout.checkInterval))))) { this.$message.warning(`${this.$t('Timeout must be a positive integer')}`) return false } @@ -157,6 +165,7 @@ waitStartTimeout: { strategy: 'FAILED', interval: parseInt(this.waitStartTimeout.interval), + checkInterval: parseInt(this.waitStartTimeout.checkInterval), enable: this.waitStartTimeout.enable }, waitCompleteTimeout: { @@ -193,6 +202,7 @@ this.waitStartTimeout.enable = o.waitStartTimeout.enable || false this.waitStartTimeout.strategy = ['FAILED'] this.waitStartTimeout.interval = o.waitStartTimeout.interval || null + this.waitStartTimeout.checkInterval = o.waitStartTimeout.checkInterval || null } } }, diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js index 2052d3e2a0..50d03ef555 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js @@ -428,6 +428,7 @@ export default { 'Timeout period': 'Timeout period', 'Waiting Dependent complete': 'Waiting Dependent complete', 'Waiting Dependent start': 'Waiting Dependent start', + 'Check interval': 'Check interval', 'Timeout strategy must be selected': 'Timeout strategy must be selected', 'Timeout must be a positive integer': 'Timeout must be a positive integer', 'Add dependency': 'Add dependency', diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js index d46fdf551c..3f4be67b1e 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -422,6 +422,7 @@ export default { 'Timeout period': '超时时长', 'Waiting Dependent complete': '等待依赖完成', 'Waiting Dependent start': '等待依赖启动', + 'Check interval': '检查间隔', 'Timeout strategy must be selected': '超时策略必须选一个', 'Timeout must be a positive integer': '超时时长必须为正整数', 'Add dependency': '添加依赖', From 61fcad772299ce369b1c130c5f7fb0516b1bdc92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9B=B9=E8=81=9A=E9=98=B3?= Date: Tue, 29 Sep 2020 14:49:33 +0800 Subject: [PATCH 32/45] add timeout verification --- .../pages/dag/_source/formModel/_source/dependentTimeout.vue | 5 +++++ dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js | 1 + dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js | 2 ++ 3 files changed, 8 insertions(+) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/_source/dependentTimeout.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/_source/dependentTimeout.vue index 235fb08412..2b2ed78ccc 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/_source/dependentTimeout.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/_source/dependentTimeout.vue @@ -161,6 +161,11 @@ this.$message.warning(`${this.$t('Timeout must be a positive integer')}`) return false } + // Verify timeout duration longer than check interval + if (this.enable && this.waitStartTimeout.enable && this.waitStartTimeout.checkInterval >= this.waitStartTimeout.interval) { + this.$message.warning(`${this.$t('Timeout must be longer than check interval')}`) + return false + } this.$emit('on-timeout', { waitStartTimeout: { strategy: 'FAILED', diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js index 50d03ef555..5148849d3b 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js @@ -429,6 +429,7 @@ export default { 'Waiting Dependent complete': 'Waiting Dependent complete', 'Waiting Dependent start': 'Waiting Dependent start', 'Check interval': 'Check interval', + 'Timeout must be longer than check interval': 'Timeout must be longer than check interval', 'Timeout strategy must be selected': 'Timeout strategy must be selected', 'Timeout must be a positive integer': 'Timeout must be a positive integer', 'Add dependency': 'Add dependency', diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js index 3f4be67b1e..9341d4fd2b 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -425,6 +425,7 @@ export default { 'Check interval': '检查间隔', 'Timeout strategy must be selected': '超时策略必须选一个', 'Timeout must be a positive integer': '超时时长必须为正整数', + 'Timeout must be longer than check interval': '超时时间必须比检查间隔长', 'Add dependency': '添加依赖', and: '且', or: '或', @@ -435,6 +436,7 @@ export default { Running: '正在运行', 'Waiting for dependency to complete': '等待依赖完成', 'Delay execution': '延时执行', + 'Forced success': '强制成功过', Selected: '已选', CurrentHour: '当前小时', Last1Hour: '前1小时', From df33d1a28ee0055ae8b64335aa9234a55e65f651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9B=B9=E8=81=9A=E9=98=B3?= Date: Tue, 29 Sep 2020 23:06:53 +0800 Subject: [PATCH 33/45] Force success --- .../js/conf/home/pages/dag/_source/config.js | 7 ++++++ .../_source/instanceConditions/common.js | 3 +++ .../pages/taskInstance/_source/list.vue | 23 +++++++++++++++++++ .../src/js/conf/home/store/dag/actions.js | 12 ++++++++++ .../src/js/module/i18n/locale/en_US.js | 2 ++ .../src/js/module/i18n/locale/zh_CN.js | 1 + 6 files changed, 48 insertions(+) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js index 2e60929577..18fbd94341 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js @@ -236,6 +236,13 @@ const tasksState = { color: '#5102ce', icoUnicode: 'ans-icon-coin', isSpin: false + }, + FORCED_SUCCESS: { + id: 13, + desc: `${i18n.$t('Forced success')}`, + color: '#5102ce', + icoUnicode: 'ans-icon-success-solid', + isSpin: false } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/_source/instanceConditions/common.js b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/_source/instanceConditions/common.js index 8a13aeacb4..e917cf3af0 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/_source/instanceConditions/common.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/_source/instanceConditions/common.js @@ -63,6 +63,9 @@ const stateType = [ }, { code: 'DELAY_EXECUTION', label: `${i18n.$t('Delay execution')}` + }, { + code: 'FORCED_SUCCESS', + label: `${i18n.$t('Forced success')}` } ] diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue index cdcf0b0785..a6c7de1a1f 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue @@ -91,6 +91,16 @@ {{item.duration}} {{item.retryTimes}} + + ` @@ -156,6 +168,17 @@ } }) }, + _forceSuccess (item) { + this.forceTaskSuccess({taskInstanceId: item.id}).then(res => { + if (res.code === 0) { + this.$message.success(res.msg) + } else { + this.$message.error(res.msg) + } + }).catch(e => { + this.$message.error(e.msg) + }) + }, _go (item) { this.$router.push({ path: `/projects/instance/list/${item.processInstanceId}` }) }, diff --git a/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js b/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js index 93fab5a224..b35d07052a 100644 --- a/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js +++ b/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js @@ -734,6 +734,18 @@ export default { }) }) }, + /** + * Force fail/kill/need_fault_tolerance task success + */ + forceTaskSuccess ({ state }, payload) { + return new Promise((resolve, reject) => { + io.post(`projects/${state.projectName}/task-instance/force-success`, payload, res => { + resolve(res) + }).catch(e => { + reject(e) + }) + }) + }, /** * Query task record list */ diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js index 5148849d3b..0f59aecc5a 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js @@ -32,6 +32,7 @@ export default { 'Current node settings': 'Current node settings', 'View history': 'View history', 'View log': 'View log', + 'Force success': 'Force success', 'Enter this child node': 'Enter this child node', 'Node name': 'Node name', 'Run flag': 'Run flag', @@ -432,6 +433,7 @@ export default { 'Timeout must be longer than check interval': 'Timeout must be longer than check interval', 'Timeout strategy must be selected': 'Timeout strategy must be selected', 'Timeout must be a positive integer': 'Timeout must be a positive integer', + 'Forced success': 'Forced success', 'Add dependency': 'Add dependency', and: 'and', or: 'or', diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js index 9341d4fd2b..0481cd0137 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -32,6 +32,7 @@ export default { 'Current node settings': '当前节点设置', 'View history': '查看历史', 'View log': '查看日志', + 'Force success': '强制成功', 'Enter this child node': '进入该子节点', 'Node name': '节点名称', 'Please enter name (required)': '请输入名称(必填)', From 1c96ae09448d59bfd1ef445af4df42a74cddfa37 Mon Sep 17 00:00:00 2001 From: Yarlung Date: Thu, 8 Oct 2020 17:08:26 +0800 Subject: [PATCH 34/45] [Fix-3840][server] Fix When the tenant code is a number, it should not be saved successfully (#3867) * bug fixed #3840 * fix bug #3840 * bug fixed #3840 * bug fixed issue #3840 * bug fix #3840 * bug fix #3540 Co-authored-by: gechunfa --- .../api/controller/TaskRecordController.java | 4 +- .../dolphinscheduler/api/enums/Status.java | 4 +- .../api/service/impl/TenantServiceImpl.java | 26 ++++++---- .../api/utils/RegexUtils.java | 47 +++++++++++++++++++ .../api/utils/RegexUtilsTest.java | 39 +++++++++++++++ 5 files changed, 108 insertions(+), 12 deletions(-) create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegexUtils.java create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/RegexUtilsTest.java diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskRecordController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskRecordController.java index e20c845d42..4ff769dce6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskRecordController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskRecordController.java @@ -22,11 +22,13 @@ import org.apache.dolphinscheduler.api.service.TaskRecordService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.dao.entity.User; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.*; + import springfox.documentation.annotations.ApiIgnore; import java.util.Map; @@ -59,7 +61,7 @@ public class TaskRecordController extends BaseController { * @param taskDate task date * @param startTime start time * @param endTime end time - * @param pageNo page numbere + * @param pageNo page number * @param pageSize page size * @return task record list */ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java index 43c03f09d8..4fda99c37d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java @@ -193,7 +193,9 @@ public enum Status { BATCH_COPY_PROCESS_DEFINITION_ERROR(10159, "batch copy process definition error", "复制工作流错误"), BATCH_MOVE_PROCESS_DEFINITION_ERROR(10160, "batch move process definition error", "移动工作流错误"), QUERY_WORKFLOW_LINEAGE_ERROR(10161, "query workflow lineage error", "查询血缘失败"), - DELETE_PROCESS_DEFINITION_BY_ID_FAIL(10162,"delete process definition by id fail, for there are {0} process instances in executing using it", "删除工作流定义失败，有[{0}]个运行中的工作流实例正在使用"), + DELETE_PROCESS_DEFINITION_BY_ID_FAIL(10162, "delete process definition by id fail, for there are {0} process instances in executing using it", "删除工作流定义失败，有[{0}]个运行中的工作流实例正在使用"), + CHECK_TENANT_CODE_ERROR(10163, "Please enter the English tenant code", "请输入英文租户编码"), + UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found", "UDF函数不存在"), UDF_FUNCTION_EXISTS(20002, "UDF function already exists", "UDF函数已存在"), diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java index 3a267bcc8c..52f0d79ead 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java @@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.BaseService; import org.apache.dolphinscheduler.api.service.TenantService; import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.api.utils.RegexUtils; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.CollectionUtils; @@ -73,11 +74,11 @@ public class TenantServiceImpl extends BaseService implements TenantService { /** * create tenant * - * @param loginUser login user + * @param loginUser login user * @param tenantCode tenant code * @param tenantName tenant name - * @param queueId queue id - * @param desc description + * @param queueId queue id + * @param desc description * @return create result code * @throws Exception exception */ @@ -94,6 +95,11 @@ public class TenantServiceImpl extends BaseService implements TenantService { return result; } + if (RegexUtils.isNumeric(tenantCode)) { + putMsg(result, Status.CHECK_TENANT_CODE_ERROR); + return result; + } + if (checkTenantExists(tenantCode)) { putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, tenantCode); return result; @@ -131,8 +137,8 @@ public class TenantServiceImpl extends BaseService implements TenantService { * * @param loginUser login user * @param searchVal search value - * @param pageNo page number - * @param pageSize page size + * @param pageNo page number + * @param pageSize page size * @return tenant list page */ public Map queryTenantList(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { @@ -157,12 +163,12 @@ public class TenantServiceImpl extends BaseService implements TenantService { /** * updateProcessInstance tenant * - * @param loginUser login user - * @param id tennat id + * @param loginUser login user + * @param id tennat id * @param tenantCode tennat code * @param tenantName tennat name - * @param queueId queue id - * @param desc description + * @param queueId queue id + * @param desc description * @return update result code * @throws Exception exception */ @@ -229,7 +235,7 @@ public class TenantServiceImpl extends BaseService implements TenantService { * delete tenant * * @param loginUser login user - * @param id tenant id + * @param id tenant id * @return delete result code * @throws Exception exception */ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegexUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegexUtils.java new file mode 100644 index 0000000000..9ff7fac463 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegexUtils.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.utils; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * This is Regex expression utils. + */ +public class RegexUtils { + + /** + * check number regex expression + */ + private static final String CHECK_NUMBER = "^-?\\d+(\\.\\d+)?$"; + + private RegexUtils() { + } + + /** + * check if the input is number + * + * @param str input + * @return + */ + public static boolean isNumeric(String str) { + Pattern pattern = Pattern.compile(CHECK_NUMBER); + Matcher isNum = pattern.matcher(str); + return isNum.matches(); + } +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/RegexUtilsTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/RegexUtilsTest.java new file mode 100644 index 0000000000..5b62d51b07 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/RegexUtilsTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.utils; + +import org.junit.Assert; +import org.junit.Test; + +/** + * RegexUtils test case + */ +public class RegexUtilsTest { + + @Test + public void testIsNumeric() { + String num1 = "123467854678"; + boolean numeric = RegexUtils.isNumeric(num1); + Assert.assertTrue(numeric); + + String num2 = "0.0.01"; + boolean numeric2 = RegexUtils.isNumeric(num2); + Assert.assertFalse(numeric2); + } + +} \ No newline at end of file From 7c1ff53a7428b9edb8d179c01a6d715920745f1e Mon Sep 17 00:00:00 2001 From: yangruochen Date: Thu, 8 Oct 2020 17:23:40 +0800 Subject: [PATCH 35/45] =?UTF-8?q?[Bug-3845]postgresql=20need=20to=20add=20?= =?UTF-8?q?IF=20NOT=20EXISTS=20for=20table=20t=5Fds=5Fprocess=E2=80=A6=20(?= =?UTF-8?q?#3852)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [Bug-3845]postgresql need to add IF NOT EXISTS for table t_ds_process_definition_version too. * restart unit test * rerun unit test --- sql/upgrade/1.3.3_schema/postgresql/dolphinscheduler_ddl.sql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sql/upgrade/1.3.3_schema/postgresql/dolphinscheduler_ddl.sql b/sql/upgrade/1.3.3_schema/postgresql/dolphinscheduler_ddl.sql index 3351cac88c..477cb3bf60 100644 --- a/sql/upgrade/1.3.3_schema/postgresql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.3.3_schema/postgresql/dolphinscheduler_ddl.sql @@ -91,7 +91,7 @@ DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_task_instance_A_var_pool(); delimiter d// CREATE OR REPLACE FUNCTION ct_dolphin_T_t_ds_process_definition_version() RETURNS void AS $$ BEGIN -CREATE TABLE t_ds_process_definition_version ( +CREATE TABLE IF NOT EXISTS t_ds_process_definition_version ( id int NOT NULL , process_definition_id int NOT NULL , version int DEFAULT NULL , @@ -140,5 +140,3 @@ DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_resources_un(); - - From 1a257b10771adef0bca6595b0751e8d0ba838c35 Mon Sep 17 00:00:00 2001 From: AhahaGe Date: Fri, 9 Oct 2020 21:00:02 +0800 Subject: [PATCH 36/45] [Feature][api]Export and import workflow add processDefinitionDescription #3808 (#3847) * add .asf.yaml file * export processDefinition add desc * clear conentent in .asf.yaml * delete .asf.yaml file Co-authored-by: guirong.ggr --- .../api/service/impl/ProcessDefinitionServiceImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index cade36a1d6..7b68003440 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -683,6 +683,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements exportProcessMeta.setProjectName(processDefinition.getProjectName()); exportProcessMeta.setProcessDefinitionName(processDefinition.getName()); exportProcessMeta.setProcessDefinitionJson(processDefinition.getProcessDefinitionJson()); + exportProcessMeta.setProcessDefinitionDescription(processDefinition.getDescription()); exportProcessMeta.setProcessDefinitionLocations(processDefinition.getLocations()); exportProcessMeta.setProcessDefinitionConnects(processDefinition.getConnects()); From c4be3b57493fe75f5a5dbb9f258a0430d0363cc6 Mon Sep 17 00:00:00 2001 From: hailin0 Date: Sat, 10 Oct 2020 11:14:32 +0800 Subject: [PATCH 37/45] [Fix][API] Log dependency package conflict (#3833) Co-authored-by: lisiying --- dolphinscheduler-api/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 76dd8980b7..cd6e04fc26 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -162,6 +162,12 @@ org.apache.hadoop hadoop-client + + + org.slf4j + slf4j-log4j12 + + From 39411ce03b864bc770da220ad6f81df47bd2487b Mon Sep 17 00:00:00 2001 From: bao liang <29528966+lenboo@users.noreply.github.com> Date: Sat, 10 Oct 2020 15:05:56 +0800 Subject: [PATCH 38/45] [FIX-3617][Service]fix 2 tasks instance are generated when fault tolerance (#3873) * fix 2 tasks instance are generated when fault tolerance * update code style * update code style * update code style * update code style * update code style * update code style * update code style * update code style * update code style * update code style * update code style * update ut * update code style Co-authored-by: baoliang --- .../server/zk/ZKMasterClient.java | 604 +++++++++--------- .../service/process/ProcessService.java | 167 +++-- .../service/process/ProcessServiceTest.java | 116 ++++ pom.xml | 1 + 4 files changed, 520 insertions(+), 368 deletions(-) create mode 100644 dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java index 6abb381583..49bfb5f9a8 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java @@ -20,6 +20,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.TreeCacheEvent; import org.apache.curator.framework.recipes.locks.InterProcessMutex; + import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ZKNodeType; @@ -33,6 +34,7 @@ import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.utils.ProcessUtils; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.zk.AbstractZKClient; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -45,309 +47,309 @@ import static org.apache.dolphinscheduler.common.Constants.SLEEP_TIME_MILLIS; /** - * zookeeper master client - * - * single instance + * zookeeper master client + *

+ * single instance */ @Component public class ZKMasterClient extends AbstractZKClient { - /** - * logger - */ - private static final Logger logger = LoggerFactory.getLogger(ZKMasterClient.class); - - /** - * process service - */ - @Autowired - private ProcessService processService; - - public void start() { - - InterProcessMutex mutex = null; - try { - // create distributed lock with the root node path of the lock space as /dolphinscheduler/lock/failover/master - String znodeLock = getMasterStartUpLockPath(); - mutex = new InterProcessMutex(getZkClient(), znodeLock); - mutex.acquire(); - - // init system znode - this.initSystemZNode(); - - while (!checkZKNodeExists(NetUtils.getHost(), ZKNodeType.MASTER)){ - ThreadUtils.sleep(SLEEP_TIME_MILLIS); - } - - - // self tolerant - if (getActiveMasterNum() == 1) { - failoverWorker(null, true); - failoverMaster(null); - } - - }catch (Exception e){ - logger.error("master start up exception",e); - }finally { - releaseMutex(mutex); - } - } - - @Override - public void close(){ - super.close(); - } - - /** - * handle path events that this class cares about - * @param client zkClient - * @param event path event - * @param path zk path - */ - @Override - protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) { - //monitor master - if(path.startsWith(getZNodeParentPath(ZKNodeType.MASTER)+Constants.SINGLE_SLASH)){ - handleMasterEvent(event,path); - }else if(path.startsWith(getZNodeParentPath(ZKNodeType.WORKER)+Constants.SINGLE_SLASH)){ - //monitor worker - handleWorkerEvent(event,path); - } - } - - /** - * remove zookeeper node path - * - * @param path zookeeper node path - * @param zkNodeType zookeeper node type - * @param failover is failover - */ - private void removeZKNodePath(String path, ZKNodeType zkNodeType, boolean failover) { - logger.info("{} node deleted : {}", zkNodeType.toString(), path); - InterProcessMutex mutex = null; - try { - String failoverPath = getFailoverLockPath(zkNodeType); - // create a distributed lock - mutex = new InterProcessMutex(getZkClient(), failoverPath); - mutex.acquire(); - - String serverHost = getHostByEventDataPath(path); - // handle dead server - handleDeadServer(path, zkNodeType, Constants.ADD_ZK_OP); - //failover server - if(failover){ - failoverServerWhenDown(serverHost, zkNodeType); - } - }catch (Exception e){ - logger.error("{} server failover failed.", zkNodeType.toString()); - logger.error("failover exception ",e); - } - finally { - releaseMutex(mutex); - } - } - - /** - * failover server when server down - * - * @param serverHost server host - * @param zkNodeType zookeeper node type - * @throws Exception exception - */ - private void failoverServerWhenDown(String serverHost, ZKNodeType zkNodeType) throws Exception { - if(StringUtils.isEmpty(serverHost) || serverHost.startsWith(NetUtils.getHost())){ - return ; - } - switch (zkNodeType){ - case MASTER: - failoverMaster(serverHost); - break; - case WORKER: - failoverWorker(serverHost, true); - default: - break; - } - } - - /** - * get failover lock path - * - * @param zkNodeType zookeeper node type - * @return fail over lock path - */ - private String getFailoverLockPath(ZKNodeType zkNodeType){ - - switch (zkNodeType){ - case MASTER: - return getMasterFailoverLockPath(); - case WORKER: - return getWorkerFailoverLockPath(); - default: - return ""; - } - } - - /** - * monitor master - * @param event event - * @param path path - */ - public void handleMasterEvent(TreeCacheEvent event, String path){ - switch (event.getType()) { - case NODE_ADDED: - logger.info("master node added : {}", path); - break; - case NODE_REMOVED: - removeZKNodePath(path, ZKNodeType.MASTER, true); - break; - default: - break; - } - } - - /** - * monitor worker - * @param event event - * @param path path - */ - public void handleWorkerEvent(TreeCacheEvent event, String path){ - switch (event.getType()) { - case NODE_ADDED: - logger.info("worker node added : {}", path); - break; - case NODE_REMOVED: - logger.info("worker node deleted : {}", path); - removeZKNodePath(path, ZKNodeType.WORKER, true); - break; - default: - break; - } - } - - /** - * task needs failover if task start before worker starts - * - * @param taskInstance task instance - * @return true if task instance need fail over - */ - private boolean checkTaskInstanceNeedFailover(TaskInstance taskInstance) throws Exception { - - boolean taskNeedFailover = true; - - //now no host will execute this task instance,so no need to failover the task - if(taskInstance.getHost() == null){ - return false; - } - - // if the worker node exists in zookeeper, we must check the task starts after the worker - if(checkZKNodeExists(taskInstance.getHost(), ZKNodeType.WORKER)){ - //if task start after worker starts, there is no need to failover the task. - if(checkTaskAfterWorkerStart(taskInstance)){ - taskNeedFailover = false; - } - } - return taskNeedFailover; - } - - /** - * check task start after the worker server starts. - * - * @param taskInstance task instance - * @return true if task instance start time after worker server start date - */ - private boolean checkTaskAfterWorkerStart(TaskInstance taskInstance) { - if(StringUtils.isEmpty(taskInstance.getHost())){ - return false; - } - Date workerServerStartDate = null; - List workerServers = getServersList(ZKNodeType.WORKER); - for(Server workerServer : workerServers){ - if(taskInstance.getHost().equals(workerServer.getHost() + Constants.COLON + workerServer.getPort())){ - workerServerStartDate = workerServer.getCreateTime(); - break; - } - } - - if(workerServerStartDate != null){ - return taskInstance.getStartTime().after(workerServerStartDate); - }else{ - return false; - } - } - - /** - * failover worker tasks - * - * 1. kill yarn job if there are yarn jobs in tasks. - * 2. change task state from running to need failover. - * 3. failover all tasks when workerHost is null - * @param workerHost worker host - */ - - /** - * failover worker tasks - * - * 1. kill yarn job if there are yarn jobs in tasks. - * 2. change task state from running to need failover. - * 3. failover all tasks when workerHost is null - * @param workerHost worker host - * @param needCheckWorkerAlive need check worker alive - * @throws Exception exception - */ - private void failoverWorker(String workerHost, boolean needCheckWorkerAlive) throws Exception { - logger.info("start worker[{}] failover ...", workerHost); - - List needFailoverTaskInstanceList = processService.queryNeedFailoverTaskInstances(workerHost); - for(TaskInstance taskInstance : needFailoverTaskInstanceList){ - if(needCheckWorkerAlive){ - if(!checkTaskInstanceNeedFailover(taskInstance)){ - continue; - } - } - - ProcessInstance processInstance = processService.findProcessInstanceDetailById(taskInstance.getProcessInstanceId()); - if(processInstance != null){ - taskInstance.setProcessInstance(processInstance); - } - - TaskExecutionContext taskExecutionContext = TaskExecutionContextBuilder.get() - .buildTaskInstanceRelatedInfo(taskInstance) - .buildProcessInstanceRelatedInfo(processInstance) - .create(); - // only kill yarn job if exists , the local thread has exited - ProcessUtils.killYarnJob(taskExecutionContext); - - taskInstance.setState(ExecutionStatus.NEED_FAULT_TOLERANCE); - processService.saveTaskInstance(taskInstance); - } - logger.info("end worker[{}] failover ...", workerHost); - } - - /** - * failover master tasks - * - * @param masterHost master host - */ - private void failoverMaster(String masterHost) { - logger.info("start master failover ..."); - - List needFailoverProcessInstanceList = processService.queryNeedFailoverProcessInstances(masterHost); - - //updateProcessInstance host is null and insert into command - for(ProcessInstance processInstance : needFailoverProcessInstanceList){ - if(Constants.NULL.equals(processInstance.getHost()) ){ - continue; - } - processService.processNeedFailoverProcessInstances(processInstance); - } - - logger.info("master failover end"); - } - - public InterProcessMutex blockAcquireMutex() throws Exception { - InterProcessMutex mutex = new InterProcessMutex(getZkClient(), getMasterLockPath()); - mutex.acquire(); - return mutex; - } - + /** + * logger + */ + private static final Logger logger = LoggerFactory.getLogger(ZKMasterClient.class); + + /** + * process service + */ + @Autowired + private ProcessService processService; + + public void start() { + + InterProcessMutex mutex = null; + try { + // create distributed lock with the root node path of the lock space as /dolphinscheduler/lock/failover/master + String znodeLock = getMasterStartUpLockPath(); + mutex = new InterProcessMutex(getZkClient(), znodeLock); + mutex.acquire(); + + // init system znode + this.initSystemZNode(); + + while (!checkZKNodeExists(NetUtils.getHost(), ZKNodeType.MASTER)) { + ThreadUtils.sleep(SLEEP_TIME_MILLIS); + } + + // self tolerant + if (getActiveMasterNum() == 1) { + failoverWorker(null, true); + failoverMaster(null); + } + + } catch (Exception e) { + logger.error("master start up exception", e); + } finally { + releaseMutex(mutex); + } + } + + @Override + public void close() { + super.close(); + } + + /** + * handle path events that this class cares about + * + * @param client zkClient + * @param event path event + * @param path zk path + */ + @Override + protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) { + //monitor master + if (path.startsWith(getZNodeParentPath(ZKNodeType.MASTER) + Constants.SINGLE_SLASH)) { + handleMasterEvent(event, path); + } else if (path.startsWith(getZNodeParentPath(ZKNodeType.WORKER) + Constants.SINGLE_SLASH)) { + //monitor worker + handleWorkerEvent(event, path); + } + } + + /** + * remove zookeeper node path + * + * @param path zookeeper node path + * @param zkNodeType zookeeper node type + * @param failover is failover + */ + private void removeZKNodePath(String path, ZKNodeType zkNodeType, boolean failover) { + logger.info("{} node deleted : {}", zkNodeType.toString(), path); + InterProcessMutex mutex = null; + try { + String failoverPath = getFailoverLockPath(zkNodeType); + // create a distributed lock + mutex = new InterProcessMutex(getZkClient(), failoverPath); + mutex.acquire(); + + String serverHost = getHostByEventDataPath(path); + // handle dead server + handleDeadServer(path, zkNodeType, Constants.ADD_ZK_OP); + //failover server + if (failover) { + failoverServerWhenDown(serverHost, zkNodeType); + } + } catch (Exception e) { + logger.error("{} server failover failed.", zkNodeType.toString()); + logger.error("failover exception ", e); + } finally { + releaseMutex(mutex); + } + } + + /** + * failover server when server down + * + * @param serverHost server host + * @param zkNodeType zookeeper node type + * @throws Exception exception + */ + private void failoverServerWhenDown(String serverHost, ZKNodeType zkNodeType) throws Exception { + if (StringUtils.isEmpty(serverHost)) { + return; + } + switch (zkNodeType) { + case MASTER: + failoverMaster(serverHost); + break; + case WORKER: + failoverWorker(serverHost, true); + break; + default: + break; + } + } + + /** + * get failover lock path + * + * @param zkNodeType zookeeper node type + * @return fail over lock path + */ + private String getFailoverLockPath(ZKNodeType zkNodeType) { + + switch (zkNodeType) { + case MASTER: + return getMasterFailoverLockPath(); + case WORKER: + return getWorkerFailoverLockPath(); + default: + return ""; + } + } + + /** + * monitor master + * + * @param event event + * @param path path + */ + public void handleMasterEvent(TreeCacheEvent event, String path) { + switch (event.getType()) { + case NODE_ADDED: + logger.info("master node added : {}", path); + break; + case NODE_REMOVED: + removeZKNodePath(path, ZKNodeType.MASTER, true); + break; + default: + break; + } + } + + /** + * monitor worker + * + * @param event event + * @param path path + */ + public void handleWorkerEvent(TreeCacheEvent event, String path) { + switch (event.getType()) { + case NODE_ADDED: + logger.info("worker node added : {}", path); + break; + case NODE_REMOVED: + logger.info("worker node deleted : {}", path); + removeZKNodePath(path, ZKNodeType.WORKER, true); + break; + default: + break; + } + } + + /** + * task needs failover if task start before worker starts + * + * @param taskInstance task instance + * @return true if task instance need fail over + */ + private boolean checkTaskInstanceNeedFailover(TaskInstance taskInstance) throws Exception { + + boolean taskNeedFailover = true; + + //now no host will execute this task instance,so no need to failover the task + if (taskInstance.getHost() == null) { + return false; + } + + // if the worker node exists in zookeeper, we must check the task starts after the worker + if (checkZKNodeExists(taskInstance.getHost(), ZKNodeType.WORKER)) { + //if task start after worker starts, there is no need to failover the task. + if (checkTaskAfterWorkerStart(taskInstance)) { + taskNeedFailover = false; + } + } + return taskNeedFailover; + } + + /** + * check task start after the worker server starts. + * + * @param taskInstance task instance + * @return true if task instance start time after worker server start date + */ + private boolean checkTaskAfterWorkerStart(TaskInstance taskInstance) { + if (StringUtils.isEmpty(taskInstance.getHost())) { + return false; + } + Date workerServerStartDate = null; + List workerServers = getServersList(ZKNodeType.WORKER); + for (Server workerServer : workerServers) { + if (taskInstance.getHost().equals(workerServer.getHost() + Constants.COLON + workerServer.getPort())) { + workerServerStartDate = workerServer.getCreateTime(); + break; + } + } + if (workerServerStartDate != null) { + return taskInstance.getStartTime().after(workerServerStartDate); + } + return false; + } + + /** + * failover worker tasks + * + * 1. kill yarn job if there are yarn jobs in tasks. + * 2. change task state from running to need failover. + * 3. failover all tasks when workerHost is null + * @param workerHost worker host + */ + + /** + * failover worker tasks + *

+ * 1. kill yarn job if there are yarn jobs in tasks. + * 2. change task state from running to need failover. + * 3. failover all tasks when workerHost is null + * + * @param workerHost worker host + * @param needCheckWorkerAlive need check worker alive + * @throws Exception exception + */ + private void failoverWorker(String workerHost, boolean needCheckWorkerAlive) throws Exception { + logger.info("start worker[{}] failover ...", workerHost); + + List needFailoverTaskInstanceList = processService.queryNeedFailoverTaskInstances(workerHost); + for (TaskInstance taskInstance : needFailoverTaskInstanceList) { + if (needCheckWorkerAlive) { + if (!checkTaskInstanceNeedFailover(taskInstance)) { + continue; + } + } + + ProcessInstance processInstance = processService.findProcessInstanceDetailById(taskInstance.getProcessInstanceId()); + if (processInstance != null) { + taskInstance.setProcessInstance(processInstance); + } + + TaskExecutionContext taskExecutionContext = TaskExecutionContextBuilder.get() + .buildTaskInstanceRelatedInfo(taskInstance) + .buildProcessInstanceRelatedInfo(processInstance) + .create(); + // only kill yarn job if exists , the local thread has exited + ProcessUtils.killYarnJob(taskExecutionContext); + + taskInstance.setState(ExecutionStatus.NEED_FAULT_TOLERANCE); + processService.saveTaskInstance(taskInstance); + } + logger.info("end worker[{}] failover ...", workerHost); + } + + /** + * failover master tasks + * + * @param masterHost master host + */ + private void failoverMaster(String masterHost) { + logger.info("start master failover ..."); + + List needFailoverProcessInstanceList = processService.queryNeedFailoverProcessInstances(masterHost); + + //updateProcessInstance host is null and insert into command + for (ProcessInstance processInstance : needFailoverProcessInstanceList) { + if (Constants.NULL.equals(processInstance.getHost())) { + continue; + } + processService.processNeedFailoverProcessInstances(processInstance); + } + + logger.info("master failover end"); + } + + public InterProcessMutex blockAcquireMutex() throws Exception { + InterProcessMutex mutex = new InterProcessMutex(getZkClient(), getMasterLockPath()); + mutex.acquire(); + return mutex; + } } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index 7344cf13e5..7fca37470d 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -896,7 +896,7 @@ public class ProcessService { return task; } if(!task.getState().typeIsFinished()){ - createSubWorkProcessCommand(processInstance, task); + createSubWorkProcess(processInstance, task); } logger.info("end submit task to db successfully:{} state:{} complete, instance id:{} state: {} ", @@ -906,20 +906,22 @@ public class ProcessService { /** * set work process instance map + * consider o + * repeat running does not generate new sub process instance + * set map {parent instance id, task instance id, 0(child instance id)} * @param parentInstance parentInstance * @param parentTask parentTask * @return process instance map */ private ProcessInstanceMap setProcessInstanceMap(ProcessInstance parentInstance, TaskInstance parentTask){ ProcessInstanceMap processMap = findWorkProcessMapByParent(parentInstance.getId(), parentTask.getId()); - if(processMap != null){ + if (processMap != null) { return processMap; - }else if(parentInstance.getCommandType() == CommandType.REPEAT_RUNNING - || parentInstance.isComplementData()){ + } + if (parentInstance.getCommandType() == CommandType.REPEAT_RUNNING) { // update current task id to map - // repeat running does not generate new sub process instance processMap = findPreviousTaskProcessMap(parentInstance, parentTask); - if(processMap!= null){ + if (processMap != null) { processMap.setParentTaskInstanceId(parentTask.getId()); updateWorkProcessInstanceMap(processMap); return processMap; @@ -944,11 +946,11 @@ public class ProcessService { Integer preTaskId = 0; List preTaskList = this.findPreviousTaskListByWorkProcessId(parentProcessInstance.getId()); - for(TaskInstance task : preTaskList){ - if(task.getName().equals(parentTask.getName())){ + for (TaskInstance task : preTaskList) { + if (task.getName().equals(parentTask.getName())) { preTaskId = task.getId(); ProcessInstanceMap map = findWorkProcessMapByParent(parentProcessInstance.getId(), preTaskId); - if(map!=null){ + if (map != null) { return map; } } @@ -960,66 +962,111 @@ public class ProcessService { /** * create sub work process command + * * @param parentProcessInstance parentProcessInstance - * @param task task + * @param task task */ - private void createSubWorkProcessCommand(ProcessInstance parentProcessInstance, - TaskInstance task){ - if(!task.isSubProcess()){ + public void createSubWorkProcess(ProcessInstance parentProcessInstance, TaskInstance task) { + if (!task.isSubProcess()) { return; } - ProcessInstanceMap instanceMap = setProcessInstanceMap(parentProcessInstance, task); - TaskNode taskNode = JSONUtils.parseObject(task.getTaskJson(), TaskNode.class); - Map subProcessParam = JSONUtils.toMap(taskNode.getParams()); - Integer childDefineId = Integer.parseInt(subProcessParam.get(Constants.CMDPARAM_SUB_PROCESS_DEFINE_ID)); - - ProcessInstance childInstance = findSubProcessInstance(parentProcessInstance.getId(), task.getId()); - - CommandType fatherType = parentProcessInstance.getCommandType(); - CommandType commandType = fatherType; - if(childInstance == null){ - String fatherHistoryCommand = parentProcessInstance.getHistoryCmd(); - // sub process must begin with schedule/complement data - // if father begin with scheduler/complement data - if(fatherHistoryCommand.startsWith(CommandType.SCHEDULER.toString()) || - fatherHistoryCommand.startsWith(CommandType.COMPLEMENT_DATA.toString())){ - commandType = CommandType.valueOf(fatherHistoryCommand.split(Constants.COMMA)[0]); - } + //check create sub work flow firstly + ProcessInstanceMap instanceMap = findWorkProcessMapByParent(parentProcessInstance.getId(), task.getId()); + if (null != instanceMap && CommandType.RECOVER_TOLERANCE_FAULT_PROCESS == parentProcessInstance.getCommandType()) { + // recover failover tolerance would not create a new command when the sub command already have been created + return; } - - if(childInstance != null){ - childInstance.setState(ExecutionStatus.SUBMITTED_SUCCESS); - updateProcessInstance(childInstance); + instanceMap = setProcessInstanceMap(parentProcessInstance, task); + ProcessInstance childInstance = null; + if (instanceMap.getProcessInstanceId() != 0) { + childInstance = findProcessInstanceById(instanceMap.getProcessInstanceId()); } + Command subProcessCommand = createSubProcessCommand(parentProcessInstance, childInstance, instanceMap, task); + updateSubProcessDefinitionByParent(parentProcessInstance, subProcessCommand.getProcessDefinitionId()); + initSubInstanceState(childInstance); + createCommand(subProcessCommand); + logger.info("sub process command created: {} ", subProcessCommand); + } + + /** + * complement data needs transform parent parameter to child. + * @param instanceMap + * @param parentProcessInstance + * @return + */ + private String getSubWorkFlowParam(ProcessInstanceMap instanceMap, ProcessInstance parentProcessInstance) { // set sub work process command String processMapStr = JSONUtils.toJsonString(instanceMap); Map cmdParam = JSONUtils.toMap(processMapStr); - - if(commandType == CommandType.COMPLEMENT_DATA || - (childInstance != null && childInstance.isComplementData())){ + if (parentProcessInstance.isComplementData()) { Map parentParam = JSONUtils.toMap(parentProcessInstance.getCommandParam()); - String endTime = parentParam.get(CMDPARAM_COMPLEMENT_DATA_END_DATE); - String startTime = parentParam.get(CMDPARAM_COMPLEMENT_DATA_START_DATE); + String endTime = parentParam.get(CMDPARAM_COMPLEMENT_DATA_END_DATE); + String startTime = parentParam.get(CMDPARAM_COMPLEMENT_DATA_START_DATE); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, endTime); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, startTime); processMapStr = JSONUtils.toJsonString(cmdParam); } + return processMapStr; + } - updateSubProcessDefinitionByParent(parentProcessInstance, childDefineId); + /** + * create sub work process command + * @param parentProcessInstance + * @param childInstance + * @param instanceMap + * @param task + */ + public Command createSubProcessCommand(ProcessInstance parentProcessInstance, + ProcessInstance childInstance, + ProcessInstanceMap instanceMap, + TaskInstance task) { + CommandType commandType = getSubCommandType(parentProcessInstance, childInstance); + TaskNode taskNode = JSONUtils.parseObject(task.getTaskJson(), TaskNode.class); + Map subProcessParam = JSONUtils.toMap(taskNode.getParams()); + Integer childDefineId = Integer.parseInt(subProcessParam.get(Constants.CMDPARAM_SUB_PROCESS_DEFINE_ID)); + String processParam = getSubWorkFlowParam(instanceMap, parentProcessInstance); + + return new Command( + commandType, + TaskDependType.TASK_POST, + parentProcessInstance.getFailureStrategy(), + parentProcessInstance.getExecutorId(), + childDefineId, + processParam, + parentProcessInstance.getWarningType(), + parentProcessInstance.getWarningGroupId(), + parentProcessInstance.getScheduleTime(), + parentProcessInstance.getProcessInstancePriority() + ); + } + + /** + * initialize sub work flow state + * child instance state would be initialized when 'recovery from pause/stop/failure' + * @param childInstance + */ + private void initSubInstanceState(ProcessInstance childInstance) { + if (childInstance != null) { + childInstance.setState(ExecutionStatus.RUNNING_EXECUTION); + updateProcessInstance(childInstance); + } + } - Command command = new Command(); - command.setWarningType(parentProcessInstance.getWarningType()); - command.setWarningGroupId(parentProcessInstance.getWarningGroupId()); - command.setFailureStrategy(parentProcessInstance.getFailureStrategy()); - command.setProcessDefinitionId(childDefineId); - command.setScheduleTime(parentProcessInstance.getScheduleTime()); - command.setExecutorId(parentProcessInstance.getExecutorId()); - command.setCommandParam(processMapStr); - command.setCommandType(commandType); - command.setProcessInstancePriority(parentProcessInstance.getProcessInstancePriority()); - command.setWorkerGroup(parentProcessInstance.getWorkerGroup()); - createCommand(command); - logger.info("sub process command created: {} ", command.toString()); + /** + * get sub work flow command type + * child instance exist: child command = fatherCommand + * child instance not exists: child command = fatherCommand[0] + * + * @param parentProcessInstance + * @return + */ + private CommandType getSubCommandType(ProcessInstance parentProcessInstance, ProcessInstance childInstance) { + CommandType commandType = parentProcessInstance.getCommandType(); + if (childInstance == null) { + String fatherHistoryCommand = parentProcessInstance.getHistoryCmd(); + commandType = CommandType.valueOf(fatherHistoryCommand.split(Constants.COMMA)[0]); + } + return commandType; } /** @@ -1497,20 +1544,6 @@ public class ProcessService { return result; } - /** - * update pid and app links field by task instance id - * @param taskInstId taskInstId - * @param pid pid - * @param appLinks appLinks - */ - public void updatePidByTaskInstId(int taskInstId, int pid,String appLinks) { - - TaskInstance taskInstance = taskInstanceMapper.selectById(taskInstId); - taskInstance.setPid(pid); - taskInstance.setAppLink(appLinks); - saveTaskInstance(taskInstance); - } - /** * query schedule by id * @param id id diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java new file mode 100644 index 0000000000..74b52bb316 --- /dev/null +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.service.process; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.WarningType; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.dao.entity.Command; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.ProcessInstanceMap; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; + +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import org.junit.Assert; +import org.junit.Test; + +import com.fasterxml.jackson.databind.JsonNode; + +/** + * process service test + */ +public class ProcessServiceTest { + + @Test + public void testCreateSubCommand() { + ProcessService processService = new ProcessService(); + ProcessInstance parentInstance = new ProcessInstance(); + parentInstance.setProcessDefinitionId(1); + parentInstance.setWarningType(WarningType.SUCCESS); + parentInstance.setWarningGroupId(0); + + TaskInstance task = new TaskInstance(); + task.setTaskJson("{\"params\":{\"processDefinitionId\":100}}"); + task.setId(10); + + ProcessInstance childInstance = null; + ProcessInstanceMap instanceMap = new ProcessInstanceMap(); + instanceMap.setParentProcessInstanceId(1); + instanceMap.setParentTaskInstanceId(10); + Command command = null; + + //father history: start; child null == command type: start + parentInstance.setHistoryCmd("START_PROCESS"); + parentInstance.setCommandType(CommandType.START_PROCESS); + command = processService.createSubProcessCommand( + parentInstance, childInstance, instanceMap, task + ); + Assert.assertEquals(CommandType.START_PROCESS, command.getCommandType()); + + //father history: start,start failure; child null == command type: start + parentInstance.setCommandType(CommandType.START_FAILURE_TASK_PROCESS); + parentInstance.setHistoryCmd("START_PROCESS,START_FAILURE_TASK_PROCESS"); + command = processService.createSubProcessCommand( + parentInstance, childInstance, instanceMap, task + ); + Assert.assertEquals(CommandType.START_PROCESS, command.getCommandType()); + + //father history: scheduler,start failure; child null == command type: scheduler + parentInstance.setCommandType(CommandType.START_FAILURE_TASK_PROCESS); + parentInstance.setHistoryCmd("SCHEDULER,START_FAILURE_TASK_PROCESS"); + command = processService.createSubProcessCommand( + parentInstance, childInstance, instanceMap, task + ); + Assert.assertEquals(CommandType.SCHEDULER, command.getCommandType()); + + //father history: complement,start failure; child null == command type: complement + + String startString = "2020-01-01 00:00:00"; + String endString = "2020-01-10 00:00:00"; + parentInstance.setCommandType(CommandType.START_FAILURE_TASK_PROCESS); + parentInstance.setHistoryCmd("COMPLEMENT_DATA,START_FAILURE_TASK_PROCESS"); + Map complementMap = new HashMap<>(); + complementMap.put(Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE, startString); + complementMap.put(Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE, endString); + parentInstance.setCommandParam(JSONUtils.toJsonString(complementMap)); + command = processService.createSubProcessCommand( + parentInstance, childInstance, instanceMap, task + ); + Assert.assertEquals(CommandType.COMPLEMENT_DATA, command.getCommandType()); + + JsonNode complementDate = JSONUtils.parseObject(command.getCommandParam()); + Date start = DateUtils.stringToDate(complementDate.get(Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE).asText()); + Date end = DateUtils.stringToDate(complementDate.get(Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE).asText()); + Assert.assertEquals(startString, DateUtils.dateToString(start)); + Assert.assertEquals(endString, DateUtils.dateToString(end)); + + //father history: start,failure,start failure; child not null == command type: start failure + childInstance = new ProcessInstance(); + parentInstance.setCommandType(CommandType.START_FAILURE_TASK_PROCESS); + parentInstance.setHistoryCmd("START_PROCESS,START_FAILURE_TASK_PROCESS"); + command = processService.createSubProcessCommand( + parentInstance, childInstance, instanceMap, task + ); + Assert.assertEquals(CommandType.START_FAILURE_TASK_PROCESS, command.getCommandType()); + } +} diff --git a/pom.xml b/pom.xml index e895b01d89..9e4934e833 100644 --- a/pom.xml +++ b/pom.xml @@ -853,6 +853,7 @@ **/server/worker/EnvFileTest.java **/server/worker/runner/TaskExecuteThreadTest.java **/service/quartz/cron/CronUtilsTest.java + **/service/process/ProcessServiceTest.java **/service/zk/DefaultEnsembleProviderTest.java **/service/zk/ZKServerTest.java **/service/zk/CuratorZookeeperClientTest.java From 9de7d3c7725588393b4e90bef28697dda4b20f5d Mon Sep 17 00:00:00 2001 From: bao liang <29528966+lenboo@users.noreply.github.com> Date: Mon, 12 Oct 2020 10:23:44 +0800 Subject: [PATCH 39/45] [FIX-3573][DAO] potential horizontal unauthorized access (#3880) * fix bug[#3573] potential horizontal unauthorized access * fix bug[#3573] potential horizontal unauthorized access Co-authored-by: baoliang --- .../dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml index 831c4a9c23..83864b5163 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml @@ -167,7 +167,7 @@ + select pd.* + from t_ds_process_definition pd + WHERE pd.project_id = #{projectId} + and pd.name = #{processDefinitionName} +