From 3d4acbc012d186910a3a836423a28bd72251cd59 Mon Sep 17 00:00:00 2001 From: bcai Date: Mon, 2 Mar 2020 15:02:17 +0800 Subject: [PATCH 01/59] fix github action rerun failed #2052 --- .github/workflows/ci_backend.yml | 4 ++-- .github/workflows/ci_e2e.yml | 2 +- .github/workflows/ci_frontend.yml | 4 ++-- .github/workflows/ci_ut.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_backend.yml b/.github/workflows/ci_backend.yml index afd241d94e..1ca15c2813 100644 --- a/.github/workflows/ci_backend.yml +++ b/.github/workflows/ci_backend.yml @@ -45,7 +45,7 @@ jobs: Compile-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: @@ -55,7 +55,7 @@ jobs: License-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml index 13f12641fe..09f66fab11 100644 --- a/.github/workflows/ci_e2e.yml +++ b/.github/workflows/ci_e2e.yml @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 with: submodules: true - uses: actions/cache@v1 diff --git a/.github/workflows/ci_frontend.yml b/.github/workflows/ci_frontend.yml index fab75c6341..1f959b40eb 100644 --- a/.github/workflows/ci_frontend.yml +++ b/.github/workflows/ci_frontend.yml @@ -34,7 +34,7 @@ jobs: matrix: os: [ubuntu-latest, macos-latest] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up Node.js uses: actions/setup-node@v1 with: @@ -49,7 +49,7 @@ jobs: License-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 6f3eda0362..6451c2f8bb 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 with: submodules: true - uses: actions/cache@v1 From 8b894de3d2caf644818d7c84444327a43d864d16 Mon Sep 17 00:00:00 2001 From: bcai Date: Mon, 2 Mar 2020 15:18:35 +0800 Subject: [PATCH 02/59] remove submodules for checkout v2 --- .github/workflows/ci_e2e.yml | 2 -- .github/workflows/ci_ut.yml | 2 -- 2 files changed, 4 deletions(-) diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml index 09f66fab11..5282681fb6 100644 --- a/.github/workflows/ci_e2e.yml +++ b/.github/workflows/ci_e2e.yml @@ -30,8 +30,6 @@ jobs: steps: - uses: actions/checkout@v2 - with: - submodules: true - uses: actions/cache@v1 with: path: ~/.m2/repository diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 6451c2f8bb..45ff3ca29d 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -30,8 +30,6 @@ jobs: steps: - uses: actions/checkout@v2 - with: - submodules: true - uses: actions/cache@v1 with: path: ~/.m2/repository From fbf05de39b5c3e2060f1a145dc7598c739916db6 Mon Sep 17 00:00:00 2001 From: Tboy Date: Tue, 3 Mar 2020 12:27:07 +0800 Subject: [PATCH 03/59] delete duplicate startLocalZkServer (#2062) --- dolphinscheduler-service/src/test/java/queue/ZKServer.java | 1 - 1 file changed, 1 deletion(-) diff --git a/dolphinscheduler-service/src/test/java/queue/ZKServer.java b/dolphinscheduler-service/src/test/java/queue/ZKServer.java index 65fb95c02b..3d503d5691 100644 --- a/dolphinscheduler-service/src/test/java/queue/ZKServer.java +++ b/dolphinscheduler-service/src/test/java/queue/ZKServer.java @@ -80,7 +80,6 @@ public class ZKServer { */ public static void startLocalZkServer(final int port) { startLocalZkServer(port, org.apache.commons.io.FileUtils.getTempDirectoryPath() + File.separator + "test-" + System.currentTimeMillis()); - startLocalZkServer(port, org.apache.commons.io.FileUtils.getTempDirectoryPath() + "test-" + System.currentTimeMillis()); } /** From 3277b90647296ab3c72d74e1b22d2ac22c394eba Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Tue, 3 Mar 2020 14:38:21 +0800 Subject: [PATCH 04/59] Change DOM label (#2064) * Change DOM label * Change name to lowercase --- .../pages/dag/_source/formModel/formModel.vue | 2 +- .../formModel/tasks/_source/nodeStatus.vue | 10 +++++----- .../dag/_source/formModel/tasks/conditions.vue | 15 +++++---------- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index 2500ce5772..ec228fa7ab 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -280,7 +280,7 @@ import mDependent from './tasks/dependent' import mHttp from './tasks/http' import mDatax from './tasks/datax' - import mConditions from './tasks/CONDITIONS' + import mConditions from './tasks/conditions' import mSqoop from './tasks/sqoop' import mSubProcess from './tasks/sub_process' import mSelectInput from './_source/selectInput' diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue index 4afb8b46c5..fa7ee89e98 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue @@ -27,17 +27,17 @@ - + - + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue index 4ac04d91a6..e4f9b3f094 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue @@ -19,16 +19,12 @@
{{$t('Custom Parameters')}}
-
+
- -  - - -  - + +
@@ -44,14 +40,13 @@ @click="!isDetails && _setRelation($index)"> {{el.relation === 'AND' ? $t('and') : $t('or')}} - -  - + Date: Wed, 4 Mar 2020 09:28:23 +0800 Subject: [PATCH 05/59] 1. add input cache for datax task (#2034) 2. add cache for codemirror component, include python, shell and sql task. --- .../dag/_source/formModel/tasks/datax.vue | 49 ++++++++++++++++++- .../dag/_source/formModel/tasks/python.vue | 34 +++++++++++-- .../dag/_source/formModel/tasks/shell.vue | 29 +++++++++-- .../pages/dag/_source/formModel/tasks/sql.vue | 44 ++++++++++++++++- 4 files changed, 145 insertions(+), 11 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue index ce918f49cf..959610f95a 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue @@ -232,6 +232,8 @@ * Processing code highlighting */ _handlerEditor () { + this._destroyEditor() + // editor editor = codemirror('code-sql-mirror', { mode: 'sql', @@ -249,9 +251,34 @@ // Monitor keyboard editor.on('keypress', this.keypress) + editor.on('changes', () => { + this._cacheParams() + }) + editor.setValue(this.sql) return editor + }, + _cacheParams () { + this.$emit('on-cache-params', { + dsType: this.dsType, + dataSource: this.rtDatasource, + dtType: this.dtType, + dataTarget: this.rtDatatarget, + sql: editor?editor.getValue():'', + targetTable: this.targetTable, + jobSpeedByte: this.jobSpeedByte * 1024, + jobSpeedRecord: this.jobSpeedRecord, + preStatements: this.preStatements, + postStatements: this.postStatements + }); + }, + _destroyEditor () { + if (editor) { + editor.toTextArea() // Uninstall + editor.off($('.code-sql-mirror'), 'keypress', this.keypress) + editor.off($('.code-sql-mirror'), 'changes', this.changes) + } } }, created () { @@ -286,7 +313,27 @@ editor.off($('.code-sql-mirror'), 'keypress', this.keypress) } }, - computed: {}, + watch: { + //Watch the cacheParams + cacheParams (val) { + this._cacheParams(); + } + }, + computed: { + cacheParams () { + return { + dsType: this.dsType, + dataSource: this.rtDatasource, + dtType: this.dtType, + dataTarget: this.rtDatatarget, + targetTable: this.targetTable, + jobSpeedByte: this.jobSpeedByte * 1024, + jobSpeedRecord: this.jobSpeedRecord, + preStatements: this.preStatements, + postStatements: this.postStatements + } + } + }, components: { mListBox, mDatasource, mLocalParams, mStatementList, mSelectInput } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue index e565b4a6bd..6f495d22a0 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue @@ -129,6 +129,8 @@ * Processing code highlighting */ _handlerEditor () { + this._destroyEditor() + // editor editor = codemirror('code-python-mirror', { mode: 'python', @@ -143,26 +145,45 @@ } } + this.changes = () => { + this._cacheParams() + } + // Monitor keyboard editor.on('keypress', this.keypress) + editor.on('changes', this.changes) + editor.setValue(this.rawScript) return editor + }, + _cacheParams () { + this.$emit('on-cache-params', { + resourceList: this.cacheResourceList, + localParams: this.localParams, + rawScript: editor ? editor.getValue() : '' + }); + }, + _destroyEditor () { + if (editor) { + editor.toTextArea() // Uninstall + editor.off($('.code-python-mirror'), 'keypress', this.keypress) + editor.off($('.code-python-mirror'), 'changes', this.changes) + } } }, watch: { //Watch the cacheParams cacheParams (val) { - this.$emit('on-cache-params', val); + this._cacheParams() } }, computed: { cacheParams () { return { resourceList: this.cacheResourceList, - localParams: this.localParams, - rawScript: editor ? editor.getValue() : '' + localParams: this.localParams } } }, @@ -193,8 +214,11 @@ }, 200) }, destroyed () { - editor.toTextArea() // Uninstall - editor.off($('.code-python-mirror'), 'keypress', this.keypress) + if (editor) { + editor.toTextArea() // Uninstall + editor.off($('.code-python-mirror'), 'keypress', this.keypress) + editor.off($('.code-python-mirror'), 'changes', this.changes) + } }, components: { mLocalParams, mListBox, mResources } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue index ad40c586b9..a4b20f3310 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue @@ -163,6 +163,8 @@ * Processing code highlighting */ _handlerEditor () { + this._destroyEditor() + // editor editor = codemirror('code-shell-mirror', { mode: 'shell', @@ -177,25 +179,45 @@ } } + this.changes = () => { + this._cacheParams() + } + // Monitor keyboard editor.on('keypress', this.keypress) + + editor.on('changes', this.changes) + editor.setValue(this.rawScript) return editor + }, + _cacheParams () { + this.$emit('on-cache-params', { + resourceList: this.cacheResourceList, + localParams: this.localParams, + rawScript: editor ? editor.getValue() : '' + }); + }, + _destroyEditor () { + if (editor) { + editor.toTextArea() // Uninstall + editor.off($('.code-sql-mirror'), 'keypress', this.keypress) + editor.off($('.code-sql-mirror'), 'changes', this.changes) + } } }, watch: { //Watch the cacheParams cacheParams (val) { - this.$emit('on-cache-params', val); + this._cacheParams() } }, computed: { cacheParams () { return { resourceList: this.cacheResourceList, - localParams: this.localParams, - rawScript: editor ? editor.getValue() : '' + localParams: this.localParams } } }, @@ -229,6 +251,7 @@ if (editor) { editor.toTextArea() // Uninstall editor.off($('.code-shell-mirror'), 'keypress', this.keypress) + editor.off($('.code-shell-mirror'), 'changes', this.changes) } }, components: { mLocalParams, mListBox, mResources, mScriptBox } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue index eb5c40992e..843c1eaf1f 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue @@ -318,6 +318,8 @@ * Processing code highlighting */ _handlerEditor () { + this._destroyEditor() + // editor editor = codemirror('code-sql-mirror', { mode: 'sql', @@ -332,9 +334,15 @@ } } + this.changes = () => { + this._cacheParams() + } + // Monitor keyboard editor.on('keypress', this.keypress) + editor.on('changes', this.changes) + editor.setValue(this.sql) return editor @@ -351,6 +359,38 @@ this.receivers = res.receivers && res.receivers.split(',') || [] this.receiversCc = res.receiversCc && res.receiversCc.split(',') || [] }) + }, + _cacheParams () { + this.$emit('on-cache-params', { + type: this.type, + datasource: this.rtDatasource, + sql: editor ? editor.getValue() : '', + udfs: this.udfs, + sqlType: this.sqlType, + title: this.title, + receivers: this.receivers.join(','), + receiversCc: this.receiversCc.join(','), + showType: (() => { + + let showType = this.showType + if (showType.length === 2 && showType[0] === 'ATTACHMENT') { + return [showType[1], showType[0]].join(',') + } else { + return showType.join(',') + } + })(), + localParams: this.localParams, + connParams: this.connParams, + preStatements: this.preStatements, + postStatements: this.postStatements + }); + }, + _destroyEditor () { + if (editor) { + editor.toTextArea() // Uninstall + editor.off($('.code-sql-mirror'), 'keypress', this.keypress) + editor.off($('.code-sql-mirror'), 'changes', this.changes) + } } }, watch: { @@ -373,7 +413,7 @@ }, //Watch the cacheParams cacheParams (val) { - this.$emit('on-cache-params', val); + this._cacheParams() } }, created () { @@ -418,6 +458,7 @@ if (editor) { editor.toTextArea() // Uninstall editor.off($('.code-sql-mirror'), 'keypress', this.keypress) + editor.off($('.code-sql-mirror'), 'changes', this.changes) } }, computed: { @@ -425,7 +466,6 @@ return { type: this.type, datasource: this.rtDatasource, - sql: editor ? editor.getValue() : '', udfs: this.udfs, sqlType: this.sqlType, title: this.title, From b8014d91cf7c03ab821583f47f84a8af6606b675 Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Thu, 5 Mar 2020 11:04:00 +0800 Subject: [PATCH 06/59] add workflow id, name for email content (#2071) * add tenantService UT * fix spell * #464 email content * modify mysql1.2.2 ddl --- .../dolphinscheduler/api/service/TenantServiceTest.java | 8 ++++++++ .../api/utils/ZookeeperMonitorUtilsTest.java | 2 +- .../java/org/apache/dolphinscheduler/dao/AlertDao.java | 6 ++++-- .../server/master/runner/MasterTaskExecThread.java | 4 +++- .../dolphinscheduler/server/utils/AlertManager.java | 4 +++- sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql | 2 +- .../1.2.2_schema/postgresql/dolphinscheduler_ddl.sql | 2 +- 7 files changed, 21 insertions(+), 7 deletions(-) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java index 31c8c0222d..d6fb6b219c 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java @@ -175,6 +175,14 @@ public class TenantServiceTest { logger.info(result.toString()); List tenantList = (List) result.get(Constants.DATA_LIST); Assert.assertTrue(CollectionUtils.isNotEmpty(tenantList)); + + Mockito.when( tenantMapper.queryByTenantCode("1")).thenReturn(getList()); + Map successRes = tenantService.queryTenantList("1"); + Assert.assertEquals(Status.SUCCESS,successRes.get(Constants.STATUS)); + + Mockito.when( tenantMapper.queryByTenantCode("1")).thenReturn(null); + Map tenantNotExistRes = tenantService.queryTenantList("1"); + Assert.assertEquals(Status.TENANT_NOT_EXIST,tenantNotExistRes.get(Constants.STATUS)); } @Test diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitorUtilsTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitorUtilsTest.java index b3626fa8a9..0d89d4b6e3 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitorUtilsTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitorUtilsTest.java @@ -28,7 +28,7 @@ public class ZookeeperMonitorUtilsTest { @Test - public void testGetMasterLsit(){ + public void testGetMasterList(){ ZookeeperMonitor zookeeperMonitor = new ZookeeperMonitor(); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java index 19dbf46a6b..1a8c09e611 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java @@ -144,9 +144,11 @@ public class AlertDao extends AbstractBaseDao { * @param taskId taskId * @param taskName taskName */ - public void sendTaskTimeoutAlert(int alertgroupId,String receivers,String receiversCc,int taskId,String taskName){ + public void sendTaskTimeoutAlert(int alertgroupId,String receivers,String receiversCc, int processInstanceId, + String processInstanceName, int taskId,String taskName){ Alert alert = new Alert(); - String content = String.format("[{'id':'%d','name':'%s','event':'timeout','warnLevel':'middle'}]",taskId,taskName); + String content = String.format("[{'process instance id':'%d','task name':'%s','task id':'%d','task name':'%s'," + + "'event':'timeout','warnLevel':'middle'}]", processInstanceId, processInstanceName, taskId, taskName); alert.setTitle("Task Timeout Warn"); alert.setShowType(ShowType.TABLE); alert.setContent(content); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java index 66d1a3f4c2..37262c05e2 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java @@ -128,7 +128,9 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { // process define ProcessDefinition processDefine = processService.findProcessDefineById(processInstance.getProcessDefinitionId()); // send warn mail - alertDao.sendTaskTimeoutAlert(processInstance.getWarningGroupId(),processDefine.getReceivers(),processDefine.getReceiversCc(),taskInstance.getId(),taskInstance.getName()); + alertDao.sendTaskTimeoutAlert(processInstance.getWarningGroupId(),processDefine.getReceivers(), + processDefine.getReceiversCc(), processInstance.getId(), processInstance.getName(), + taskInstance.getId(),taskInstance.getName()); checkTimeout = false; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java index f3441edd17..050af1eec5 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java @@ -133,6 +133,8 @@ public class AlertManager { continue; } LinkedHashMap failedTaskMap = new LinkedHashMap(); + failedTaskMap.put("process instance id", String.valueOf(processInstance.getId())); + failedTaskMap.put("process instance name", processInstance.getName()); failedTaskMap.put("task id", String.valueOf(task.getId())); failedTaskMap.put("task name", task.getName()); failedTaskMap.put("task type", task.getTaskType()); @@ -193,7 +195,7 @@ public class AlertManager { logger.info("add alert to db , alert : {}", alert.toString()); }catch (Exception e){ - logger.error("send alert failed! " + e); + logger.error("send alert failed:{} ", e.getMessage()); } } diff --git a/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql b/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql index 790a9a8ada..e5785d9997 100644 --- a/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql @@ -36,7 +36,7 @@ delimiter ; CALL uc_dolphin_T_t_ds_process_definition_A_modify_by; DROP PROCEDURE uc_dolphin_T_t_ds_process_definition_A_modify_by; --- uc_dolphin_T_t_ds_process_definition_A_modify_by +-- uc_dolphin_T_t_ds_task_instance_A_executor_id drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_task_instance_A_executor_id; delimiter d// CREATE PROCEDURE uc_dolphin_T_t_ds_task_instance_A_executor_id() diff --git a/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql b/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql index cbe7c22bbe..70471cb26d 100644 --- a/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql @@ -32,7 +32,7 @@ delimiter ; SELECT uc_dolphin_T_t_ds_process_definition_A_modify_by(); DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_process_definition_A_modify_by(); --- uc_dolphin_T_t_ds_process_definition_A_modify_by +-- uc_dolphin_T_t_ds_task_instance_A_executor_id delimiter d// CREATE OR REPLACE FUNCTION uc_dolphin_T_t_ds_task_instance_A_executor_id() RETURNS void AS $$ BEGIN From 3e5a939913527c6aaafca5f170b2460cae2de51f Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Thu, 5 Mar 2020 11:07:44 +0800 Subject: [PATCH 07/59] delete unused import (#2066) --- .../alert/template/impl/DefaultHTMLTemplate.java | 1 - .../apache/dolphinscheduler/alert/utils/JSONUtilsTest.java | 1 - .../apache/dolphinscheduler/api/service/MonitorService.java | 1 - .../api/controller/LoginControllerTest.java | 1 - .../api/controller/MonitorControllerTest.java | 1 - .../api/controller/ProjectControllerTest.java | 2 -- .../api/controller/ResourcesControllerTest.java | 2 -- .../api/controller/TaskRecordControllerTest.java | 5 ----- .../api/controller/WorkerGroupControllerTest.java | 1 - .../dolphinscheduler/api/utils/FourLetterWordMainTest.java | 1 - .../apache/dolphinscheduler/common/log/WorkerLogFilter.java | 3 --- .../common/log/TaskLogDiscriminatorTest.java | 2 -- .../dolphinscheduler/common/log/TaskLogFilterTest.java | 1 - .../apache/dolphinscheduler/common/utils/HttpUtilsTest.java | 1 - .../dolphinscheduler/common/utils/SchemaUtilsTest.java | 3 --- .../dolphinscheduler/common/utils/StringUtilsTest.java | 2 -- .../common/utils/TaskParametersUtilsTest.java | 3 --- .../dolphinscheduler/dao/entity/ProcessDefinition.java | 1 - .../org/apache/dolphinscheduler/dao/entity/UdfFunc.java | 1 - .../dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java | 1 - .../dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java | 1 - .../dolphinscheduler/dao/mapper/ProjectUserMapperTest.java | 2 -- .../apache/dolphinscheduler/dao/mapper/QueueMapperTest.java | 1 - .../remote/command/ExecuteTaskRequestCommand.java | 2 +- .../remote/command/ExecuteTaskResponseCommand.java | 2 +- .../org/apache/dolphinscheduler/remote/command/Ping.java | 1 - .../remote/command/log/GetLogBytesRequestCommand.java | 1 - .../remote/command/log/RollViewLogRequestCommand.java | 1 - .../remote/command/log/ViewLogRequestCommand.java | 1 - .../server/worker/task/sqoop/SqoopTaskTest.java | 6 ------ .../dolphinscheduler/service/log/LogClientService.java | 1 - 31 files changed, 2 insertions(+), 51 deletions(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java index 428fa4cb62..79123a1e80 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.alert.template.impl; import org.apache.dolphinscheduler.alert.template.AlertTemplate; import org.apache.dolphinscheduler.alert.utils.Constants; import org.apache.dolphinscheduler.alert.utils.JSONUtils; -import org.apache.dolphinscheduler.alert.utils.MailUtils; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java index cb63a22d79..277c42b5bd 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.alert.utils; -import org.junit.After; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java index 118c5ce936..3370961fd4 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java @@ -28,7 +28,6 @@ import org.apache.dolphinscheduler.dao.entity.ZookeeperRecord; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java index bddc055de3..f5a28d01ae 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java @@ -28,7 +28,6 @@ import org.springframework.test.web.servlet.MvcResult; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java index 8fc055daf1..41674d3e54 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import com.alibaba.fastjson.JSONObject; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java index bab82df59d..42cdd1705a 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java @@ -29,8 +29,6 @@ import org.springframework.test.web.servlet.MvcResult; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; -import javax.ws.rs.POST; - import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java index 46d85f4d8d..40bdd5490d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java @@ -18,13 +18,11 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.utils.JSONUtils; import com.alibaba.fastjson.JSONObject; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java index 8bddb0f905..943e14607b 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java @@ -18,9 +18,6 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.enums.FailureStrategy; -import org.apache.dolphinscheduler.common.enums.Priority; -import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.junit.Assert; import org.junit.Test; @@ -31,9 +28,7 @@ import org.springframework.test.web.servlet.MvcResult; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; -import static org.junit.Assert.*; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java index 65ecd3f759..8517ad2d45 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java @@ -29,7 +29,6 @@ import org.springframework.test.web.servlet.MvcResult; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; -import static org.junit.Assert.*; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMainTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMainTest.java index e8adc6ca9c..69d1f21c37 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMainTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMainTest.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.utils; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/WorkerLogFilter.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/WorkerLogFilter.java index abcc8bc619..6240ed9a2e 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/WorkerLogFilter.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/WorkerLogFilter.java @@ -20,9 +20,6 @@ import ch.qos.logback.classic.Level; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.filter.Filter; import ch.qos.logback.core.spi.FilterReply; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; - -import java.util.Arrays; /** * worker log filter diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java index 8745a4f6b4..ff298000f5 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java @@ -27,8 +27,6 @@ import org.slf4j.Marker; import java.util.Map; -import static org.junit.Assert.*; - public class TaskLogDiscriminatorTest { /** diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java index 52767074da..5cca6403c8 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java @@ -21,7 +21,6 @@ import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.classic.spi.IThrowableProxy; import ch.qos.logback.classic.spi.LoggerContextVO; import ch.qos.logback.core.spi.FilterReply; -import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.junit.Assert; import org.junit.Test; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java index 20994ac99f..41049ec335 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.common.utils; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java index 907a09e458..7885806b96 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java @@ -19,15 +19,12 @@ package org.apache.dolphinscheduler.common.utils; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.Mockito; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileNotFoundException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java index 947e7310db..eca22def30 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java @@ -19,8 +19,6 @@ package org.apache.dolphinscheduler.common.utils; import org.junit.Assert; import org.junit.Test; -import java.util.ArrayList; - public class StringUtilsTest { @Test public void testIsNotEmpty() { diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java index db4a86bc26..b316b17469 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java @@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.common.utils; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java index 6e7ea7d64f..dbb880c025 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java @@ -29,7 +29,6 @@ import com.baomidou.mybatisplus.core.toolkit.StringUtils; import java.util.Date; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.stream.Collectors; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java index 9093034ae8..3518676337 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.dao.entity; -import com.baomidou.mybatisplus.annotation.TableField; import org.apache.dolphinscheduler.common.enums.UdfType; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java index 5e5277b997..58bd673fc5 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.dao.upgrade; import org.apache.dolphinscheduler.common.utils.ConnectionUtils; -import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java index c0dc905848..5db273642a 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.dao.upgrade; import org.apache.dolphinscheduler.common.utils.ConnectionUtils; -import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java index 102d8cd5db..6e995ef3c7 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.dao.mapper; -import org.apache.dolphinscheduler.common.enums.ExecutionStatus; -import org.apache.dolphinscheduler.dao.entity.ProjectUser; import org.apache.dolphinscheduler.dao.entity.ProjectUser; import org.junit.Assert; import org.junit.Test; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java index 62b034a081..30d2be03e0 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.dao.mapper; -import org.apache.dolphinscheduler.dao.entity.Queue; import org.apache.dolphinscheduler.dao.entity.Queue; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskRequestCommand.java index beec055403..93c536c347 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskRequestCommand.java @@ -1 +1 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; import java.util.List; import java.util.concurrent.atomic.AtomicLong; /** * execute task request command */ public class ExecuteTaskRequestCommand implements Serializable { /** * task id */ private String taskId; /** * attempt id */ private String attemptId; /** * application name */ private String applicationName; /** * group name */ private String groupName; /** * task name */ private String taskName; /** * connector port */ private int connectorPort; /** * description info */ private String description; /** * class name */ private String className; /** * method name */ private String methodName; /** * parameters */ private String params; /** * shard itemds */ private List shardItems; public List getShardItems() { return shardItems; } public void setShardItems(List shardItems) { this.shardItems = shardItems; } public String getParams() { return params; } public void setParams(String params) { this.params = params; } public String getTaskId() { return taskId; } public void setTaskId(String taskId) { this.taskId = taskId; } public String getApplicationName() { return applicationName; } public void setApplicationName(String applicationName) { this.applicationName = applicationName; } public String getGroupName() { return groupName; } public void setGroupName(String groupName) { this.groupName = groupName; } public String getTaskName() { return taskName; } public void setTaskName(String taskName) { this.taskName = taskName; } public int getConnectorPort() { return connectorPort; } public void setConnectorPort(int connectorPort) { this.connectorPort = connectorPort; } public String getDescription() { return description; } public void setDescription(String description) { this.description = description; } public String getClassName() { return className; } public void setClassName(String className) { this.className = className; } public String getMethodName() { return methodName; } public void setMethodName(String methodName) { this.methodName = methodName; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.EXECUTE_TASK_REQUEST); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } } \ No newline at end of file +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; import java.util.List; /** * execute task request command */ public class ExecuteTaskRequestCommand implements Serializable { /** * task id */ private String taskId; /** * attempt id */ private String attemptId; /** * application name */ private String applicationName; /** * group name */ private String groupName; /** * task name */ private String taskName; /** * connector port */ private int connectorPort; /** * description info */ private String description; /** * class name */ private String className; /** * method name */ private String methodName; /** * parameters */ private String params; /** * shard itemds */ private List shardItems; public List getShardItems() { return shardItems; } public void setShardItems(List shardItems) { this.shardItems = shardItems; } public String getParams() { return params; } public void setParams(String params) { this.params = params; } public String getTaskId() { return taskId; } public void setTaskId(String taskId) { this.taskId = taskId; } public String getApplicationName() { return applicationName; } public void setApplicationName(String applicationName) { this.applicationName = applicationName; } public String getGroupName() { return groupName; } public void setGroupName(String groupName) { this.groupName = groupName; } public String getTaskName() { return taskName; } public void setTaskName(String taskName) { this.taskName = taskName; } public int getConnectorPort() { return connectorPort; } public void setConnectorPort(int connectorPort) { this.connectorPort = connectorPort; } public String getDescription() { return description; } public void setDescription(String description) { this.description = description; } public String getClassName() { return className; } public void setClassName(String className) { this.className = className; } public String getMethodName() { return methodName; } public void setMethodName(String methodName) { this.methodName = methodName; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.EXECUTE_TASK_REQUEST); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } } \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskResponseCommand.java index 7e35fa6e75..7543fc3d0e 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskResponseCommand.java @@ -1 +1 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; import java.util.concurrent.atomic.AtomicLong; /** * execute task response command */ public class ExecuteTaskResponseCommand implements Serializable { /** * task id */ private String taskId; /** * attempt id */ private String attemptId; /** * return result */ private Object result; /** * received time */ private long receivedTime; /** * execute count */ private int executeCount; /** * execute time */ private long executeTime; public String getAttemptId() { return attemptId; } public void setAttemptId(String attemptId) { this.attemptId = attemptId; } public String getTaskId() { return taskId; } public void setTaskId(String taskId) { this.taskId = taskId; } public Object getResult() { return result; } public void setResult(Object result) { this.result = result; } public long getReceivedTime() { return receivedTime; } public void setReceivedTime(long receivedTime) { this.receivedTime = receivedTime; } public int getExecuteCount() { return executeCount; } public void setExecuteCount(int executeCount) { this.executeCount = executeCount; } public long getExecuteTime() { return executeTime; } public void setExecuteTime(long executeTime) { this.executeTime = executeTime; } public Command convert2Command(long opaque){ Command command = new Command(); command.setType(CommandType.EXECUTE_TASK_RESPONSE); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } } \ No newline at end of file +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; /** * execute task response command */ public class ExecuteTaskResponseCommand implements Serializable { /** * task id */ private String taskId; /** * attempt id */ private String attemptId; /** * return result */ private Object result; /** * received time */ private long receivedTime; /** * execute count */ private int executeCount; /** * execute time */ private long executeTime; public String getAttemptId() { return attemptId; } public void setAttemptId(String attemptId) { this.attemptId = attemptId; } public String getTaskId() { return taskId; } public void setTaskId(String taskId) { this.taskId = taskId; } public Object getResult() { return result; } public void setResult(Object result) { this.result = result; } public long getReceivedTime() { return receivedTime; } public void setReceivedTime(long receivedTime) { this.receivedTime = receivedTime; } public int getExecuteCount() { return executeCount; } public void setExecuteCount(int executeCount) { this.executeCount = executeCount; } public long getExecuteTime() { return executeTime; } public void setExecuteTime(long executeTime) { this.executeTime = executeTime; } public Command convert2Command(long opaque){ Command command = new Command(); command.setType(CommandType.EXECUTE_TASK_RESPONSE); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } } \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java index c50413e98a..c5e4d075af 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java @@ -21,7 +21,6 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import java.io.Serializable; -import java.util.concurrent.atomic.AtomicLong; /** * ping machine diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java index 4cc32ed42a..20cf8d9102 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java @@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; -import java.util.concurrent.atomic.AtomicLong; /** * get log bytes request command diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java index 621d35a804..433c4a0b1f 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java @@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; -import java.util.concurrent.atomic.AtomicLong; /** * roll view log request command diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java index 8835348ee3..b4773d0c0d 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java @@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; -import java.util.concurrent.atomic.AtomicLong; /** * view log request command diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java index f8688e7c0c..511102e4b5 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java @@ -19,12 +19,6 @@ package org.apache.dolphinscheduler.server.worker.task.sqoop; import com.alibaba.fastjson.JSONObject; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; -import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceHdfsParameter; -import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceHiveParameter; -import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceMysqlParameter; -import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetHdfsParameter; -import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetHiveParameter; -import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetMysqlParameter; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.server.worker.task.TaskProps; import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.SqoopJobGenerator; diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java index 5daf535625..01a49910df 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.service.log; -import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.remote.NettyRemotingClient; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.log.*; From d5c795bd45b490720ead48a5fe68430d4402fd5f Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Thu, 5 Mar 2020 11:10:48 +0800 Subject: [PATCH 08/59] fix ugly logger usage (#2063) * fix ugly logger usage * remove space --- .../api/controller/DataAnalysisController.java | 4 ++-- .../dolphinscheduler/api/controller/ExecutorController.java | 2 +- .../apache/dolphinscheduler/remote/NettyRemotingServer.java | 2 +- .../dolphinscheduler/remote/handler/NettyClientHandler.java | 2 +- .../dolphinscheduler/remote/handler/NettyServerHandler.java | 2 +- .../server/master/runner/MasterExecThread.java | 2 +- .../dolphinscheduler/service/process/ProcessService.java | 6 +++--- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java index 1b1dc65e01..f93e7d6944 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java @@ -103,7 +103,7 @@ public class DataAnalysisController extends BaseController{ @RequestParam(value="endDate", required=false) String endDate, @RequestParam(value="projectId", required=false, defaultValue = "0") int projectId){ try{ - logger.info("count process instance state, user:{}, start date: {}, end date:{}, project id", + logger.info("count process instance state, user:{}, start date: {}, end date:{}, project id:{}", loginUser.getUserName(), startDate, endDate, projectId); Map result = dataAnalysisService.countProcessInstanceStateByProject(loginUser, projectId, startDate, endDate); return returnDataList(result); @@ -129,7 +129,7 @@ public class DataAnalysisController extends BaseController{ public Result countDefinitionByUser(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value="projectId", required=false, defaultValue = "0") int projectId){ try{ - logger.info("count process definition , user:{}, project id", + logger.info("count process definition , user:{}, project id:{}", loginUser.getUserName(), projectId); Map result = dataAnalysisService.countDefinitionByUser(loginUser, projectId); return returnDataList(result); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java index 93b095ddf8..cae4993942 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java @@ -173,7 +173,7 @@ public class ExecutorController extends BaseController { @ResponseStatus(HttpStatus.OK) public Result startCheckProcessDefinition(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "processDefinitionId") int processDefinitionId) { - logger.info("login user {}, check process definition", loginUser.getUserName(), processDefinitionId); + logger.info("login user {}, check process definition {}", loginUser.getUserName(), processDefinitionId); try { Map result = execService.startCheckByProcessDefinedId(processDefinitionId); return returnDataList(result); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java index 29b2317633..dbeb318f2d 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java @@ -145,7 +145,7 @@ public class NettyRemotingServer { try { future = serverBootstrap.bind(serverConfig.getListenPort()).sync(); } catch (Exception e) { - logger.error("NettyRemotingServer bind fail {}, exit", e); + logger.error("NettyRemotingServer bind fail {}, exit",e.getMessage(), e); throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort())); } if (future.isSuccess()) { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java index d5d0d4df83..80e561d05c 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java @@ -107,7 +107,7 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { */ @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { - logger.error("exceptionCaught : {}", cause); + logger.error("exceptionCaught : {}",cause.getMessage(), cause); nettyRemotingClient.closeChannel(ChannelUtils.toAddress(ctx.channel())); ctx.channel().close(); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java index eabd6560de..22251c35c1 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java @@ -140,7 +140,7 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { */ @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { - logger.error("exceptionCaught : {}", cause); + logger.error("exceptionCaught : {}",cause.getMessage(), cause); ctx.channel().close(); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 4b22b27bec..0b81a30191 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -565,7 +565,7 @@ public class MasterExecThread implements Runnable { TaskInstance taskInstance = completeTaskList.get(nodeName); if(taskInstance == null){ - logger.error("task instance cannot find, please check it!", nodeName); + logger.error("task instance {} cannot find, please check it!", nodeName); return conditionTaskList; } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index b589cd4295..8bcd64f1fd 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -116,12 +116,12 @@ public class ProcessService { ProcessInstance processInstance = constructProcessInstance(command, host); //cannot construct process instance, return null; if(processInstance == null){ - logger.error("scan command, command parameter is error: %s", command.toString()); + logger.error("scan command, command parameter is error: {}", command); moveToErrorCommand(command, "process instance is null"); return null; } if(!checkThreadNum(command, validThreadNum)){ - logger.info("there is not enough thread for this command: {}",command.toString() ); + logger.info("there is not enough thread for this command: {}", command); return setWaitingThreadProcess(command, processInstance); } processInstance.setCommandType(command.getCommandType()); @@ -991,7 +991,7 @@ public class ProcessService { return insertQueueResult; }catch (Exception e){ logger.error("submit task to queue Exception: ", e); - logger.error("task queue error : %s", JSONUtils.toJson(taskInstance)); + logger.error("task queue error : {}", JSONUtils.toJson(taskInstance)); return false; } } From 18d7ff274b8757b2e37f42790473ca472cf66274 Mon Sep 17 00:00:00 2001 From: "xiaochun.liu" Date: Thu, 5 Mar 2020 11:33:24 +0800 Subject: [PATCH 09/59] action push delete --- .github/workflows/ci_e2e.yml | 2 +- .github/workflows/ci_ut.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml index 5282681fb6..7d9c5e3e72 100644 --- a/.github/workflows/ci_e2e.yml +++ b/.github/workflows/ci_e2e.yml @@ -15,7 +15,7 @@ # limitations under the License. # -on: ["push", "pull_request"] +on: ["pull_request"] env: DOCKER_DIR: ./docker LOG_DIR: /tmp/dolphinscheduler diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 45ff3ca29d..e7e47437d4 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -15,7 +15,7 @@ # limitations under the License. # -on: ["push", "pull_request"] +on: ["pull_request"] env: DOCKER_DIR: ./docker LOG_DIR: /tmp/dolphinscheduler From 6ec85540bf6d367fa11ede093325c199afc930b5 Mon Sep 17 00:00:00 2001 From: dailidong Date: Thu, 5 Mar 2020 16:36:20 +0800 Subject: [PATCH 10/59] fix typo (#2093) --- .../apache/dolphinscheduler/server/worker/task/TaskProps.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskProps.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskProps.java index 8e5644ed9c..edec419384 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskProps.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskProps.java @@ -78,7 +78,7 @@ public class TaskProps { private Map definedParams; /** - * task path + * task app id */ private String taskAppId; From 4230d5a737b7889254806969ae153817fd19d181 Mon Sep 17 00:00:00 2001 From: dailidong Date: Thu, 5 Mar 2020 17:32:40 +0800 Subject: [PATCH 11/59] add embedded database h2 and update embedded zookeeper (#2082) add embedded database h2 and update embedded zookeeper --- dolphinscheduler-dao/pom.xml | 19 +++- .../src/main/resources/application.properties | 4 + .../server/zk/StandaloneZKServerForTest.java | 100 ------------------ dolphinscheduler-service/pom.xml | 5 + .../service/zk}/ZKServer.java | 87 ++++++++------- .../src/main/resources/quartz.properties | 4 + .../service/zk/ZKServerTest.java | 41 +++++++ .../test/java/queue/BaseTaskQueueTest.java | 1 + pom.xml | 14 ++- sql/h2.mv.db | Bin 0 -> 110592 bytes 10 files changed, 121 insertions(+), 154 deletions(-) delete mode 100644 dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/zk/StandaloneZKServerForTest.java rename dolphinscheduler-service/src/{test/java/queue => main/java/org/apache/dolphinscheduler/service/zk}/ZKServer.java (67%) create mode 100644 dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java create mode 100644 sql/h2.mv.db diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml index 20d19410e2..fd43458f87 100644 --- a/dolphinscheduler-dao/pom.xml +++ b/dolphinscheduler-dao/pom.xml @@ -44,6 +44,12 @@ com.baomidou mybatis-plus-boot-starter ${mybatis-plus.version} + + + org.apache.logging.log4j + log4j-to-slf4j + + org.postgresql @@ -71,6 +77,14 @@ log4j-api org.apache.logging.log4j + + org.springframework.boot + spring-boot-starter-tomcat + + + org.apache.logging.log4j + log4j-to-slf4j + @@ -78,7 +92,10 @@ mysql mysql-connector-java - + + com.h2database + h2 + com.alibaba druid diff --git a/dolphinscheduler-dao/src/main/resources/application.properties b/dolphinscheduler-dao/src/main/resources/application.properties index 34bb9f916b..06b0ee94d5 100644 --- a/dolphinscheduler-dao/src/main/resources/application.properties +++ b/dolphinscheduler-dao/src/main/resources/application.properties @@ -23,6 +23,10 @@ spring.datasource.url=jdbc:postgresql://localhost:5432/dolphinscheduler # mysql #spring.datasource.driver-class-name=com.mysql.jdbc.Driver #spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 +# h2 +#spring.datasource.driver-class-name=org.h2.Driver +#spring.datasource.url=jdbc:h2:file:../sql/h2;AUTO_SERVER=TRUE + spring.datasource.username=test spring.datasource.password=test diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/zk/StandaloneZKServerForTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/zk/StandaloneZKServerForTest.java deleted file mode 100644 index 679862f102..0000000000 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/zk/StandaloneZKServerForTest.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.server.zk; - -import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors; -import org.apache.zookeeper.server.ServerConfig; -import org.apache.zookeeper.server.ZooKeeperServerMain; -import org.apache.zookeeper.server.quorum.QuorumPeerConfig; -import org.junit.Before; -import org.junit.Ignore; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.util.Properties; - - -/** - * just for test - */ -@Ignore -public class StandaloneZKServerForTest { - - private static final Logger logger = LoggerFactory.getLogger(StandaloneZKServerForTest.class); - - private static volatile ZooKeeperServerMain zkServer = null; - - - @Before - public void before() { - logger.info("standalone zookeeper server for test service start "); - - ThreadPoolExecutors.getInstance().execute(new Runnable() { - @Override - public void run() { - - //delete zk data dir ? - File zkFile = new File(System.getProperty("java.io.tmpdir"), "zookeeper"); - - startStandaloneServer("2000", zkFile.getAbsolutePath(), "2181", "10", "5"); - } - }); - - } - - - /** - * start zk server - * @param tickTime zookeeper ticktime - * @param dataDir zookeeper data dir - * @param clientPort zookeeper client port - * @param initLimit zookeeper init limit - * @param syncLimit zookeeper sync limit - */ - private void startStandaloneServer(String tickTime, String dataDir, String clientPort, String initLimit, String syncLimit) { - Properties props = new Properties(); - props.setProperty("tickTime", tickTime); - props.setProperty("dataDir", dataDir); - props.setProperty("clientPort", clientPort); - props.setProperty("initLimit", initLimit); - props.setProperty("syncLimit", syncLimit); - - QuorumPeerConfig quorumConfig = new QuorumPeerConfig(); - try { - quorumConfig.parseProperties(props); - - if(zkServer == null ){ - - synchronized (StandaloneZKServerForTest.class){ - if(zkServer == null ){ - zkServer = new ZooKeeperServerMain(); - final ServerConfig config = new ServerConfig(); - config.readFrom(quorumConfig); - zkServer.runFromConfig(config); - } - } - - } - - } catch (Exception e) { - logger.error("start standalone server fail！", e); - } - } - - -} \ No newline at end of file diff --git a/dolphinscheduler-service/pom.xml b/dolphinscheduler-service/pom.xml index 7d775d5497..03bb94dc29 100644 --- a/dolphinscheduler-service/pom.xml +++ b/dolphinscheduler-service/pom.xml @@ -48,6 +48,11 @@ + + org.apache.logging.log4j + log4j-core + + org.quartz-scheduler quartz-jobs diff --git a/dolphinscheduler-service/src/test/java/queue/ZKServer.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java similarity index 67% rename from dolphinscheduler-service/src/test/java/queue/ZKServer.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java index 3d503d5691..96331405d4 100644 --- a/dolphinscheduler-service/src/test/java/queue/ZKServer.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java @@ -14,11 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package queue; +package org.apache.dolphinscheduler.service.zk; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.zookeeper.server.ZooKeeperServer; import org.apache.zookeeper.server.ZooKeeperServerMain; import org.apache.zookeeper.server.quorum.QuorumPeerConfig; import org.slf4j.Logger; @@ -26,27 +24,45 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; /** - * just for test + * just speed experience version + * embedded zookeeper service */ public class ZKServer { - private static final Logger logger = LoggerFactory.getLogger(ZKServer.class); private static volatile PublicZooKeeperServerMain zkServer = null; public static final int DEFAULT_ZK_TEST_PORT = 2181; - public static final String DEFAULT_ZK_STR = "localhost:" + DEFAULT_ZK_TEST_PORT; - private static String dataDir = null; private static final AtomicBoolean isStarted = new AtomicBoolean(false); + public static void main(String[] args) { + if(!isStarted()){ + ZKServer.start(); + + /** + * register hooks, which are called before the process exits + */ + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + @Override + public void run() { + stop(); + } + })); + }else{ + logger.info("zk server aleady started"); + } + } + + /** + * start service + */ public static void start() { try { startLocalZkServer(DEFAULT_ZK_TEST_PORT); @@ -79,7 +95,8 @@ public class ZKServer { * @param port The port to listen on */ public static void startLocalZkServer(final int port) { - startLocalZkServer(port, org.apache.commons.io.FileUtils.getTempDirectoryPath() + File.separator + "test-" + System.currentTimeMillis()); + + startLocalZkServer(port, System.getProperty("user.dir") +"/zookeeper_data", ZooKeeperServer.DEFAULT_TICK_TIME,"20"); } /** @@ -87,48 +104,28 @@ public class ZKServer { * * @param port The port to listen on * @param dataDirPath The path for the Zk data directory + * @param tickTime zk tick time + * @param maxClientCnxns zk max client connections */ - private static synchronized void startLocalZkServer(final int port, final String dataDirPath) { + private static synchronized void startLocalZkServer(final int port, final String dataDirPath,final int tickTime,String maxClientCnxns) { if (zkServer != null) { throw new RuntimeException("Zookeeper server is already started!"); } - try { - zkServer = new PublicZooKeeperServerMain(); - logger.info("Zookeeper data path : {} ", dataDirPath); - dataDir = dataDirPath; - final String[] args = new String[]{Integer.toString(port), dataDirPath}; - Thread init = new Thread(new Runnable() { - @Override - public void run() { - try { - zkServer.initializeAndRun(args); - } catch (QuorumPeerConfig.ConfigException e) { - logger.warn("Caught exception while starting ZK", e); - } catch (IOException e) { - logger.warn("Caught exception while starting ZK", e); - } - } - }, "init-zk-thread"); - init.start(); - } catch (Exception e) { - logger.warn("Caught exception while starting ZK", e); - throw new RuntimeException(e); - } - - CuratorFramework zkClient = CuratorFrameworkFactory.builder() - .connectString(DEFAULT_ZK_STR) - .retryPolicy(new ExponentialBackoffRetry(10,100)) - .sessionTimeoutMs(1000 * 30) - .connectionTimeoutMs(1000 * 30) - .build(); + zkServer = new PublicZooKeeperServerMain(); + logger.info("Zookeeper data path : {} ", dataDirPath); + dataDir = dataDirPath; + final String[] args = new String[]{Integer.toString(port), dataDirPath, Integer.toString(tickTime), maxClientCnxns}; try { - zkClient.blockUntilConnected(10, TimeUnit.SECONDS); - zkClient.close(); - } catch (InterruptedException ignore) { + logger.info("Zookeeper server started "); + isStarted.compareAndSet(false, true); + + zkServer.initializeAndRun(args); + } catch (QuorumPeerConfig.ConfigException e) { + logger.warn("Caught exception while starting ZK", e); + } catch (IOException e) { + logger.warn("Caught exception while starting ZK", e); } - isStarted.compareAndSet(false, true); - logger.info("zk server started"); } /** diff --git a/dolphinscheduler-service/src/main/resources/quartz.properties b/dolphinscheduler-service/src/main/resources/quartz.properties index 9c8930b647..b01be394c6 100644 --- a/dolphinscheduler-service/src/main/resources/quartz.properties +++ b/dolphinscheduler-service/src/main/resources/quartz.properties @@ -26,6 +26,10 @@ org.quartz.dataSource.myDs.URL = jdbc:postgresql://localhost:5432/dolphinschedul # mysql #org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver #org.quartz.dataSource.myDs.URL = jdbc:mysql://localhost:3306/dolphinscheduler?characterEncoding=utf8 +#h2 +#org.quartz.dataSource.myDs.driver=org.h2.Driver +#org.quartz.dataSource.myDs.URL=jdbc:h2:file:/Users/stone/work/myworkspace/incubator-dolphinscheduler/h2;AUTO_SERVER=TRUE + org.quartz.dataSource.myDs.user = test org.quartz.dataSource.myDs.password = test diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java new file mode 100644 index 0000000000..48cde32287 --- /dev/null +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.service.zk; + +import org.junit.Ignore; +import org.junit.Test; + +import static org.junit.Assert.*; + +@Ignore +public class ZKServerTest { + + @Test + public void start() { + //ZKServer is a process, can't unit test + } + + @Test + public void isStarted() { + + } + + @Test + public void stop() { + ZKServer.stop(); + } +} \ No newline at end of file diff --git a/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java b/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java index a0cc457e22..97ab9969a3 100644 --- a/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java +++ b/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java @@ -18,6 +18,7 @@ package queue; import org.apache.dolphinscheduler.service.queue.ITaskQueue; import org.apache.dolphinscheduler.service.queue.TaskQueueFactory; +import org.apache.dolphinscheduler.service.zk.ZKServer; import org.junit.*; /** diff --git a/pom.xml b/pom.xml index 8a1e9b98f2..57063a86c2 100644 --- a/pom.xml +++ b/pom.xml @@ -72,7 +72,7 @@ 5.0.5 1.2.61 1.1.14 - 1.3.163 + 1.4.200 1.6 1.1.1 4.4.1 @@ -200,13 +200,6 @@ ${spring.version} test - - com.h2database - h2 - ${h2.version} - test - - org.apache.dolphinscheduler dolphinscheduler-server @@ -349,6 +342,11 @@ ${mysql.connector.version} test + + com.h2database + h2 + ${h2.version} + org.slf4j slf4j-api diff --git a/sql/h2.mv.db b/sql/h2.mv.db new file mode 100644 index 0000000000000000000000000000000000000000..42ea540bd37b54496241f1669d3455f3b9806cf4 GIT binary patch literal 110592 zcmeIbdz2hkeJ9>snrr-$(ZmKDzvyXO9?2e!x*q)+4E8*Fq?YETo*7xPWZFI5)zaY6 zj64rpLMX^@gTcl;Y-|+A7#r9uXE~gcO*Y|`B{^9qyGb}Wu+V7&#_TVTBf)>~k`1=d?&y#>}=V7&#_TVTBf)?45g)&kBX zza^Fd7<~&io$5bQJ35dr;BjmqSI_h<+Q(M0MBr%cki7zz!Sd45>VQ=*Vx0i~nVHO7 z-|GAkdmvXZjYdB2T?=6K9i3ksURqmR9mu54^*_3-;E_w~iSv?)u0O&1o^K|)UOtuR zUHd3|ZrQV!J-0shb@sgRiVAa)$NH@z|5~lhE!h2f;Sv_Mw%Ax2TRPNlBobYBy!jo8%M%x}2JOch2*Z6fS$;ap zT+i@__%isH$sfFYJJ>GS1KSEzI9zkAv;?92SAlk15Xdx)&4mIzS=NU`sd@3IKzVZ8TLl=zVL8Yb$nR zaCNV{M8xo7#PDF`1(!0hBxOLa2~bMXymV!}WKpR4Fj}YwVl*d+av&Q(qv@p);CFER zCd&PRrK9%Z%yMmUrB+|X!k3k6*slU1STw^H%g8F5Jd~qJ&jp7{bOncsw8h|1p4O0y zbOOsYjL5WBa40XAE&~dxktjDG94gWlfkZ77q$$GJ+Q6xy@{aPvOlo2h|DPQj>${Nl zQ|Z#gNOh(>UP_B_@$>ND7(xas)rmnGHoSANGB`X_s-*S~R`!=Cc5E>+nXMx3czJrX zTq#w{6Vo%5bZUBhaBK`EiO^k>L)DSNnL!k=f2K5e;nwc1-c--Vdxk5e!I@HOs!|?D zmZ{yP{i*cS?9fU)l8=*Cp=+$Dl@lNXDa0#J4%)5^p^DW@Xpc*r?W4W z76C2aJ1Ud2Q)v{wCAE3uOUc18TA7)_p|MhE)~Ugfk<{=ct<@lHIhQ>5M@pTi3LR3D zDo>1*_IWZ^>)qY8HI>@Cg+S2yDk0EHcX#yxfuE0qz~Rc|gyaFl@->e?MsI1~RHZaM zU7nl>(s6I(n?Gm+R;a zvtcx?%-p%Ppt+68m+Z@J0M!1U^BPsm)?*F;eNI|ej?GuJ0&Tvc*L2EyOs8{9KWzT; zrJvhCsQrIyA~Ez%_9P3Z+4F)6zQvv!f5e}c&hzKx-~aX%^xt;!7NV0fsgdJ}#AW~H zyoWYik$FxX$+{!qpS-r?JG~uOsCS*81+qKsFmui)fz_8 zwzEYuZ{>>lMsI58;PlQ^ZzfYKwMVF(@P z4S@ekBnJFl!|63YLUY<8pmPf{LtMtiOb}nlCQAEes?5j@u$3`{+#3;Co~)E-0Cse- zC77(DGnIDM^b7HwGE>0~%B({uzl$E^RAv;R|R$3Xzk zL4boGm8Ld`<3bPyi5MxlLDcQQZ42sR>)HgH;JvYJ0u%1K)_DY~aaBK(&oYp#mezs% zb}WRhZkPz4k1`Qj%7{0x0KE~0f}|sau1JFcO@Ze7HJA+Fm)3jK+``ZpEFw|fS=h`9 z#jmDjXY~4}oVqUU$RRrv=#cd_M8`>B--VC-Mj;$kxMuT*OEC3brv0R35dZ|vj94!8LlroS> zrql7*3yLDxD{~VBL`pp>2Li7EJD-ok&Z$9o^e{KfCtBeVnK}!`y1CF<*mlBy;nqar zlC|@{3r8hlWuhxk&|H^jGqM&SJ!2m=?puk5-Bnt8r1XlqB|257u3w3!o7&r_5 z!+luUA2MTB%(l2KWQAV!F&SO5FgP(>>PX>;M)53~aCD9svhZnludB!(1Kd>ILjDMy z_=Vaz$^=zy{bM2?e8a(xUlAVe$Ym<`YKY&V*eW+ry{rWYE0i=3sL)V9DU0$GdotQR5|OrP* zO%Crq3yn*YhiWy;GaE2tFF7qdixd=gMwD*g)&JN#Zyu&+=QYYmI^4Mwt zOOvpWrSpnFH+M*R(2qjp0@$cR zM`uv!ju0$coHsC9JfQULDE&l?CP(RFX>Ug)sLO$aNyvbNC*pvRwZN-FMk^Wo76~YbZ{1{*QgG~^7AddtJTIzwN|g&D=XF2rMv9K>U^WRV&A=HFV^ke7B>c6 z633h#N@nif3tlMQQbAX5Zpm3&-6GvsSffmLS8$_@{)KKIp9+e}5gaF`Sg#e8C$kHR z;95lphG(?xSu!DhWji(WtWQJ(S~!iyrFa;CDR6&?DbSiMo<)G-Jw$F{MMaPPvvm0X zHeluDHcJONNwCgmVuEDBUS5qx$4q&KuD)x-{pp>Pw^jLqpAZybh29KXo;L%S3KtD_ z@6dHWY@JX@G9HDF6K3Iy_D1T>^onf8CLxXfY?+Wn!5ZuPN@XnYL$;o@&ac&B49YzL z=N`2;1GT$iqqZ0bTFz>T*0%$#$Ak9_d5{qcr6{g5kQOOM-vNyNaV&jC3FmrHc#zs? zGYZs0%S&rVqtVK_&Kj zXzhbZch>I$J>`iqUFIC6QDl}=*o8Q*D36)J>D_R;OtEE*QR;3K3%+O3jnd70Scl2i zV>;k4##_p(vO5TtP|wlrc-fno=! zFqV=y3`sl@p|K_IV@V;3>`Vv;sSJ)l27eSsXVbL`)049ma4VM1Zq!z5D@$w3bvs%T zRc%d&m%h!v_*4LbCC35k+*Rbhq8D1k-3cweCq23sAJ^&JcwZfYc5^>gR1HZkarkDivr5334B*e61_BtIOvA z5X8|sQa`LKb-BAfwo;FQaG`{~s@*e+mTi9yb+afC2IutP4gi*x#>8wV#4Ns5SO}6b zA3m9&B@%B=7|j1xCcDUPGDgP91eqjLWDlv3X);4*$zF0B*+=%1+sOfPklaDuNvh;d z>VGRkMDL3ld8sAFl2TCKnUEH!jP8Pr{xpu-AIF?4o$kdvYN--W^B<|LtlGa2z;H?C6yl}8Hfp%+Mkn2fMmD%R z2~qh0I)_b4?<`?lG(?XIXzQ)m-4QPR>p_{`^zlS?0AuaYrKXF8F8}&Y>KKXiRk6=FERgC!*0$m&eJo0qH(4UGQtL#)E&U~GI|{i9;{VL*n&7s zhv#fNR7fn;X&3MqWhy9Rk^oG#HcNHaG~NF1(WQ!U>iK;K7Qzl84YYm8x*l!#nps`13?q}rVDy#GWtL~Le;YjX)cDF&@QM615<)uq!sSV4-xG}cy5 z&koU@F4X;{;=^t17Guk2v31a+?8He77{67A$1&dU^kuFYQB}NDYJb$y5>R+&>G2T_ zaSl2pW{XsyP4~v`J_ZAwNB&y6oyRmfkB8%Q9ug|?R`pXdI_o-+0$s-pDl1Px2+*=`gXr9XHxhgv)0G!Cw<=}fQ?s9?3@`14wlG3mJtd>xN1 zJ-M*K7iWkZZtR25oQ!b2N)E;vWzjM_6BVR@sO(40c84HVa3>P3qDqpHbR>B@Ao^q+ zh~k`+*_f_eEFD;DGhkx5i3KoTnKZ?U|7_ZF0W9Q#+M}Cz8C!UU4 zwijy5%=WnTaguNw_l^W68)5o}2d+0dYS27Ys$ee7J3w{EFy|g3?*v5dh^=V?5fa|a z5;RqS<^xgPb!Zl`bWz7RVlcWBF#1$SFlrr-@UwdS=DE5q$-$BI;KZ}XU^rm)h#ZkZ zC!%AfJEB?&bEh>$5;pf5E54ZQjf_h-{dNCja+$8btdTiVCkzkau-=3 zN5~>sB1g&HWSOjxRkB9zA@`DF)c;!pzT)p-7!92IanRAq0Ew_UAnb`)2y4kB7Q`rF zo#`s1K=tbY*%vzn8S@mJfuaJy*r@94w3HYIL;~#>s-hZz(1F-0s_5Uj#yluq2zi=U z!p8=D9*YYfH+?MVP=Y#x4pIO*O;kI~9~D>~a4dZ26qo`QhU~BCK4xNfmgKBbF0Tyg2e26Y|Q#{h=tmY z^&L}p?*f9p&=G=WM@G*GJZp{S+2RI8zjl<|T|i8HcYDddHB(h`o<@~uSpZrh-R+Ko za1+M^ihm;}b#7z$8h(H}h=;N{0;qhhGpNvI+iZEPSl!pr=!@x?TFh0IZYfa+k-AeE ztb5F~b3a;N1WMi+yDKaO`UPA@``c4uWeHdb+YagjnlFoZi0RmMfQkSyM}e3Z;;aXE z^j%y-PxeERy9Nfw%7gI2wlVcb%Q)^|^<0HG1%e${A(Qj%pcc7CXu5nm)Va84BETzl z?s0lIa5@nir~b{L0%Kxm;d*_Sfy*cdmdp28I7G3YzBz{bVVI(D1t|PhoTXmUeeY}* zZyB8^+gDcVhwaANg1r(wO()G4clm&$p?uy3k0UqbPK)^>1$CCU9s@<5V2m?RG+ z$-_zVNRm97B=1X-_b16?N%Dat`CyVfo+M8s$%m5U$t3x3l6)jdKAI#SOOmIOP0#Zy!_!C zgg8XuMe%*b>;AE3`>tv{pPbsShpnk%H1G~tf-PhSAQe%7GWUW_pj*av)K&!284WE9 zsL&t~vPUgq0(`t2y_`=_65O=}kQ~VoFq?1E3em$1xhW~VF0VKaRY;g~V` zhP&38KTpe4N^RF+OhqtbaB8ZGo40m*s1Dho)k=ly`{zV}cjA1X5H8YHBcb$q>ZC{t z+|1{w2W(j);bk||n(v0z#3c$bZy6iP1odnR@$LZe1a#*EQEognT`Uz!3Fu6PA_Y{| zd!VxZHjaJE*Z9TMRrX?SvCXu(STi_0IRa5{qku$QRj-98tB&jyJ~Q9KEwM8oAq9Z+UI6Kl zI0grG+{A+sbDl*SyHP{+(b2b;>T_Lt}a`a8bw#fSxocoiU}4LML1mprBfY;N6#D9LwNT5qV4XDCMsp#1(MVE$j?cy}l?W5PV%Y`x075behVVhY#!VCy7S z>`CNA#dd)m0IkwEHomsaXKE*Iv~y$Wdf*Z$p~Zhu+E(c0Hj7%APJV!5RWA4Oz#?QK zWW)>xE91BX>w^@mQA(KsBqmtT0IdwJ4^g;&ERF>$cUX8VSY_m0lD#W>DE)kR@jwaV0<@NOdu zix|@D1n8rXp2r)LA?y9L_2UoUi=%KZdea%HD|io!&_mUhU84E`fNv8 z$8JR`WuKr~%gN2QRLka!B$;M=nr17{K5Cn-v^@FRnMiE@2F>R)0k1`5eu2 zPa6QZdD3YaWl2Q{=g^Pj%uG5 zXm;%swPV$VH`b?UK5uW+Sd61l?erqerJXcK~t|XREUvhreLh@%DF6uf?ezxIO1l&CTgRTt;I$4IPvFDQ_yqkx! zua`Gm+O>zgyx~^yby?Rq{X*I+4|mZ_zavv!#wDaGtJO5)LFo7RZtf8Y*$FH=&= zi#Uzsp4visK*uj@i=zv*Lx?ajSs5Q3LnK;(ve*?qwLHJHJimGz;p3%|^6a<>r`1_n zTV*8J$Syr_B8}OHxdl{BwBEc^Oy(pq(l%e7Ve&~aArgK0|o>hYtr znWlG^#wbzTB-%lbJS?M?x9KO>xews{3UmKu$o`>)r8y}GTuhnt;q`Ge>Ofh4ck$nn=KdydHTa&IQ9O1@@Oy8n$LdjDy+@i5G&#NNRUWO zJp1s~T0%Dx-C9ekDYiCIT>B6ZUXgc6aBnzrn|fyA8ZC$bB`ZCELSJ46BaZAl{ZHlv zpw6_iId+NZjzssxJqJc>Fga7JOV~L?57A29TU%b7Up$oBbmxH+z0`#19q1ir$3JeH zX2(DF_VpfKT3N-628K=(``f<(0pA`mOFu_r>X0RNcmA9bY}F{`}v4oKwYKx-Y&ioT~k9TX9#* ztI$ZF{8K>=d+EOT`fzf7fvo$7pFhVNiM@1Ry!kaj)o=anS2$JdrTgL=Vo~*K1L_Y? zf0a|kUb-*tibvL~Dt+Jk&@B&r>&sUsE|M=GtM`}RTA=3CKM1ne3&`5>njq_okN-)E zwHAAUYOS7r#!4piME7|;a}D0f$h}U>jy7dWBT{X`FZ%iWZ}}fT{dRZ4c|rEiJn%0? zmi?tYJE_Ml)zj0n>F2M!^2!^>pNG}C3Es<16Yv#ye0K0;sg8lw+VUYAf8gYzX8a<2 zii>MU=BU$)+1bb7NVD>NX*kL9xsxY1X$S7ul-R^H^3Q+khu0=jLrJQp*?YOxI5NKo z!Ztnd{|wL$vZqI<9Zd2|_bu;FZmY}WqlRg?0}PhaiAi- zlp&EGN}>t6Q)zF#3`4gtbSSWT0k7Zq$rrC>!1#;jzjSRxy%6;W|Mbe0PQ6ZRzvOy> zd2hb&qc_vB26%}-X+B~|Tz}t}-i}{2hif|CWlcq#E%*KXI~Y9Qz~MR4(?g?w@7v!; zW5^ej@WxZu-2${|94v{K=W0eXJ6GIhnuY8(tC`JiD>RDDZ8a-znN~BGohukUzZycx>O>nkPocC2D5EndF%k9bjJVB?M3SC1Us?pGz-?PGR*yEtQKyUzc^RS#W{y`+smVt~<}?#t6m zNBByA0<#RP_`i{&_xHaoWbjM>2`^__ys>e|_F0VfR<^IKE-l*IsRwQQk>m8Sar zx6d!u*XC-gOUv6@YO}C(Xoc0m=*KIn@%2w@z>cWKb>3>ExdJ%;9aatS>ZMm8K9p(y z_kE|XVlNLrjh8d4hOyl|`)VK)QH?MBm4_SG2UX)qjdBCj|Nc*f!Z>{j3WNUg;*)qe z(`s1fq8gT{#y|hi;Rb}F|Dit{7eZcUtmc15n5aMUzrV{d&0Y}u&WFA*$zosF!#j(O zseG+rSbAp8Xj=75-m>z=TA^s>Yeu1F7;}YYYU0?jW4GRYcRz0W89jEa=c0dp^p>yx z^@qV+$jX2I(f6)RNbP|9FMH6QV)<)5Wxe0%&#>S2@1bJLe_$O_;*kfxl{k;j{oRoG zCY$@~-FX>%Zn=(sZ@qhpeQ*7{zhuuF@A*1^{`Fs8amD7VQdeAc-jUkTe#2r9bB@rj z=0@7P_RnK3?O4B6Q=9bx3fj^4;QR9y}{;2augFkHkXtIaF%QJX+ z1~1PrKb2%O4^W!><}iV9*Fdvx!CoAg>sy$=#~!a8!(VtCnPS~$5scBA#YAeX^|1yLa8Jzc57N5O5a4$JFtYAnHhKkR%&&6 zK;X(X>{o&CY+rrZ##E$v0H0ZhDP$?b3p(1TDIFzR~^x#mQ761*TN~E=dLq*z< z8i};|$h3vvP@dMz1c&l+&5#<2v}R=5$Qqd;ToC0(*2oO0kw_cDBOYo+*4JvOZz8c_ z>4Wcra^zZwYc_*xHiK(6gKIW}Yc_*xHiK(6gKIW}Yc_*xHiK(6qsjCs*O?~QZzk7o zCf9GK2otKEmum7-Oc_Dj-55 z%gu!}I!}wvPgPu^X+&s9eMM*_vkNT@!&oHuJEs$yxwx5JdQ2`oCYK(QOOMH=hl_^A zrN`pZV{z%Rxb#>e3YQ*>OOM5+$Kujsap|$R%v!uuiP|vi$od@&8dZwjhHMpf{H8`Yf zk+~jSw%*e6O6Gctu<>9~F+IBMmNABn8sln@{`ObJqD#28UV*BYCZtXc@qkQVZGL&3X0?nM) zh}6|DC81$>V;WiItKj`Ib>+)azLXNoyb`A7Qe@q)G;h5Tt&H-e6lW=9TBZ_HA;nUv zU@3%I3Sm}(t2EhyKJhb?4b8GNYcAQ;vbv_@f2O8qH9fECMNOa6^oFL}n%>lmjArP` zqGlMHVQPk@8ClK9X+~Z%3Yvl6Yl(!G`JJyMQVE4_g%Me$_F1L&S*7+_rS@5+_Svkr zd6nuGlmZG$vVziR1;HaBvrLh|aYH*0=N(FgS#$kfeh2uq(r&QVy2;pIdV5j)Xd$}Os!h=kK0`aAUWreWtu&h7<|5Ac-0*@j{fJb~O zROS^b^9q%D0Z*Q;pu{ZXyfsjqU`3?{MWqHsh0LOWl+)d4x}oWorsp)hkSu6=P1C_9 zLo>}JIA&>PRx@*&nb*vMW)?LQoV1hRP)4(K%`!C0)be@A^7QW~p|LrRWah>js2i$Z zuZicJc-F}n~zaUGu<_g}gx&dhwVY*$bqhD}h)^iz8 zx`v^A)e0`1mR+~px?9a6G_lMjJEs@CUyY2DPPc7T-;LSGW*l1JpscxHdNX4*>WImm zmzJQKeK}OiFiy4j5>(4CPPOh&@L`_kGOjKCcCbyj#?6%TE;ueo5aVma9IfMMCWW;WAcTXoLap9>vFICJ3Y zbJBzBIK=ZzezOg`E*#?7LLpbrngQNrz2WgLNU%-a_8Fqm*uxCpF87) zav_W0&Y8>6GuYBph-9WI0^5s*B#Lzq0lPsKO&kHX27JF{|5r1Af&!3ln|Jzs4 zf7{7hh)&A5RP#92Z2p_`?#B(AeD_(NoKG$w7vdtD9XmS#F;!FfYnV6gy z#{Xt3^orQ1h~iXFdURT7vk2e7RFA>o|As3f0dd2qEv%tRGAlnK4Wtxs;z+p;EnRyk3jNY_d9W7T% zRen7HKZRFF;8dlwHzLq;;YDdjsUi}|7Gc|w(*ZTqb1hm&rr`&Ew+ecpB?aG2<)zs# zj3r&-g`vmq-`_&ECsw~)+u^pnsFQkZ)o%=g3=X8%#$ zJ{bE1NVX9}C!|?^I}*JG68%z?M2i&D=~&|IKiFH0hngUW1@vqR#Q|x8dN-M-Bk2}f`-UfHS=Um=O;|I2aEdqx5qHsu z)31HKF}tio*1d!ya(&Tg2XZ4YcQQ8S{MVcXw{fS?y9vvL3vM4mnSCzRdA4e;9{8S?YJ!gr#A+rDP*=% zxNQU6-V+~g{%dv_-h3_EDcbtmp-s&{9@hMq@Sa1k(c5~r18pz1xr4@$#meN=>j`k^ z9C7^>XvVd)t$lY^H{^)kiD<-)h)yUnM<`IIXt!r}enmO%uVa%FxY10|sf-ES5F}7S zMoPm@l8M>zQl&h+MK}9WmaLK!0Vx9`P#eU_LKq}s0O1DFV!5rtt_au!8GCPR88acS ztwsUWxGIi16|=5x$l2%HX?3NHSfWPhjW85spL-^H9a-bR+FygLeP3GdQ6|IC7_8UV z(a3_)>l<=*UD}aC9tu>*zJ}-+DVMd=$Z7$xU_vQRBePkQyC)-H|{>rYmfamdjMgIHw z*zh8-@;>IPb)w9_63mZu76oM+OUnIM`UMZWgpC`C0(RBkRRF<%jWf)mcD;jvv*176 zhn4*yBjzG*eeoM~mGTa_N;*;lDmwzkLqV3SbDah2Mq?%tjl%AU^jVr>$imqZ<#vHg zyElPTw~)UHI`Ip&bCh{JF_8i&;=wl@?D!So;f`FUa<7K?9g3}T12wK6c0S2-&_spq z8VKD}Q5x5mSuA;r*1B%_k)p=*?X?iKAIH(R3^TL1G^fg0f5sl1Gr}~xIca!jX?VAD zA$c&gWgHj`!9E#=fwuw!M`B|jaIn?V9zGDL)uc#B0STw0kl@cU78Ih#S3W8t@xX>t zI8tll2jk#jY;t(_S!i6EJXC*Lo*6C1FC6u?u^O9SFko9PDM7D~OD-9dL)U>rk4JHc zXA+AqAR=O5D}C_IT#5T3djsVP|I1*AA^ZleWiZ zH2a;R*&_!SYsgYBw?;&yag zo*3bvN?SaLji{3URzSwe&?@!xNQsP+ z9b_jpI(5|O#g1!4&FXX7xSm=?fZhPHEA0?~rl)Aa6}QtfVkFUudf(vM?;!CFkI+sg zivB>`c0$>#SlY+1Zg+SuT`zK$<)d+e`(X=29gWc*yQ+bTzD*;9ca&L_+g9zPErNT> z7{$8%We!EP)w7i4XH-S0c7hc@Re5=oFRj8*l}F1zlNOf6yVWgVnEjPD)0Qolp2oXU zx5n-RtFA8HWiLiwpe^pQW3wwM?+wSoke->`UBbj;n>*eyN9is+VN=~RgX2>{`Edk` zB`T3Il4v{7M9oJ4!{(+Kg1MhO%~50K=B?TV_ftNf&tC9 z0V^-JSvtr`qF36+1PPAuUyVk`OnHW`zH7t%>7A3eRr!J+nB0L}pM)t+*HFx0j3SW5 zF2RgW#-q@I*era}-n@HpiNJ+)1!{V164L0;mI)d2<#`iL%#MvY{-NPgnO?xCq{XT~ z{%ds@gK|%%=N`2;1GT$^I`;q~1hs*8(zDiqzaNwQRC~Q0Xg!XFtnky#&_$D;tUSnw zh0>01qi2`VzXKTk<5>EPVwCGa;X$J76ckP=SH{;UP!G||@S@Smxz63B60$}%_`+@; zHxjhU`qp;&oqL4b0))sraN8qf)IUey@?5mE#L2C|$*D-K=CUUiT4>Q80%PbjqyS(C zftMHCtOsSa5LaTy(`q;s_(*NB5segi$#x|F#R2c|mw4`|_TkjkC^E|_tj!)*ln1*D zcLb-zPhgs2{}lA80(CcwQ7rhL#rjPwt{9vt?Z9Bt8_`vHw+&Y4QS$+rwZ`E%Z;>yz zQ0Z`t(Hocd5ABP}~W3lx$R+uYxfC^(NiNlb@6A>C);y#uXqR7sKaFEL22xRa_ zadftO<9IwVY~Xyjm8G@ix*aWvsu zsh_u*xMlKCs#KsMB*=Xu0blD%?HZ{+<7geJAJ&z+916!)>M;;5l(1K|dq&Z+?a!fZ z76p!Df$ZQ80G5`<#B3+TEWTA(2$C`%KAE85`jdpg{BLEli|i(2WSmTpNis$DkP4Y5 zGh~+RCAX1%WIwr`93ThD9ps&)O75iow=zWZzNnFxT4F3I1?8OyX@SbRwp0hnrS zmg=r)y8PdxOBLhP^ZO1ggdM~(Wom_}&W+KrL3m0+>V~=KK{^mplZXM5`lI6{QRs1g zJ#*Iym7U31pRBtU4ID^=SR?K?WqL6}_m9}(_f+GPBjr(S#`Gu;)1wl4`QU#7Lc1^a z;NSAEJz!GX%2d~NGYQdsG^%3=NE}OkDJh+G4@iOjVG0#~m#A>7xK9qFyH(6$su6;^ z&;Yu(r17Ja5X^Q^m4XuYKxcg80#RhhN^hkd%QeLrRR*Ima58%SdP88W|tlhnc?pK_t5x;x+F- zkqr@>nc>=8f=`M;YY)+??|XIWG!IsgVla)hmD96Bbf*iIa}^(MW49PvK8vk`9%Uy^ zV!-&VIy{c?hNmxc&4{YvrBeH&mX?6RJ4=s`Xoz#rA+a%E1=@6P?CxVQ(0SyqrQ3N- zqw{z;PUj(^5^vKmC8M*h11X@$XHZ$WvOTQpkd$CF-lpS_F;QLF0`6dus#~`QnMJ|3 zhxA#lz;#rln~e2Qd(lVzIF8i>QfUttyK8!vU2iiHr;cFt3>HTQ3>=j~p)`irn(~0i z?M9(d`hz!fsI?A4Mj9gi(Nxv;?(XNVkb?1RvpjBve5 z4#pZ~(K0&|6{LWu>_^RZhagsPClaorN|KRu7QsBgc zz_PtiV`jF;t&eLMm!pT#5}0g+=^Gw+$naZq9_CJ!Dws?24p7}O%(;iiI{}e9Vr!Z} zgoHP<1Wgs7`9M^69hyZfUDPp-7>w=&j6T&7jAU=@cF=q|Mhi2QnS2LF(t{Jv9)sb4 z)gy953Z00K8S{ow=9Wmp=3ZmP=cC=|yy>RD?w?Gq(e;-#GDqs9L2S|_hsa?vPu@lD zA`9dQStLv3D7l*~lNGW`*2q2NUUH23e`~;3{2dIVfm1&YI$9YZ5jF>eJrN6GEqTO( z7$vMTU4;~=ejOnDVy7Tuo`N$_Q~($oRh^xd62pKa#g%6zqQ^3NI{S_TZ;hOP{|FI*m zVs>G2wgQv+O#0Pjn@^UnRL>5uE)gw?*j@7Ri|thpgLiVLjfeN1agg|xO&~-oJ-2#X zB}mN9$HuHLhghiXSl=;q_bwpl3mqY7c4YL7z_Zq9o-J-r^lL}S-37$Ncej`PTQgN9 z=V?@lmIa_C(%tST2sd#&p!hdpLI|Vh%RoGo%@IK5bDcp2L-#6M04i4Zbu{{7IwtO_ zO1G3Kgh<_~4Awm(`Ex&7Uj$0t8M`Yi1^NYCM*G`SVr2O+ zQs5kyQ$fk0V{Me2Oc0Adw9L-6>_ z@)k_VIIp(J@YNghd(N0a1zN%H6;=oKBM8NRsE04pGuM!ljNl& z`E-(eCP}?W$AOnWe1i~&D7+}XuXx=*)@6Hk#8xS7vU57@Fq!pm-^HQx=bIUT!a8_ERrYzp!20PzHL=L1o0JT+Y` z6-o){Oobu^RMvZ-vi>%XeaqMQ#ne^yVr{X_w7FO_I6OImL4O+sB;u-iEks#$WUuha z(FmCc87X;gVCW(m5dJVn?2wvQ|gN@q;zqtFRg1t_SNA$a#?7{@X=l?2e^ zZ=)VDgA^#oiiG)nN#Ok55jbbjW1*ZD*BQ!@0w}*f37G$vINlu!&6qHcH(ReVFGTw> zftVVcp2pTmtk{!)j*9I9J3ywaiYB&OV&N0)uyTim$AVQxZi*8~ z++P;$0+Km8AXH9JW$pejZTI-5X#s~Hw66S-CNC&XN4_^HKWmpi|2eWp)HZ zScPfJJw@{@w;@sH>G25$SKCC6PtzRR@d7!r1x`_Lh(iVL_PFBu@ zS7d!ZNwaFFE)?ftXm(Wlyg;*Sr>GsPF1)cmMe})ko5o@sjcTVCX)f)gIU2uInqQ*X zw9_um_&lT=qZxggX4I}z^o|5W+nS=_Gc>n$tW&!D@a%Bmb9VmIT^sm?&)5FmDR#m0 zwO{=s{`~Vl;Lrayl-!Wiup;^kN%E5<61`>que*|1K7GmgT?@&dZMdlGIQiLzTM=;c z{CB!GBKZ54n+_lB z(h}rJjhxm*me(m+Zsu7clN$~{NRpyDKf0fj|IxkSWOS4PaI6Itg-VxCzcxa!Suj^gXx3m zJNgc$o3;4`yP^DgGTrB9u`+|{zVtw|wyP^N)TVOrstPOZD2q6iZ60N2?~+%TkKj zZZ+TtSyc%)D9uS#AF3`z#ml>)Z!$eucUJGGzbI!qS`+ z1TLmb`tW+f51;_q43@+biYW11J@UqXa6}al<$~%J({vh$Z`dnLm~j>hA_2kyqSm6j zPIxsJib_Lmd9)V)g=K!!(MBim?Nc-93@uL;_Owo=N{S&05;Dh$bES20ZDFBL6=oFB zPARm=r&swLxZ@m;4mKODF!8D9QE}JbF26_6!UpV#te)4E9 z(wfiy+pDlPb3?3z=OaNPG4bquS8EB~NOWs0sixT4#47i72ZUGTT@pM7a+`Xle2o@F zfRdFSK%p-$gAqseo&KqL0jM*rY>r)Gx+BqjanFI#8cfdA>JrXlr-x{z?yW5^&MzKH zZMyToiC${L^bYinv*RDPO~dH2mwWqq4==5(;%B{R^y|4of4)D5qx*a9d+bH}GoSQH zHJ0vO#IG{H1q3hf(D|jcmEMy$<{;6NNL=*Y58d+0-#q^3{VlUknUl&f*ez&c-tL0T_q)+~-AcwtlUwnNyIln;G{lm|ns6J$?|tZ&hradYs}mQ=7m(Ha%Wo}E zbLt-iS?mR5?RZU)^~J~kB*j{by+E~APd{fR6MCZiyq>uR?_}g&r)5W*vZWEJHsKfj z{QbB5kDq?KJK?+_`)3~be~T>pOM7-wk6Wszr)Sg8UwP$~H;z9Kt8){)mzyTwEAaU2 z;K@=Q1FN;=LpJ`v$wkfhMfemK*N)6lrx&xckHL{<<@?fblI3$JPj1o<+_5RKiD~4Y z|JDz$O{9jBR86z@a;ei4Lidf-O`w1e#F(P;;h{L=kvZ{xjRs%IaibX#T{?Ot`u z%;-iopY7>Q(Oe$;%nMu!{^gr^Nlz)g$R-Vr$y5)z1{(G6{sM(3cy0GQh1&KcQm;;J zujWhs#Y9q>TQ9@VEestBtX{zDH-7TPYZ);9;`uLK z8&NMr{lP!Ia-~zR)7me&USQsv@B7HjbgThh;!m277!udt_rzy(ZBcY@1rr~3rcw7sq1e6S~L!pMCR0mx3XlUXE#p#rDG8Jq~QPT z-~W(H0eeC6^r@XYM0^ww6l2G!H||O%F!t)&c)gvg8O`imahqutvfHd?HoL9RC^om% zth{Af&0Kb_VDyaZM!r9T|8!cX-+k_>Yfw(YI`x^|l+JjWutw0nbN>xvU2J5P=)Rmc zLz#JfrKBFO{cJ-oZH@o+mjE<<`SG{K%h&!9FNzFoym9;Lk)zxFs${!;%&u=2XY6d( z`G2_Tq3f}iv=K-QFxu07d79}6U+GU^mVp)jH&XQe{ssUcT^a{j>GVTAq@6=W7<>9CCa%R;qwwq^P4P+v!@rA$gaO3)*YCNe? zZh-pV|EW+Ir%yp)&|hAB5-(?34eMM~!xGi_=N~%UfKc>5^rz!O$jgk?{O<@8^+*2q zcR8ln3u52-(C$i&hYn9F%_3MR8SlHTPV`*&ZP(Sno z`Nre_n0QO#!usL0#k=}-Q@7pMrt@lK+*h5IqO&6P41b6(gMXR)!Ku=-{42*FdHyKy zN0C2j{4vKLb^d7Zhs_^N_Aq#P1~1RxDXXU5ZUgXNu2YPC>mQO37WDoKmWc%Zo>fou@^|tg1E9G$J&lz9KY|*%&GKa>fWM_q%%%o4L4|TzX6{ zJtmhPlS_}urH6}##ihsM(qnPyvAFbDA_|uti%XBirN`pZV{z%RxXfC-REw8t@lq{b zs>Ms?d1ZO2Szct1Z3;aWxmr%=zdz?}#4 z#CoQsWHq>@Xf-&bY>~MhUAErR@=E4|sEVR+nE!V6`;V7=Cuq7*~7rx4$YT z59C$xH>e2YQ1HV@N$IbnlG4v*b%^ShDiJF1=dX_mf9wx_`>Tno5=@0C7OS4&HmV~w z%BL>uS;;je(9C&FUtRrD5*mg#rjb>?3f?bMSH3LeODVz3D`9FbMb`aFqud+O$|zq- zah5`+WhyZhQY@tkmO_}N5M~v)N|P<<>(I<(L$fT+noBmdtgh+!pQ-6tP0wq3QPbx% zy`ky0rZ+VsqZxX#s2PT4n3`c}MpiR&nvvIxf@a|NS|Xukes3j_N+@(IjL0gr&nmUg zDz(olwa+TG&t|>Nt5mn36i`r-6_iFR2p$QUWr_suCfb2G?@%htn(OylbG&>GcL895 z{fx;mDnE1TXHogo4fQjtehN%UsZs(BUH!Dw&%E+!sx+D^jiwq|B2*L$F3)L3T{9Y* zVQWTHGc%g0C+9S?rkQokY-naPnM+#9Y*J5}nw8b8oMz=UtDsp$&8j7B&8lk_V1^ff zETd%&EenvrQIi@@E}v5Kzu1-Ss^SuEGtmJ zzm%Yyz@rEf;1OR6m3f8Ayh3GOz>}vdC@~8;Zw(aJT~Vn)QK>;uA+snT<#aciZfLrt z={Zd=Bnz5e({!-O&`dK4j#-+S)y$k`<~6gRnMKV6C+#FSl+i3*vkc8LwR|43JpJW^ z(AXSDGIQe%)MM7K*Ti#9JnQ1wIFE&!{L_?AOMDl_QzSP#9Yij7VNDW0^0loxxZ}`mEt+<38)A*Vk$EH5cqvEC*X2FwAVN zX_yVVQ|H|M;~stLvNlEedQCV`J>GHZ)b)FFso%N9N1D%hsfP>LaeSXUl9ik;#JtKQ zmp1qu${!>WdwzLyC||k!f5wSBGm1QUsh{cN-t=3W!d@VjS}v@ISI#cp4pE|KT5`%Z#$ zPfq103yDLwhOv!)Vq~x)&fgo}i94@{aUv)^3u-@>Vr`*kL5aBI<>^t}6VI-9s^G8? z?ES-Xjf@4`;3OV~n%{tbJEdGPNI-&;0sfS

9+dVANTZwqCRi-f@HUQ_ zNG$Ceo}8V)wdCShD<3f;4!|VNq79D54(pV4FX4zBUo?uj8-cl#u`%aA<~+EKJB8j& zfZpe#py%geECl+GM)&aCgZ~hDh(sVgLcI?N{Cpe)(&PIj51ivcJ+uVC;+Kj)MsI1~ z6izsqW`}gfl#lXwsd_t3OT_67>3}*FGFvI!wgGPMi4QmbH9HJ%z838iZT;=grsf|H zYyL}kJPbZtu+iIkwgYW1wmE~=k;Tg7)awawxTBQbvN%=6wKT0*%j$;gPo0RkZ!%s) zbV894p+KFY-JaR`73H|Uj!jPBL~}u>GA3|CkU$9;DGfVGCT7P=mGbZw-Rw(Qyh@J! d@3Ft#wPUc#9S1<(Y#KSEIj7jaXZvyC{|~3$rtbg% literal 0 HcmV?d00001 From 7ed28af4fe3a205e83f38834c3db0b749a214120 Mon Sep 17 00:00:00 2001 From: Tboy Date: Thu, 5 Mar 2020 19:00:06 +0800 Subject: [PATCH 12/59] Upgrade curator version (#2083) * upgrade curator version issue: https://github.com/apache/incubator-dolphinscheduler/issues/2020 curator issue: https://github.com/apache/curator/pull/297 * add DefaultEnsembleProvider override * add some unit test * add License --- dolphinscheduler-api/pom.xml | 6 ++ dolphinscheduler-server/pom.xml | 6 ++ .../service/zk/DefaultEnsembleProvider.java | 10 +++ .../zk/DefaultEnsembleProviderTest.java | 65 +++++++++++++++++++ pom.xml | 9 ++- 5 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProviderTest.java diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 41971734ec..b7c3f3da69 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -140,6 +140,12 @@ org.apache.curator curator-recipes + + + org.apache.zookeeper + zookeeper + + diff --git a/dolphinscheduler-server/pom.xml b/dolphinscheduler-server/pom.xml index 080b87ebaa..86490197b6 100644 --- a/dolphinscheduler-server/pom.xml +++ b/dolphinscheduler-server/pom.xml @@ -86,6 +86,12 @@ org.apache.curator curator-recipes + + + org.apache.zookeeper + zookeeper + + org.apache.zookeeper diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProvider.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProvider.java index 9eedf7a4ca..dbe8bd6395 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProvider.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProvider.java @@ -45,4 +45,14 @@ public class DefaultEnsembleProvider implements EnsembleProvider { public void close() throws IOException { //NOP } + + @Override + public void setConnectionString(String connectionString) { + //NOP + } + + @Override + public boolean updateServerListEnabled() { + return false; + } } diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProviderTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProviderTest.java new file mode 100644 index 0000000000..cdec9d0547 --- /dev/null +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProviderTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.service.zk; + +import org.apache.curator.ensemble.EnsembleProvider; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.*; + +public class DefaultEnsembleProviderTest { + private static final String DEFAULT_SERVER_LIST = "localhost:2181"; + + @Test + public void startAndClose() { + EnsembleProvider ensembleProvider = new DefaultEnsembleProvider(DEFAULT_SERVER_LIST); + try { + ensembleProvider.start(); + } catch (Exception e) { + Assert.fail("EnsembleProvider start error: " + e.getMessage()); + } + try { + ensembleProvider.close(); + } catch (IOException e) { + Assert.fail("EnsembleProvider close error: " + e.getMessage()); + } + } + + @Test + public void getConnectionString() { + EnsembleProvider ensembleProvider = new DefaultEnsembleProvider(DEFAULT_SERVER_LIST); + Assert.assertEquals(DEFAULT_SERVER_LIST, ensembleProvider.getConnectionString()); + } + + @Test + public void setConnectionString() { + EnsembleProvider ensembleProvider = new DefaultEnsembleProvider(DEFAULT_SERVER_LIST); + ensembleProvider.setConnectionString("otherHost:2181"); + Assert.assertEquals(DEFAULT_SERVER_LIST, ensembleProvider.getConnectionString()); + } + + @Test + public void updateServerListEnabled() { + EnsembleProvider ensembleProvider = new DefaultEnsembleProvider(DEFAULT_SERVER_LIST); + Assert.assertFalse(ensembleProvider.updateServerListEnabled()); + } +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 57063a86c2..dee1dce8b2 100644 --- a/pom.xml +++ b/pom.xml @@ -59,7 +59,7 @@ UTF-8 UTF-8 - 2.12.0 + 4.3.0 5.1.5.RELEASE 2.1.3.RELEASE 1.8 @@ -244,6 +244,12 @@ org.apache.curator curator-recipes ${curator.version} + + + org.apache.zookeeper + zookeeper + + org.apache.zookeeper @@ -728,6 +734,7 @@ **/server/worker/task/datax/DataxTaskTest.java **/server/worker/task/sqoop/SqoopTaskTest.java **/server/utils/DataxUtilsTest.java + **/service/zk/DefaultEnsembleProviderTest.java From 1f92b4c4db3c2b96d767476bf77002d894de49c2 Mon Sep 17 00:00:00 2001 From: dailidong Date: Thu, 5 Mar 2020 23:21:10 +0800 Subject: [PATCH 13/59] Optimize ParameterUtils.curingGlobalParams() execution efficiency (#2090) * Optimize ParameterUtils.curingGlobalParams() execution efficiency * Remove excess null check --- .../common/utils/ParameterUtils.java | 36 +++++++++---------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java index 7f2888384e..9492b49cb1 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java @@ -119,10 +119,15 @@ public class ParameterUtils { */ public static String curingGlobalParams(Map globalParamMap, List globalParamList, CommandType commandType, Date scheduleTime){ - Map globalMap = new HashMap<>(); - if(globalParamMap!= null){ - globalMap.putAll(globalParamMap); - } + + if (globalParamList == null || globalParamList.isEmpty()) { + return null; + } + + Map globalMap = new HashMap<>(); + if (globalParamMap!= null){ + globalMap.putAll(globalParamMap); + } Map allParamMap = new HashMap<>(); //If it is a complement, a complement time needs to be passed in, according to the task type Map timeParams = BusinessTimeUtils @@ -132,9 +137,7 @@ public class ParameterUtils { allParamMap.putAll(timeParams); } - if (globalMap != null) { - allParamMap.putAll(globalMap); - } + allParamMap.putAll(globalMap); Set> entries = allParamMap.entrySet(); @@ -146,22 +149,15 @@ public class ParameterUtils { resolveMap.put(entry.getKey(),str); } } + globalMap.putAll(resolveMap); - if (globalMap != null){ - globalMap.putAll(resolveMap); - } - - if (globalParamList != null && globalParamList.size() > 0){ - - for (Property property : globalParamList){ - String val = globalMap.get(property.getProp()); - if (val != null){ - property.setValue(val); - } + for (Property property : globalParamList){ + String val = globalMap.get(property.getProp()); + if (val != null){ + property.setValue(val); } - return JSONObject.toJSONString(globalParamList); } - return null; + return JSONObject.toJSONString(globalParamList); } From 6070738acbb89e0a7ed49b487fc1225888e29019 Mon Sep 17 00:00:00 2001 From: dailidong Date: Thu, 5 Mar 2020 23:34:37 +0800 Subject: [PATCH 14/59] Support worker server to run bat script (#2023) * Support worker server to run bat script 1. Reimplement ProcessImpl.java, ProcessEnvironment.java and ProcessBuilder.java for Windows 2. Modify shell task code for windows 3. Add ASF License * Add Unit Test --- .../dolphinscheduler/common/Constants.java | 2 +- .../utils/process/ProcessBuilderForWin32.java | 1065 +++++++++++++++++ .../process/ProcessEnvironmentForWin32.java | 286 +++++ .../utils/process/ProcessImplForWin32.java | 752 ++++++++++++ .../common/ConstantsTest.java | 40 + .../common/utils/OSUtilsTest.java | 24 +- .../process/ProcessBuilderForWin32Test.java | 210 ++++ .../ProcessEnvironmentForWin32Test.java | 124 ++ .../process/ProcessImplForWin32Test.java | 70 ++ .../worker/task/AbstractCommandExecutor.java | 92 +- .../worker/task/ShellCommandExecutor.java | 39 +- .../server/worker/task/datax/DataxTask.java | 11 +- .../server/worker/task/shell/ShellTask.java | 9 +- .../server/utils/ProcessUtilsTest.java | 17 + .../worker/task/shell/ShellTaskTest.java | 198 +++ pom.xml | 4 + 16 files changed, 2887 insertions(+), 56 deletions(-) create mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32.java create mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32.java create mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32Test.java create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32Test.java create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32Test.java create mode 100644 dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 73125f4926..6af0e6445f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -746,7 +746,7 @@ public final class Constants { * application regex */ public static final String APPLICATION_REGEX = "application_\\d+_\\d+"; - public static final String PID = "pid"; + public static final String PID = OSUtils.isWindows() ? "handle" : "pid"; /** * month_begin */ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32.java new file mode 100644 index 0000000000..4fb5f94616 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32.java @@ -0,0 +1,1065 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils.process; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * This class is used to create operating system processes. + * + *

Each {@code ProcessBuilderForWindows} instance manages a collection + * of process attributes. The {@link #start()} method creates a new + * {@link Process} instance with those attributes. The {@link + * #start()} method can be invoked repeatedly from the same instance + * to create new subprocesses with identical or related attributes. + * + *

Each process builder manages these process attributes: + * + *

    + * + *
  • a command, a list of strings which signifies the + * external program file to be invoked and its arguments, if any. + * Which string lists represent a valid operating system command is + * system-dependent. For example, it is common for each conceptual + * argument to be an element in this list, but there are operating + * systems where programs are expected to tokenize command line + * strings themselves - on such a system a Java implementation might + * require commands to contain exactly two elements. + * + *
  • an environment, which is a system-dependent mapping from + * variables to values. The initial value is a copy of + * the environment of the current process (see {@link System#getenv()}). + * + *
  • a working directory. The default value is the current + * working directory of the current process, usually the directory + * named by the system property {@code user.dir}. + * + *
  • a source of standard input. + * By default, the subprocess reads input from a pipe. Java code + * can access this pipe via the output stream returned by + * {@link Process#getOutputStream()}. However, standard input may + * be redirected to another source using + * {@link #redirectInput(ProcessBuilderForWin32.Redirect) redirectInput}. + * In this case, {@link Process#getOutputStream()} will return a + * null output stream, for which: + * + *
      + *
    • the {@link OutputStream#write(int) write} methods always + * throw {@code IOException} + *
    • the {@link OutputStream#close() close} method does nothing + *
    + * + *
  • a destination for standard output + * and standard error. By default, the subprocess writes standard + * output and standard error to pipes. Java code can access these pipes + * via the input streams returned by {@link Process#getInputStream()} and + * {@link Process#getErrorStream()}. However, standard output and + * standard error may be redirected to other destinations using + * {@link #redirectOutput(ProcessBuilderForWin32.Redirect) redirectOutput} and + * {@link #redirectError(ProcessBuilderForWin32.Redirect) redirectError}. + * In this case, {@link Process#getInputStream()} and/or + * {@link Process#getErrorStream()} will return a null input + * stream, for which: + * + *
      + *
    • the {@link InputStream#read() read} methods always return + * {@code -1} + *
    • the {@link InputStream#available() available} method always returns + * {@code 0} + *
    • the {@link InputStream#close() close} method does nothing + *
    + * + *
  • a redirectErrorStream property. Initially, this property + * is {@code false}, meaning that the standard output and error + * output of a subprocess are sent to two separate streams, which can + * be accessed using the {@link Process#getInputStream()} and {@link + * Process#getErrorStream()} methods. + * + *

    If the value is set to {@code true}, then: + * + *

      + *
    • standard error is merged with the standard output and always sent + * to the same destination (this makes it easier to correlate error + * messages with the corresponding output) + *
    • the common destination of standard error and standard output can be + * redirected using + * {@link #redirectOutput(ProcessBuilderForWin32.Redirect) redirectOutput} + *
    • any redirection set by the + * {@link #redirectError(ProcessBuilderForWin32.Redirect) redirectError} + * method is ignored when creating a subprocess + *
    • the stream returned from {@link Process#getErrorStream()} will + * always be a null input stream + *
    + * + *
+ * + *

Modifying a process builder's attributes will affect processes + * subsequently started by that object's {@link #start()} method, but + * will never affect previously started processes or the Java process + * itself. + * + *

Most error checking is performed by the {@link #start()} method. + * It is possible to modify the state of an object so that {@link + * #start()} will fail. For example, setting the command attribute to + * an empty list will not throw an exception unless {@link #start()} + * is invoked. + * + *

Note that this class is not synchronized. + * If multiple threads access a {@code ProcessBuilderForWindows} instance + * concurrently, and at least one of the threads modifies one of the + * attributes structurally, it must be synchronized externally. + * + *

Starting a new process which uses the default working directory + * and environment is easy: + * + * 

 {@code
+ * Process p = new ProcessBuilderForWindows("myCommand", "myArg").start();
+ * }
+ * + *

Here is an example that starts a process with a modified working + * directory and environment, and redirects standard output and error + * to be appended to a log file: + * + * 

 {@code
+ * ProcessBuilderForWindows pb =
+ *   new ProcessBuilderForWindows("myCommand", "myArg1", "myArg2");
+ * Map env = pb.environment();
+ * env.put("VAR1", "myValue");
+ * env.remove("OTHERVAR");
+ * env.put("VAR2", env.get("VAR1") + "suffix");
+ * pb.directory(new File("myDir"));
+ * File log = new File("log");
+ * pb.redirectErrorStream(true);
+ * pb.redirectOutput(Redirect.appendTo(log));
+ * Process p = pb.start();
+ * assert pb.redirectInput() == Redirect.PIPE;
+ * assert pb.redirectOutput().file() == log;
+ * assert p.getInputStream().read() == -1;
+ * }
+ * + *

To start a process with an explicit set of environment + * variables, first call {@link Map#clear() Map.clear()} + * before adding environment variables. + * + * @author Martin Buchholz + * @since 1.5 + */ + +public class ProcessBuilderForWin32 { + + private String username; + private String password; + private List command; + private File directory; + private Map environment; + private boolean redirectErrorStream; + private ProcessBuilderForWin32.Redirect[] redirects; + + /** + * Constructs a process builder with the specified operating + * system program and arguments. This constructor does not + * make a copy of the {@code command} list. Subsequent + * updates to the list will be reflected in the state of the + * process builder. It is not checked whether + * {@code command} corresponds to a valid operating system + * command. + * + * @param command the list containing the program and its arguments + * @throws NullPointerException if the argument is null + */ + public ProcessBuilderForWin32(List command) { + if (command == null) + throw new NullPointerException(); + this.command = command; + } + + /** + * Constructs a process builder with the specified operating + * system program and arguments. This is a convenience + * constructor that sets the process builder's command to a string + * list containing the same strings as the {@code command} + * array, in the same order. It is not checked whether + * {@code command} corresponds to a valid operating system + * command. + * + * @param command a string array containing the program and its arguments + */ + public ProcessBuilderForWin32(String... command) { + this.command = new ArrayList<>(command.length); + for (String arg : command) + this.command.add(arg); + } + + /** + * set username and password for process + * + * @param username username + * @param password password + * @return this process builder + */ + public ProcessBuilderForWin32 user(String username, String password) { + this.username = username; + this.password = password; + return this; + } + + /** + * Sets this process builder's operating system program and + * arguments. This method does not make a copy of the + * {@code command} list. Subsequent updates to the list will + * be reflected in the state of the process builder. It is not + * checked whether {@code command} corresponds to a valid + * operating system command. + * + * @param command the list containing the program and its arguments + * @return this process builder + * + * @throws NullPointerException if the argument is null + */ + public ProcessBuilderForWin32 command(List command) { + if (command == null) + throw new NullPointerException(); + this.command = command; + return this; + } + + /** + * Sets this process builder's operating system program and + * arguments. This is a convenience method that sets the command + * to a string list containing the same strings as the + * {@code command} array, in the same order. It is not + * checked whether {@code command} corresponds to a valid + * operating system command. + * + * @param command a string array containing the program and its arguments + * @return this process builder + */ + public ProcessBuilderForWin32 command(String... command) { + this.command = new ArrayList<>(command.length); + for (String arg : command) + this.command.add(arg); + return this; + } + + /** + * Returns this process builder's operating system program and + * arguments. The returned list is not a copy. Subsequent + * updates to the list will be reflected in the state of this + * process builder. + * + * @return this process builder's program and its arguments + */ + public List command() { + return command; + } + + /** + * Returns a string map view of this process builder's environment. + * + * Whenever a process builder is created, the environment is + * initialized to a copy of the current process environment (see + * {@link System#getenv()}). Subprocesses subsequently started by + * this object's {@link #start()} method will use this map as + * their environment. + * + *

The returned object may be modified using ordinary {@link + * Map Map} operations. These modifications will be + * visible to subprocesses started via the {@link #start()} + * method. Two {@code ProcessBuilderForWindows} instances always + * contain independent process environments, so changes to the + * returned map will never be reflected in any other + * {@code ProcessBuilderForWindows} instance or the values returned by + * {@link System#getenv System.getenv}. + * + *

If the system does not support environment variables, an + * empty map is returned. + * + *

The returned map does not permit null keys or values. + * Attempting to insert or query the presence of a null key or + * value will throw a {@link NullPointerException}. + * Attempting to query the presence of a key or value which is not + * of type {@link String} will throw a {@link ClassCastException}. + * + *

The behavior of the returned map is system-dependent. A + * system may not allow modifications to environment variables or + * may forbid certain variable names or values. For this reason, + * attempts to modify the map may fail with + * {@link UnsupportedOperationException} or + * {@link IllegalArgumentException} + * if the modification is not permitted by the operating system. + * + *

Since the external format of environment variable names and + * values is system-dependent, there may not be a one-to-one + * mapping between them and Java's Unicode strings. Nevertheless, + * the map is implemented in such a way that environment variables + * which are not modified by Java code will have an unmodified + * native representation in the subprocess. + * + *

The returned map and its collection views may not obey the + * general contract of the {@link Object#equals} and + * {@link Object#hashCode} methods. + * + *

The returned map is typically case-sensitive on all platforms. + * + *

If a security manager exists, its + * {@link SecurityManager#checkPermission checkPermission} method + * is called with a + * {@link RuntimePermission}{@code ("getenv.*")} permission. + * This may result in a {@link SecurityException} being thrown. + * + *

When passing information to a Java subprocess, + * system properties + * are generally preferred over environment variables. + * + * @return this process builder's environment + * + * @throws SecurityException + * if a security manager exists and its + * {@link SecurityManager#checkPermission checkPermission} + * method doesn't allow access to the process environment + * + * @see Runtime#exec(String[],String[], File) + * @see System#getenv() + */ + public Map environment() { + SecurityManager security = System.getSecurityManager(); + if (security != null) + security.checkPermission(new RuntimePermission("getenv.*")); + + if (environment == null) + environment = ProcessEnvironmentForWin32.environment(); + + assert environment != null; + + return environment; + } + + // Only for use by Runtime.exec(...envp...) + ProcessBuilderForWin32 environment(String[] envp) { + assert environment == null; + if (envp != null) { + environment = ProcessEnvironmentForWin32.emptyEnvironment(envp.length); + assert environment != null; + + for (String envstring : envp) { + // Before 1.5, we blindly passed invalid envstrings + // to the child process. + // We would like to throw an exception, but do not, + // for compatibility with old broken code. + + // Silently discard any trailing junk. + if (envstring.indexOf((int) '\u0000') != -1) + envstring = envstring.replaceFirst("\u0000.*", ""); + + int eqlsign = + envstring.indexOf('=', ProcessEnvironmentForWin32.MIN_NAME_LENGTH); + // Silently ignore envstrings lacking the required `='. + if (eqlsign != -1) + environment.put(envstring.substring(0,eqlsign), + envstring.substring(eqlsign+1)); + } + } + return this; + } + + /** + * Returns this process builder's working directory. + * + * Subprocesses subsequently started by this object's {@link + * #start()} method will use this as their working directory. + * The returned value may be {@code null} -- this means to use + * the working directory of the current Java process, usually the + * directory named by the system property {@code user.dir}, + * as the working directory of the child process. + * + * @return this process builder's working directory + */ + public File directory() { + return directory; + } + + /** + * Sets this process builder's working directory. + * + * Subprocesses subsequently started by this object's {@link + * #start()} method will use this as their working directory. + * The argument may be {@code null} -- this means to use the + * working directory of the current Java process, usually the + * directory named by the system property {@code user.dir}, + * as the working directory of the child process. + * + * @param directory the new working directory + * @return this process builder + */ + public ProcessBuilderForWin32 directory(File directory) { + this.directory = directory; + return this; + } + + // ---------------- I/O Redirection ---------------- + + /** + * Implements a null input stream. + */ + static class NullInputStream extends InputStream { + static final ProcessBuilderForWin32.NullInputStream INSTANCE = new ProcessBuilderForWin32.NullInputStream(); + private NullInputStream() {} + public int read() { return -1; } + public int available() { return 0; } + } + + /** + * Implements a null output stream. + */ + static class NullOutputStream extends OutputStream { + static final ProcessBuilderForWin32.NullOutputStream INSTANCE = new ProcessBuilderForWin32.NullOutputStream(); + private NullOutputStream() {} + public void write(int b) throws IOException { + throw new IOException("Stream closed"); + } + } + + /** + * Represents a source of subprocess input or a destination of + * subprocess output. + * + * Each {@code Redirect} instance is one of the following: + * + *

    + *
  • the special value {@link #PIPE Redirect.PIPE} + *
  • the special value {@link #INHERIT Redirect.INHERIT} + *
  • a redirection to read from a file, created by an invocation of + * {@link ProcessBuilderForWin32.Redirect#from Redirect.from(File)} + *
  • a redirection to write to a file, created by an invocation of + * {@link ProcessBuilderForWin32.Redirect#to Redirect.to(File)} + *
  • a redirection to append to a file, created by an invocation of + * {@link ProcessBuilderForWin32.Redirect#appendTo Redirect.appendTo(File)} + *
+ * + *

Each of the above categories has an associated unique + * {@link ProcessBuilderForWin32.Redirect.Type Type}. + * + * @since 1.7 + */ + public static abstract class Redirect { + /** + * The type of a {@link ProcessBuilderForWin32.Redirect}. + */ + public enum Type { + /** + * The type of {@link ProcessBuilderForWin32.Redirect#PIPE Redirect.PIPE}. + */ + PIPE, + + /** + * The type of {@link ProcessBuilderForWin32.Redirect#INHERIT Redirect.INHERIT}. + */ + INHERIT, + + /** + * The type of redirects returned from + * {@link ProcessBuilderForWin32.Redirect#from Redirect.from(File)}. + */ + READ, + + /** + * The type of redirects returned from + * {@link ProcessBuilderForWin32.Redirect#to Redirect.to(File)}. + */ + WRITE, + + /** + * The type of redirects returned from + * {@link ProcessBuilderForWin32.Redirect#appendTo Redirect.appendTo(File)}. + */ + APPEND + }; + + /** + * Returns the type of this {@code Redirect}. + * @return the type of this {@code Redirect} + */ + public abstract ProcessBuilderForWin32.Redirect.Type type(); + + /** + * Indicates that subprocess I/O will be connected to the + * current Java process over a pipe. + * + * This is the default handling of subprocess standard I/O. + * + *

It will always be true that + * 

 {@code
+         * Redirect.PIPE.file() == null &&
+         * Redirect.PIPE.type() == Redirect.Type.PIPE
+         * }
+ */ + public static final ProcessBuilderForWin32.Redirect PIPE = new ProcessBuilderForWin32.Redirect() { + public Type type() { return Type.PIPE; } + public String toString() { return type().toString(); }}; + + /** + * Indicates that subprocess I/O source or destination will be the + * same as those of the current process. This is the normal + * behavior of most operating system command interpreters (shells). + * + *

It will always be true that + * 

 {@code
+         * Redirect.INHERIT.file() == null &&
+         * Redirect.INHERIT.type() == Redirect.Type.INHERIT
+         * }
+ */ + public static final ProcessBuilderForWin32.Redirect INHERIT = new ProcessBuilderForWin32.Redirect() { + public Type type() { return Type.INHERIT; } + public String toString() { return type().toString(); }}; + + /** + * Returns the {@link File} source or destination associated + * with this redirect, or {@code null} if there is no such file. + * + * @return the file associated with this redirect, + * or {@code null} if there is no such file + */ + public File file() { return null; } + + /** + * When redirected to a destination file, indicates if the output + * is to be written to the end of the file. + */ + boolean append() { + throw new UnsupportedOperationException(); + } + + /** + * Returns a redirect to read from the specified file. + * + *

It will always be true that + * 

 {@code
+         * Redirect.from(file).file() == file &&
+         * Redirect.from(file).type() == Redirect.Type.READ
+         * }
+ * + * @param file The {@code File} for the {@code Redirect}. + * @throws NullPointerException if the specified file is null + * @return a redirect to read from the specified file + */ + public static ProcessBuilderForWin32.Redirect from(final File file) { + if (file == null) + throw new NullPointerException(); + return new ProcessBuilderForWin32.Redirect() { + public Type type() { return Type.READ; } + public File file() { return file; } + public String toString() { + return "redirect to read from file \"" + file + "\""; + } + }; + } + + /** + * Returns a redirect to write to the specified file. + * If the specified file exists when the subprocess is started, + * its previous contents will be discarded. + * + *

It will always be true that + * 

 {@code
+         * Redirect.to(file).file() == file &&
+         * Redirect.to(file).type() == Redirect.Type.WRITE
+         * }
+ * + * @param file The {@code File} for the {@code Redirect}. + * @throws NullPointerException if the specified file is null + * @return a redirect to write to the specified file + */ + public static ProcessBuilderForWin32.Redirect to(final File file) { + if (file == null) + throw new NullPointerException(); + return new ProcessBuilderForWin32.Redirect() { + public Type type() { return Type.WRITE; } + public File file() { return file; } + public String toString() { + return "redirect to write to file \"" + file + "\""; + } + boolean append() { return false; } + }; + } + + /** + * Returns a redirect to append to the specified file. + * Each write operation first advances the position to the + * end of the file and then writes the requested data. + * Whether the advancement of the position and the writing + * of the data are done in a single atomic operation is + * system-dependent and therefore unspecified. + * + *

It will always be true that + * 

 {@code
+         * Redirect.appendTo(file).file() == file &&
+         * Redirect.appendTo(file).type() == Redirect.Type.APPEND
+         * }
+ * + * @param file The {@code File} for the {@code Redirect}. + * @throws NullPointerException if the specified file is null + * @return a redirect to append to the specified file + */ + public static ProcessBuilderForWin32.Redirect appendTo(final File file) { + if (file == null) + throw new NullPointerException(); + return new ProcessBuilderForWin32.Redirect() { + public Type type() { return Type.APPEND; } + public File file() { return file; } + public String toString() { + return "redirect to append to file \"" + file + "\""; + } + boolean append() { return true; } + }; + } + + /** + * Compares the specified object with this {@code Redirect} for + * equality. Returns {@code true} if and only if the two + * objects are identical or both objects are {@code Redirect} + * instances of the same type associated with non-null equal + * {@code File} instances. + */ + public boolean equals(Object obj) { + if (obj == this) + return true; + if (! (obj instanceof ProcessBuilderForWin32.Redirect)) + return false; + ProcessBuilderForWin32.Redirect r = (ProcessBuilderForWin32.Redirect) obj; + if (r.type() != this.type()) + return false; + assert this.file() != null; + return this.file().equals(r.file()); + } + + /** + * Returns a hash code value for this {@code Redirect}. + * @return a hash code value for this {@code Redirect} + */ + public int hashCode() { + File file = file(); + if (file == null) + return super.hashCode(); + else + return file.hashCode(); + } + + /** + * No public constructors. Clients must use predefined + * static {@code Redirect} instances or factory methods. + */ + private Redirect() {} + } + + private ProcessBuilderForWin32.Redirect[] redirects() { + if (redirects == null) + redirects = new ProcessBuilderForWin32.Redirect[] { + ProcessBuilderForWin32.Redirect.PIPE, ProcessBuilderForWin32.Redirect.PIPE, ProcessBuilderForWin32.Redirect.PIPE + }; + return redirects; + } + + /** + * Sets this process builder's standard input source. + * + * Subprocesses subsequently started by this object's {@link #start()} + * method obtain their standard input from this source. + * + *

If the source is {@link ProcessBuilderForWin32.Redirect#PIPE Redirect.PIPE} + * (the initial value), then the standard input of a + * subprocess can be written to using the output stream + * returned by {@link Process#getOutputStream()}. + * If the source is set to any other value, then + * {@link Process#getOutputStream()} will return a + * null output stream. + * + * @param source the new standard input source + * @return this process builder + * @throws IllegalArgumentException + * if the redirect does not correspond to a valid source + * of data, that is, has type + * {@link ProcessBuilderForWin32.Redirect.Type#WRITE WRITE} or + * {@link ProcessBuilderForWin32.Redirect.Type#APPEND APPEND} + * @since 1.7 + */ + public ProcessBuilderForWin32 redirectInput(ProcessBuilderForWin32.Redirect source) { + if (source.type() == ProcessBuilderForWin32.Redirect.Type.WRITE || + source.type() == ProcessBuilderForWin32.Redirect.Type.APPEND) + throw new IllegalArgumentException( + "Redirect invalid for reading: " + source); + redirects()[0] = source; + return this; + } + + /** + * Sets this process builder's standard output destination. + * + * Subprocesses subsequently started by this object's {@link #start()} + * method send their standard output to this destination. + * + *

If the destination is {@link ProcessBuilderForWin32.Redirect#PIPE Redirect.PIPE} + * (the initial value), then the standard output of a subprocess + * can be read using the input stream returned by {@link + * Process#getInputStream()}. + * If the destination is set to any other value, then + * {@link Process#getInputStream()} will return a + * null input stream. + * + * @param destination the new standard output destination + * @return this process builder + * @throws IllegalArgumentException + * if the redirect does not correspond to a valid + * destination of data, that is, has type + * {@link ProcessBuilderForWin32.Redirect.Type#READ READ} + * @since 1.7 + */ + public ProcessBuilderForWin32 redirectOutput(ProcessBuilderForWin32.Redirect destination) { + if (destination.type() == ProcessBuilderForWin32.Redirect.Type.READ) + throw new IllegalArgumentException( + "Redirect invalid for writing: " + destination); + redirects()[1] = destination; + return this; + } + + /** + * Sets this process builder's standard error destination. + * + * Subprocesses subsequently started by this object's {@link #start()} + * method send their standard error to this destination. + * + *

If the destination is {@link ProcessBuilderForWin32.Redirect#PIPE Redirect.PIPE} + * (the initial value), then the error output of a subprocess + * can be read using the input stream returned by {@link + * Process#getErrorStream()}. + * If the destination is set to any other value, then + * {@link Process#getErrorStream()} will return a + * null input stream. + * + *

If the {@link #redirectErrorStream redirectErrorStream} + * attribute has been set {@code true}, then the redirection set + * by this method has no effect. + * + * @param destination the new standard error destination + * @return this process builder + * @throws IllegalArgumentException + * if the redirect does not correspond to a valid + * destination of data, that is, has type + * {@link ProcessBuilderForWin32.Redirect.Type#READ READ} + * @since 1.7 + */ + public ProcessBuilderForWin32 redirectError(ProcessBuilderForWin32.Redirect destination) { + if (destination.type() == ProcessBuilderForWin32.Redirect.Type.READ) + throw new IllegalArgumentException( + "Redirect invalid for writing: " + destination); + redirects()[2] = destination; + return this; + } + + /** + * Sets this process builder's standard input source to a file. + * + *

This is a convenience method. An invocation of the form + * {@code redirectInput(file)} + * behaves in exactly the same way as the invocation + * {@link #redirectInput(ProcessBuilderForWin32.Redirect) redirectInput} + * {@code (Redirect.from(file))}. + * + * @param file the new standard input source + * @return this process builder + * @since 1.7 + */ + public ProcessBuilderForWin32 redirectInput(File file) { + return redirectInput(ProcessBuilderForWin32.Redirect.from(file)); + } + + /** + * Sets this process builder's standard output destination to a file. + * + *

This is a convenience method. An invocation of the form + * {@code redirectOutput(file)} + * behaves in exactly the same way as the invocation + * {@link #redirectOutput(ProcessBuilderForWin32.Redirect) redirectOutput} + * {@code (Redirect.to(file))}. + * + * @param file the new standard output destination + * @return this process builder + * @since 1.7 + */ + public ProcessBuilderForWin32 redirectOutput(File file) { + return redirectOutput(ProcessBuilderForWin32.Redirect.to(file)); + } + + /** + * Sets this process builder's standard error destination to a file. + * + *

This is a convenience method. An invocation of the form + * {@code redirectError(file)} + * behaves in exactly the same way as the invocation + * {@link #redirectError(ProcessBuilderForWin32.Redirect) redirectError} + * {@code (Redirect.to(file))}. + * + * @param file the new standard error destination + * @return this process builder + * @since 1.7 + */ + public ProcessBuilderForWin32 redirectError(File file) { + return redirectError(ProcessBuilderForWin32.Redirect.to(file)); + } + + /** + * Returns this process builder's standard input source. + * + * Subprocesses subsequently started by this object's {@link #start()} + * method obtain their standard input from this source. + * The initial value is {@link ProcessBuilderForWin32.Redirect#PIPE Redirect.PIPE}. + * + * @return this process builder's standard input source + * @since 1.7 + */ + public ProcessBuilderForWin32.Redirect redirectInput() { + return (redirects == null) ? ProcessBuilderForWin32.Redirect.PIPE : redirects[0]; + } + + /** + * Returns this process builder's standard output destination. + * + * Subprocesses subsequently started by this object's {@link #start()} + * method redirect their standard output to this destination. + * The initial value is {@link ProcessBuilderForWin32.Redirect#PIPE Redirect.PIPE}. + * + * @return this process builder's standard output destination + * @since 1.7 + */ + public ProcessBuilderForWin32.Redirect redirectOutput() { + return (redirects == null) ? ProcessBuilderForWin32.Redirect.PIPE : redirects[1]; + } + + /** + * Returns this process builder's standard error destination. + * + * Subprocesses subsequently started by this object's {@link #start()} + * method redirect their standard error to this destination. + * The initial value is {@link ProcessBuilderForWin32.Redirect#PIPE Redirect.PIPE}. + * + * @return this process builder's standard error destination + * @since 1.7 + */ + public ProcessBuilderForWin32.Redirect redirectError() { + return (redirects == null) ? ProcessBuilderForWin32.Redirect.PIPE : redirects[2]; + } + + /** + * Sets the source and destination for subprocess standard I/O + * to be the same as those of the current Java process. + * + *

This is a convenience method. An invocation of the form + * 

 {@code
+     * pb.inheritIO()
+     * }
+ * behaves in exactly the same way as the invocation + * 
 {@code
+     * pb.redirectInput(Redirect.INHERIT)
+     *   .redirectOutput(Redirect.INHERIT)
+     *   .redirectError(Redirect.INHERIT)
+     * }
+ * + * This gives behavior equivalent to most operating system + * command interpreters, or the standard C library function + * {@code system()}. + * + * @return this process builder + * @since 1.7 + */ + public ProcessBuilderForWin32 inheritIO() { + Arrays.fill(redirects(), ProcessBuilderForWin32.Redirect.INHERIT); + return this; + } + + /** + * Tells whether this process builder merges standard error and + * standard output. + * + *

If this property is {@code true}, then any error output + * generated by subprocesses subsequently started by this object's + * {@link #start()} method will be merged with the standard + * output, so that both can be read using the + * {@link Process#getInputStream()} method. This makes it easier + * to correlate error messages with the corresponding output. + * The initial value is {@code false}. + * + * @return this process builder's {@code redirectErrorStream} property + */ + public boolean redirectErrorStream() { + return redirectErrorStream; + } + + /** + * Sets this process builder's {@code redirectErrorStream} property. + * + *

If this property is {@code true}, then any error output + * generated by subprocesses subsequently started by this object's + * {@link #start()} method will be merged with the standard + * output, so that both can be read using the + * {@link Process#getInputStream()} method. This makes it easier + * to correlate error messages with the corresponding output. + * The initial value is {@code false}. + * + * @param redirectErrorStream the new property value + * @return this process builder + */ + public ProcessBuilderForWin32 redirectErrorStream(boolean redirectErrorStream) { + this.redirectErrorStream = redirectErrorStream; + return this; + } + + /** + * Starts a new process using the attributes of this process builder. + * + *

The new process will + * invoke the command and arguments given by {@link #command()}, + * in a working directory as given by {@link #directory()}, + * with a process environment as given by {@link #environment()}. + * + *

This method checks that the command is a valid operating + * system command. Which commands are valid is system-dependent, + * but at the very least the command must be a non-empty list of + * non-null strings. + * + *

A minimal set of system dependent environment variables may + * be required to start a process on some operating systems. + * As a result, the subprocess may inherit additional environment variable + * settings beyond those in the process builder's {@link #environment()}. + * + *

If there is a security manager, its + * {@link SecurityManager#checkExec checkExec} + * method is called with the first component of this object's + * {@code command} array as its argument. This may result in + * a {@link SecurityException} being thrown. + * + *

Starting an operating system process is highly system-dependent. + * Among the many things that can go wrong are: + *

    + *
  • The operating system program file was not found. + *
  • Access to the program file was denied. + *
  • The working directory does not exist. + *
+ * + *

In such cases an exception will be thrown. The exact nature + * of the exception is system-dependent, but it will always be a + * subclass of {@link IOException}. + * + *

Subsequent modifications to this process builder will not + * affect the returned {@link Process}. + * + * @return a new {@link Process} object for managing the subprocess + * + * @throws NullPointerException + * if an element of the command list is null + * + * @throws IndexOutOfBoundsException + * if the command is an empty list (has size {@code 0}) + * + * @throws SecurityException + * if a security manager exists and + *

    + * + *
  • its + * {@link SecurityManager#checkExec checkExec} + * method doesn't allow creation of the subprocess, or + * + *
  • the standard input to the subprocess was + * {@linkplain #redirectInput redirected from a file} + * and the security manager's + * {@link SecurityManager#checkRead checkRead} method + * denies read access to the file, or + * + *
  • the standard output or standard error of the + * subprocess was + * {@linkplain #redirectOutput redirected to a file} + * and the security manager's + * {@link SecurityManager#checkWrite checkWrite} method + * denies write access to the file + * + *
+ * + * @throws IOException if an I/O error occurs + * + * @see Runtime#exec(String[], String[], File) + */ + public Process start() throws IOException { + // Must convert to array first -- a malicious user-supplied + // list might try to circumvent the security check. + String[] cmdarray = command.toArray(new String[command.size()]); + cmdarray = cmdarray.clone(); + + for (String arg : cmdarray) + if (arg == null) + throw new NullPointerException(); + // Throws IndexOutOfBoundsException if command is empty + String prog = cmdarray[0]; + + SecurityManager security = System.getSecurityManager(); + if (security != null) + security.checkExec(prog); + + String dir = directory == null ? null : directory.toString(); + + for (int i = 1; i < cmdarray.length; i++) { + if (cmdarray[i].indexOf('\u0000') >= 0) { + throw new IOException("invalid null character in command"); + } + } + + try { + return ProcessImplForWin32.start( + username, + password, + cmdarray, + environment, + dir, + redirects, + redirectErrorStream); + } catch (IOException | IllegalArgumentException e) { + String exceptionInfo = ": " + e.getMessage(); + Throwable cause = e; + if ((e instanceof IOException) && security != null) { + // Can not disclose the fail reason for read-protected files. + try { + security.checkRead(prog); + } catch (SecurityException se) { + exceptionInfo = ""; + cause = se; + } + } + // It's much easier for us to create a high-quality error + // message than the low-level C code which found the problem. + throw new IOException( + "Cannot run program \"" + prog + "\"" + + (dir == null ? "" : " (in directory \"" + dir + "\")") + + exceptionInfo, + cause); + } + } + +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32.java new file mode 100644 index 0000000000..3dbe7cb50f --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils.process; + +import com.sun.jna.platform.win32.Kernel32Util; + +import java.util.*; + +final class ProcessEnvironmentForWin32 extends HashMap { + + private static final long serialVersionUID = -8017839552603542824L; + + private static String validateName(String name) { + // An initial `=' indicates a magic Windows variable name -- OK + if (name.indexOf('=', 1) != -1 || + name.indexOf('\u0000') != -1) + throw new IllegalArgumentException + ("Invalid environment variable name: \"" + name + "\""); + return name; + } + + private static String validateValue(String value) { + if (value.indexOf('\u0000') != -1) + throw new IllegalArgumentException + ("Invalid environment variable value: \"" + value + "\""); + return value; + } + + private static String nonNullString(Object o) { + if (o == null) + throw new NullPointerException(); + return (String) o; + } + + public String put(String key, String value) { + return super.put(validateName(key), validateValue(value)); + } + + public String get(Object key) { + return super.get(nonNullString(key)); + } + + public boolean containsKey(Object key) { + return super.containsKey(nonNullString(key)); + } + + public boolean containsValue(Object value) { + return super.containsValue(nonNullString(value)); + } + + public String remove(Object key) { + return super.remove(nonNullString(key)); + } + + private static class CheckedEntry implements Entry { + private final Entry e; + public CheckedEntry(Entry e) {this.e = e;} + public String getKey() { return e.getKey();} + public String getValue() { return e.getValue();} + public String setValue(String value) { + return e.setValue(validateValue(value)); + } + public String toString() { return getKey() + "=" + getValue();} + public boolean equals(Object o) {return e.equals(o);} + public int hashCode() {return e.hashCode();} + } + + private static class CheckedEntrySet extends AbstractSet> { + private final Set> s; + public CheckedEntrySet(Set> s) {this.s = s;} + public int size() {return s.size();} + public boolean isEmpty() {return s.isEmpty();} + public void clear() { s.clear();} + public Iterator> iterator() { + return new Iterator>() { + Iterator> i = s.iterator(); + public boolean hasNext() { return i.hasNext();} + public Entry next() { + return new CheckedEntry(i.next()); + } + public void remove() { i.remove();} + }; + } + private static Entry checkedEntry(Object o) { + @SuppressWarnings("unchecked") + Entry e = (Entry) o; + nonNullString(e.getKey()); + nonNullString(e.getValue()); + return e; + } + public boolean contains(Object o) {return s.contains(checkedEntry(o));} + public boolean remove(Object o) {return s.remove(checkedEntry(o));} + } + + private static class CheckedValues extends AbstractCollection { + private final Collection c; + public CheckedValues(Collection c) {this.c = c;} + public int size() {return c.size();} + public boolean isEmpty() {return c.isEmpty();} + public void clear() { c.clear();} + public Iterator iterator() {return c.iterator();} + public boolean contains(Object o) {return c.contains(nonNullString(o));} + public boolean remove(Object o) {return c.remove(nonNullString(o));} + } + + private static class CheckedKeySet extends AbstractSet { + private final Set s; + public CheckedKeySet(Set s) {this.s = s;} + public int size() {return s.size();} + public boolean isEmpty() {return s.isEmpty();} + public void clear() { s.clear();} + public Iterator iterator() {return s.iterator();} + public boolean contains(Object o) {return s.contains(nonNullString(o));} + public boolean remove(Object o) {return s.remove(nonNullString(o));} + } + + public Set keySet() { + return new CheckedKeySet(super.keySet()); + } + + public Collection values() { + return new CheckedValues(super.values()); + } + + public Set> entrySet() { + return new CheckedEntrySet(super.entrySet()); + } + + private static final class NameComparator implements Comparator { + public int compare(String s1, String s2) { + // We can't use String.compareToIgnoreCase since it + // canonicalizes to lower case, while Windows + // canonicalizes to upper case! For example, "_" should + // sort *after* "Z", not before. + int n1 = s1.length(); + int n2 = s2.length(); + int min = Math.min(n1, n2); + for (int i = 0; i < min; i++) { + char c1 = s1.charAt(i); + char c2 = s2.charAt(i); + if (c1 != c2) { + c1 = Character.toUpperCase(c1); + c2 = Character.toUpperCase(c2); + if (c1 != c2) + // No overflow because of numeric promotion + return c1 - c2; + } + } + return n1 - n2; + } + } + + private static final class EntryComparator implements Comparator> { + public int compare(Entry e1, + Entry e2) { + return nameComparator.compare(e1.getKey(), e2.getKey()); + } + } + + // Allow `=' as first char in name, e.g. =C:=C:\DIR + static final int MIN_NAME_LENGTH = 1; + + private static final NameComparator nameComparator; + private static final EntryComparator entryComparator; + private static final ProcessEnvironmentForWin32 theEnvironment; + private static final Map theUnmodifiableEnvironment; + private static final Map theCaseInsensitiveEnvironment; + + static { + nameComparator = new NameComparator(); + entryComparator = new EntryComparator(); + theEnvironment = new ProcessEnvironmentForWin32(); + theUnmodifiableEnvironment = Collections.unmodifiableMap(theEnvironment); + + theEnvironment.putAll(environmentBlock()); + + theCaseInsensitiveEnvironment = new TreeMap<>(nameComparator); + theCaseInsensitiveEnvironment.putAll(theEnvironment); + } + + private ProcessEnvironmentForWin32() { + super(); + } + + private ProcessEnvironmentForWin32(int capacity) { + super(capacity); + } + + // Only for use by System.getenv(String) + static String getenv(String name) { + // The original implementation used a native call to _wgetenv, + // but it turns out that _wgetenv is only consistent with + // GetEnvironmentStringsW (for non-ASCII) if `wmain' is used + // instead of `main', even in a process created using + // CREATE_UNICODE_ENVIRONMENT. Instead we perform the + // case-insensitive comparison ourselves. At least this + // guarantees that System.getenv().get(String) will be + // consistent with System.getenv(String). + return theCaseInsensitiveEnvironment.get(name); + } + + // Only for use by System.getenv() + static Map getenv() { + return theUnmodifiableEnvironment; + } + + // Only for use by ProcessBuilder.environment() + @SuppressWarnings("unchecked") + static Map environment() { + return (Map) theEnvironment.clone(); + } + + // Only for use by ProcessBuilder.environment(String[] envp) + static Map emptyEnvironment(int capacity) { + return new ProcessEnvironmentForWin32(capacity); + } + + private static Map environmentBlock() { + return Kernel32Util.getEnvironmentVariables(); + } + + // Only for use by ProcessImpl.start() + String toEnvironmentBlock() { + // Sort Unicode-case-insensitively by name + List> list = new ArrayList<>(entrySet()); + Collections.sort(list, entryComparator); + + StringBuilder sb = new StringBuilder(size()*30); + int cmp = -1; + + // Some versions of MSVCRT.DLL require SystemRoot to be set. + // So, we make sure that it is always set, even if not provided + // by the caller. + final String SYSTEMROOT = "SystemRoot"; + + for (Entry e : list) { + String key = e.getKey(); + String value = e.getValue(); + if (cmp < 0 && (cmp = nameComparator.compare(key, SYSTEMROOT)) > 0) { + // Not set, so add it here + addToEnvIfSet(sb, SYSTEMROOT); + } + addToEnv(sb, key, value); + } + if (cmp < 0) { + // Got to end of list and still not found + addToEnvIfSet(sb, SYSTEMROOT); + } + if (sb.length() == 0) { + // Environment was empty and SystemRoot not set in parent + sb.append('\u0000'); + } + // Block is double NUL terminated + sb.append('\u0000'); + return sb.toString(); + } + + // add the environment variable to the child, if it exists in parent + private static void addToEnvIfSet(StringBuilder sb, String name) { + String s = getenv(name); + if (s != null) + addToEnv(sb, name, s); + } + + private static void addToEnv(StringBuilder sb, String name, String val) { + sb.append(name).append('=').append(val).append('\u0000'); + } + + static String toEnvironmentBlock(Map map) { + return map == null ? null : ((ProcessEnvironmentForWin32)map).toEnvironmentBlock(); + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java new file mode 100644 index 0000000000..4583be8aff --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java @@ -0,0 +1,752 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils.process; + +import com.sun.jna.Pointer; +import com.sun.jna.platform.win32.*; +import com.sun.jna.ptr.IntByReference; +import sun.security.action.GetPropertyAction; + +import java.io.*; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.sun.jna.platform.win32.WinBase.STILL_ACTIVE; + +public class ProcessImplForWin32 extends Process { + private static final sun.misc.JavaIOFileDescriptorAccess fdAccess + = sun.misc.SharedSecrets.getJavaIOFileDescriptorAccess(); + + private static final int PIPE_SIZE = 4096 + 24; + + private static final int HANDLE_STORAGE_SIZE = 6; + + private static final int OFFSET_READ = 0; + + private static final int OFFSET_WRITE = 1; + + private static final WinNT.HANDLE JAVA_INVALID_HANDLE_VALUE = new WinNT.HANDLE(Pointer.createConstant(-1)); + + /** + * Open a file for writing. If {@code append} is {@code true} then the file + * is opened for atomic append directly and a FileOutputStream constructed + * with the resulting handle. This is because a FileOutputStream created + * to append to a file does not open the file in a manner that guarantees + * that writes by the child process will be atomic. + */ + private static FileOutputStream newFileOutputStream(File f, boolean append) + throws IOException + { + if (append) { + String path = f.getPath(); + SecurityManager sm = System.getSecurityManager(); + if (sm != null) + sm.checkWrite(path); + long handle = openForAtomicAppend(path); + final FileDescriptor fd = new FileDescriptor(); + fdAccess.setHandle(fd, handle); + return AccessController.doPrivileged( + new PrivilegedAction() { + public FileOutputStream run() { + return new FileOutputStream(fd); + } + } + ); + } else { + return new FileOutputStream(f); + } + } + + // System-dependent portion of ProcessBuilderForWindows.start() + static Process start(String username, + String password, + String cmdarray[], + java.util.Map environment, + String dir, + ProcessBuilderForWin32.Redirect[] redirects, + boolean redirectErrorStream) + throws IOException + { + String envblock = ProcessEnvironmentForWin32.toEnvironmentBlock(environment); + + FileInputStream f0 = null; + FileOutputStream f1 = null; + FileOutputStream f2 = null; + + try { + long[] stdHandles; + if (redirects == null) { + stdHandles = new long[] { -1L, -1L, -1L }; + } else { + stdHandles = new long[3]; + + if (redirects[0] == ProcessBuilderForWin32.Redirect.PIPE) + stdHandles[0] = -1L; + else if (redirects[0] == ProcessBuilderForWin32.Redirect.INHERIT) + stdHandles[0] = fdAccess.getHandle(FileDescriptor.in); + else { + f0 = new FileInputStream(redirects[0].file()); + stdHandles[0] = fdAccess.getHandle(f0.getFD()); + } + + if (redirects[1] == ProcessBuilderForWin32.Redirect.PIPE) + stdHandles[1] = -1L; + else if (redirects[1] == ProcessBuilderForWin32.Redirect.INHERIT) + stdHandles[1] = fdAccess.getHandle(FileDescriptor.out); + else { + f1 = newFileOutputStream(redirects[1].file(), + redirects[1].append()); + stdHandles[1] = fdAccess.getHandle(f1.getFD()); + } + + if (redirects[2] == ProcessBuilderForWin32.Redirect.PIPE) + stdHandles[2] = -1L; + else if (redirects[2] == ProcessBuilderForWin32.Redirect.INHERIT) + stdHandles[2] = fdAccess.getHandle(FileDescriptor.err); + else { + f2 = newFileOutputStream(redirects[2].file(), + redirects[2].append()); + stdHandles[2] = fdAccess.getHandle(f2.getFD()); + } + } + + return new ProcessImplForWin32(username, password, cmdarray, envblock, dir, stdHandles, redirectErrorStream); + } finally { + // In theory, close() can throw IOException + // (although it is rather unlikely to happen here) + try { if (f0 != null) f0.close(); } + finally { + try { if (f1 != null) f1.close(); } + finally { if (f2 != null) f2.close(); } + } + } + + } + + private static class LazyPattern { + // Escape-support version: + // "(\")((?:\\\\\\1|.)+?)\\1|([^\\s\"]+)"; + private static final Pattern PATTERN = + Pattern.compile("[^\\s\"]+|\"[^\"]*\""); + }; + + /* Parses the command string parameter into the executable name and + * program arguments. + * + * The command string is broken into tokens. The token separator is a space + * or quota character. The space inside quotation is not a token separator. + * There are no escape sequences. + */ + private static String[] getTokensFromCommand(String command) { + ArrayList matchList = new ArrayList<>(8); + Matcher regexMatcher = ProcessImplForWin32.LazyPattern.PATTERN.matcher(command); + while (regexMatcher.find()) + matchList.add(regexMatcher.group()); + return matchList.toArray(new String[matchList.size()]); + } + + private static final int VERIFICATION_CMD_BAT = 0; + private static final int VERIFICATION_WIN32 = 1; + private static final int VERIFICATION_WIN32_SAFE = 2; // inside quotes not allowed + private static final int VERIFICATION_LEGACY = 3; + // See Command shell overview for documentation of special characters. + // https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-xp/bb490954(v=technet.10) + private static final char ESCAPE_VERIFICATION[][] = { + // We guarantee the only command file execution for implicit [cmd.exe] run. + // http://technet.microsoft.com/en-us/library/bb490954.aspx + {' ', '\t', '<', '>', '&', '|', '^'}, + {' ', '\t', '<', '>'}, + {' ', '\t', '<', '>'}, + {' ', '\t'} + }; + + private static String createCommandLine(int verificationType, + final String executablePath, + final String cmd[]) + { + StringBuilder cmdbuf = new StringBuilder(80); + + cmdbuf.append(executablePath); + + for (int i = 1; i < cmd.length; ++i) { + cmdbuf.append(' '); + String s = cmd[i]; + if (needsEscaping(verificationType, s)) { + cmdbuf.append('"'); + + if (verificationType == VERIFICATION_WIN32_SAFE) { + // Insert the argument, adding '\' to quote any interior quotes + int length = s.length(); + for (int j = 0; j < length; j++) { + char c = s.charAt(j); + if (c == DOUBLEQUOTE) { + int count = countLeadingBackslash(verificationType, s, j); + while (count-- > 0) { + cmdbuf.append(BACKSLASH); // double the number of backslashes + } + cmdbuf.append(BACKSLASH); // backslash to quote the quote + } + cmdbuf.append(c); + } + } else { + cmdbuf.append(s); + } + // The code protects the [java.exe] and console command line + // parser, that interprets the [\"] combination as an escape + // sequence for the ["] char. + // http://msdn.microsoft.com/en-us/library/17w5ykft.aspx + // + // If the argument is an FS path, doubling of the tail [\] + // char is not a problem for non-console applications. + // + // The [\"] sequence is not an escape sequence for the [cmd.exe] + // command line parser. The case of the [""] tail escape + // sequence could not be realized due to the argument validation + // procedure. + int count = countLeadingBackslash(verificationType, s, s.length()); + while (count-- > 0) { + cmdbuf.append(BACKSLASH); // double the number of backslashes + } + cmdbuf.append('"'); + } else { + cmdbuf.append(s); + } + } + return cmdbuf.toString(); + } + + /** + * Return the argument without quotes (1st and last) if present, else the arg. + * @param str a string + * @return the string without 1st and last quotes + */ + private static String unQuote(String str) { + int len = str.length(); + return (len >= 2 && str.charAt(0) == DOUBLEQUOTE && str.charAt(len - 1) == DOUBLEQUOTE) + ? str.substring(1, len - 1) + : str; + } + + private static boolean needsEscaping(int verificationType, String arg) { + // Switch off MS heuristic for internal ["]. + // Please, use the explicit [cmd.exe] call + // if you need the internal ["]. + // Example: "cmd.exe", "/C", "Extended_MS_Syntax" + + // For [.exe] or [.com] file the unpaired/internal ["] + // in the argument is not a problem. + String unquotedArg = unQuote(arg); + boolean argIsQuoted = !arg.equals(unquotedArg); + boolean embeddedQuote = unquotedArg.indexOf(DOUBLEQUOTE) >= 0; + + switch (verificationType) { + case VERIFICATION_CMD_BAT: + if (embeddedQuote) { + throw new IllegalArgumentException("Argument has embedded quote, " + + "use the explicit CMD.EXE call."); + } + break; // break determine whether to quote + case VERIFICATION_WIN32_SAFE: + if (argIsQuoted && embeddedQuote) { + throw new IllegalArgumentException("Malformed argument has embedded quote: " + + unquotedArg); + } + break; + default: + break; + } + + if (!argIsQuoted) { + char testEscape[] = ESCAPE_VERIFICATION[verificationType]; + for (int i = 0; i < testEscape.length; ++i) { + if (arg.indexOf(testEscape[i]) >= 0) { + return true; + } + } + } + return false; + } + + private static String getExecutablePath(String path) + throws IOException + { + String name = unQuote(path); + if (name.indexOf(DOUBLEQUOTE) >= 0) { + throw new IllegalArgumentException("Executable name has embedded quote, " + + "split the arguments: " + name); + } + // Win32 CreateProcess requires path to be normalized + File fileToRun = new File(name); + + // From the [CreateProcess] function documentation: + // + // "If the file name does not contain an extension, .exe is appended. + // Therefore, if the file name extension is .com, this parameter + // must include the .com extension. If the file name ends in + // a period (.) with no extension, or if the file name contains a path, + // .exe is not appended." + // + // "If the file name !does not contain a directory path!, + // the system searches for the executable file in the following + // sequence:..." + // + // In practice ANY non-existent path is extended by [.exe] extension + // in the [CreateProcess] function with the only exception: + // the path ends by (.) + + return fileToRun.getPath(); + } + + /** + * An executable is any program that is an EXE or does not have an extension + * and the Windows createProcess will be looking for .exe. + * The comparison is case insensitive based on the name. + * @param executablePath the executable file + * @return true if the path ends in .exe or does not have an extension. + */ + private boolean isExe(String executablePath) { + File file = new File(executablePath); + String upName = file.getName().toUpperCase(Locale.ROOT); + return (upName.endsWith(".EXE") || upName.indexOf('.') < 0); + } + + // Old version that can be bypassed + private boolean isShellFile(String executablePath) { + String upPath = executablePath.toUpperCase(); + return (upPath.endsWith(".CMD") || upPath.endsWith(".BAT")); + } + + private String quoteString(String arg) { + StringBuilder argbuf = new StringBuilder(arg.length() + 2); + return argbuf.append('"').append(arg).append('"').toString(); + } + + // Count backslashes before start index of string. + // .bat files don't include backslashes as part of the quote + private static int countLeadingBackslash(int verificationType, + CharSequence input, int start) { + if (verificationType == VERIFICATION_CMD_BAT) + return 0; + int j; + for (j = start - 1; j >= 0 && input.charAt(j) == BACKSLASH; j--) { + // just scanning backwards + } + return (start - 1) - j; // number of BACKSLASHES + } + + private static final char DOUBLEQUOTE = '\"'; + private static final char BACKSLASH = '\\'; + + private WinNT.HANDLE handle; + private OutputStream stdin_stream; + private InputStream stdout_stream; + private InputStream stderr_stream; + + private ProcessImplForWin32( + String username, + String password, + String cmd[], + final String envblock, + final String path, + final long[] stdHandles, + final boolean redirectErrorStream) + throws IOException + { + String cmdstr; + final SecurityManager security = System.getSecurityManager(); + GetPropertyAction action = new GetPropertyAction("jdk.lang.Process.allowAmbiguousCommands", + (security == null) ? "true" : "false"); + final boolean allowAmbiguousCommands = !"false".equalsIgnoreCase(action.run()); + if (allowAmbiguousCommands && security == null) { + // Legacy mode. + + // Normalize path if possible. + String executablePath = new File(cmd[0]).getPath(); + + // No worry about internal, unpaired ["], and redirection/piping. + if (needsEscaping(VERIFICATION_LEGACY, executablePath) ) + executablePath = quoteString(executablePath); + + cmdstr = createCommandLine( + //legacy mode doesn't worry about extended verification + VERIFICATION_LEGACY, + executablePath, + cmd); + } else { + String executablePath; + try { + executablePath = getExecutablePath(cmd[0]); + } catch (IllegalArgumentException e) { + // Workaround for the calls like + // Runtime.getRuntime().exec("\"C:\\Program Files\\foo\" bar") + + // No chance to avoid CMD/BAT injection, except to do the work + // right from the beginning. Otherwise we have too many corner + // cases from + // Runtime.getRuntime().exec(String[] cmd [, ...]) + // calls with internal ["] and escape sequences. + + // Restore original command line. + StringBuilder join = new StringBuilder(); + // terminal space in command line is ok + for (String s : cmd) + join.append(s).append(' '); + + // Parse the command line again. + cmd = getTokensFromCommand(join.toString()); + executablePath = getExecutablePath(cmd[0]); + + // Check new executable name once more + if (security != null) + security.checkExec(executablePath); + } + + // Quotation protects from interpretation of the [path] argument as + // start of longer path with spaces. Quotation has no influence to + // [.exe] extension heuristic. + boolean isShell = allowAmbiguousCommands ? isShellFile(executablePath) + : !isExe(executablePath); + cmdstr = createCommandLine( + // We need the extended verification procedures + isShell ? VERIFICATION_CMD_BAT + : (allowAmbiguousCommands ? VERIFICATION_WIN32 : VERIFICATION_WIN32_SAFE), + quoteString(executablePath), + cmd); + } + + handle = create(username, password, cmdstr, envblock, path, stdHandles, redirectErrorStream); + + AccessController.doPrivileged( + new PrivilegedAction() { + public Void run() { + if (stdHandles[0] == -1L) + stdin_stream = ProcessBuilderForWin32.NullOutputStream.INSTANCE; + else { + FileDescriptor stdin_fd = new FileDescriptor(); + fdAccess.setHandle(stdin_fd, stdHandles[0]); + stdin_stream = new BufferedOutputStream( + new FileOutputStream(stdin_fd)); + } + + if (stdHandles[1] == -1L) + stdout_stream = ProcessBuilderForWin32.NullInputStream.INSTANCE; + else { + FileDescriptor stdout_fd = new FileDescriptor(); + fdAccess.setHandle(stdout_fd, stdHandles[1]); + stdout_stream = new BufferedInputStream( + new FileInputStream(stdout_fd)); + } + + if (stdHandles[2] == -1L) + stderr_stream = ProcessBuilderForWin32.NullInputStream.INSTANCE; + else { + FileDescriptor stderr_fd = new FileDescriptor(); + fdAccess.setHandle(stderr_fd, stdHandles[2]); + stderr_stream = new FileInputStream(stderr_fd); + } + + return null; }}); + } + + public OutputStream getOutputStream() { + return stdin_stream; + } + + public InputStream getInputStream() { + return stdout_stream; + } + + public InputStream getErrorStream() { + return stderr_stream; + } + + protected void finalize() { + closeHandle(handle); + } + + public int exitValue() { + int exitCode = getExitCodeProcess(handle); + if (exitCode == STILL_ACTIVE) + throw new IllegalThreadStateException("process has not exited"); + return exitCode; + } + + public int waitFor() throws InterruptedException { + waitForInterruptibly(handle); + if (Thread.interrupted()) + throw new InterruptedException(); + return exitValue(); + } + + @Override + public boolean waitFor(long timeout, TimeUnit unit) + throws InterruptedException + { + if (getExitCodeProcess(handle) != STILL_ACTIVE) return true; + if (timeout <= 0) return false; + + long remainingNanos = unit.toNanos(timeout); + long deadline = System.nanoTime() + remainingNanos ; + + do { + // Round up to next millisecond + long msTimeout = TimeUnit.NANOSECONDS.toMillis(remainingNanos + 999_999L); + waitForTimeoutInterruptibly(handle, msTimeout); + if (Thread.interrupted()) + throw new InterruptedException(); + if (getExitCodeProcess(handle) != STILL_ACTIVE) { + return true; + } + remainingNanos = deadline - System.nanoTime(); + } while (remainingNanos > 0); + + return (getExitCodeProcess(handle) != STILL_ACTIVE); + } + + public void destroy() { terminateProcess(handle); } + + public Process destroyForcibly() { + destroy(); + return this; + } + + public boolean isAlive() { + return isProcessAlive(handle); + } + + private static boolean initHolder(WinNT.HANDLEByReference pjhandles, + WinNT.HANDLEByReference[] pipe, + int offset, + WinNT.HANDLEByReference phStd) { + if (!pjhandles.getValue().equals(JAVA_INVALID_HANDLE_VALUE)) { + phStd.setValue(pjhandles.getValue()); + pjhandles.setValue(JAVA_INVALID_HANDLE_VALUE); + } else { + if (!Kernel32.INSTANCE.CreatePipe(pipe[0], pipe[1], null, PIPE_SIZE)) { + throw new Win32Exception(Kernel32.INSTANCE.GetLastError()); + } else { + WinNT.HANDLE thisProcessEnd = offset == OFFSET_READ ? pipe[1].getValue() : pipe[0].getValue(); + phStd.setValue(pipe[offset].getValue()); + pjhandles.setValue(thisProcessEnd); + } + } + Kernel32.INSTANCE.SetHandleInformation(phStd.getValue(), Kernel32.HANDLE_FLAG_INHERIT, Kernel32.HANDLE_FLAG_INHERIT); + return true; + } + + private static void releaseHolder(boolean complete, WinNT.HANDLEByReference[] pipe, int offset) { + closeHandle(pipe[offset].getValue()); + if (complete) { + closeHandle(pipe[offset == OFFSET_READ ? OFFSET_WRITE : OFFSET_READ].getValue()); + } + } + + private static void prepareIOEHandleState(WinNT.HANDLE[] stdIOE, Boolean[] inherit) { + for(int i = 0; i < HANDLE_STORAGE_SIZE; ++i) { + WinNT.HANDLE hstd = stdIOE[i]; + if (!Kernel32.INVALID_HANDLE_VALUE.equals(hstd)) { + inherit[i] = Boolean.TRUE; + Kernel32.INSTANCE.SetHandleInformation(hstd, Kernel32.HANDLE_FLAG_INHERIT, 0); + } + } + } + + private static void restoreIOEHandleState(WinNT.HANDLE[] stdIOE, Boolean[] inherit) { + for (int i = HANDLE_STORAGE_SIZE - 1; i >= 0; --i) { + if (!Kernel32.INVALID_HANDLE_VALUE.equals(stdIOE[i])) { + Kernel32.INSTANCE.SetHandleInformation(stdIOE[i], Kernel32.HANDLE_FLAG_INHERIT, inherit[i] ? Kernel32.HANDLE_FLAG_INHERIT : 0); + } + } + } + + private static WinNT.HANDLE processCreate(String username, + String password, + String cmd, + final String envblock, + final String path, + final WinNT.HANDLEByReference[] stdHandles, + final boolean redirectErrorStream) { + WinNT.HANDLE ret = new WinNT.HANDLE(Pointer.createConstant(0)); + + WinNT.HANDLE[] stdIOE = new WinNT.HANDLE[] { + Kernel32.INVALID_HANDLE_VALUE, Kernel32.INVALID_HANDLE_VALUE, Kernel32.INVALID_HANDLE_VALUE, + stdHandles[0].getValue(), stdHandles[1].getValue(), stdHandles[2].getValue() + }; + stdIOE[0] = Kernel32.INSTANCE.GetStdHandle(Kernel32.STD_INPUT_HANDLE); + stdIOE[1] = Kernel32.INSTANCE.GetStdHandle(Kernel32.STD_OUTPUT_HANDLE); + stdIOE[2] = Kernel32.INSTANCE.GetStdHandle(Kernel32.STD_ERROR_HANDLE); + + Boolean[] inherit = new Boolean[] { + Boolean.FALSE, Boolean.FALSE, Boolean.FALSE, + Boolean.FALSE, Boolean.FALSE, Boolean.FALSE + }; + + prepareIOEHandleState(stdIOE, inherit); + + // input + WinNT.HANDLEByReference hStdInput = new WinNT.HANDLEByReference(); + WinNT.HANDLEByReference[] pipeIn = new WinNT.HANDLEByReference[] { + new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE), new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE) }; + + // output + WinNT.HANDLEByReference hStdOutput = new WinNT.HANDLEByReference(); + WinNT.HANDLEByReference[] pipeOut = new WinNT.HANDLEByReference[] { + new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE), new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE) }; + + // error + WinNT.HANDLEByReference hStdError = new WinNT.HANDLEByReference(); + WinNT.HANDLEByReference[] pipeError = new WinNT.HANDLEByReference[] { + new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE), new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE) }; + + boolean success; + if (initHolder(stdHandles[0], pipeIn, OFFSET_READ, hStdInput)) { + if (initHolder(stdHandles[1], pipeOut, OFFSET_WRITE, hStdOutput)) { + WinBase.STARTUPINFO si = new WinBase.STARTUPINFO(); + si.hStdInput = hStdInput.getValue(); + si.hStdOutput = hStdOutput.getValue(); + + if (redirectErrorStream) { + si.hStdError = si.hStdOutput; + stdHandles[2].setValue(JAVA_INVALID_HANDLE_VALUE); + success = true; + } else { + success = initHolder(stdHandles[2], pipeError, OFFSET_WRITE, hStdError); + si.hStdError = hStdError.getValue(); + } + + if (success) { + WTypes.LPSTR lpEnvironment = envblock == null ? new WTypes.LPSTR() : new WTypes.LPSTR(envblock); + Kernel32.PROCESS_INFORMATION pi = new WinBase.PROCESS_INFORMATION(); + si.dwFlags = Kernel32.STARTF_USESTDHANDLES; + if (!Advapi32.INSTANCE.CreateProcessWithLogonW( + username + , null + , password + , Advapi32.LOGON_WITH_PROFILE + , null + , cmd + , Kernel32.CREATE_NO_WINDOW + , lpEnvironment.getPointer() + , path + , si + , pi)) { + throw new Win32Exception(Kernel32.INSTANCE.GetLastError()); + } else { + closeHandle(pi.hThread); + ret = pi.hProcess; + } + } + releaseHolder(ret.getPointer().equals(Pointer.createConstant(0)), pipeError, OFFSET_WRITE); + releaseHolder(ret.getPointer().equals(Pointer.createConstant(0)), pipeOut, OFFSET_WRITE); + } + releaseHolder(ret.getPointer().equals(Pointer.createConstant(0)), pipeIn, OFFSET_READ); + } + restoreIOEHandleState(stdIOE, inherit); + return ret; + } + + private static synchronized WinNT.HANDLE create(String username, + String password, + String cmd, + final String envblock, + final String path, + final long[] stdHandles, + final boolean redirectErrorStream) { + WinNT.HANDLE ret = new WinNT.HANDLE(Pointer.createConstant(0)); + WinNT.HANDLEByReference[] handles = new WinNT.HANDLEByReference[stdHandles.length]; + for (int i = 0; i < stdHandles.length; i++) { + handles[i] = new WinNT.HANDLEByReference(new WinNT.HANDLE(Pointer.createConstant(stdHandles[i]))); + } + + if (cmd != null) { + if (username != null && password != null) { + ret = processCreate(username, password, cmd, envblock, path, handles, redirectErrorStream); + } + } + + for (int i = 0; i < stdHandles.length; i++) { + stdHandles[i] = handles[i].getPointer().getLong(0); + } + + return ret; + } + + private static int getExitCodeProcess(WinNT.HANDLE handle) { + IntByReference exitStatus = new IntByReference(); + if (!Kernel32.INSTANCE.GetExitCodeProcess(handle, exitStatus)) { + throw new Win32Exception(Kernel32.INSTANCE.GetLastError()); + } + return exitStatus.getValue(); + } + + private static void terminateProcess(WinNT.HANDLE handle) { + Kernel32.INSTANCE.TerminateProcess(handle, 1); + } + + private static boolean isProcessAlive(WinNT.HANDLE handle) { + IntByReference exitStatus = new IntByReference(); + Kernel32.INSTANCE.GetExitCodeProcess(handle, exitStatus); + return exitStatus.getValue() == STILL_ACTIVE; + } + + private static void closeHandle(WinNT.HANDLE handle) { + Kernel32Util.closeHandle(handle); + } + + /** + * Opens a file for atomic append. The file is created if it doesn't + * already exist. + * + * @param path the file to open or create + * @return the native HANDLE + */ + private static long openForAtomicAppend(String path) throws IOException { + int access = Kernel32.GENERIC_READ | Kernel32.GENERIC_WRITE; + int sharing = Kernel32.FILE_SHARE_READ | Kernel32.FILE_SHARE_WRITE; + int disposition = Kernel32.OPEN_ALWAYS; + int flagsAndAttributes = Kernel32.FILE_ATTRIBUTE_NORMAL; + if (path == null || path.isEmpty()) { + return -1; + } else { + WinNT.HANDLE handle = Kernel32.INSTANCE.CreateFile(path, access, sharing, null, disposition, flagsAndAttributes, null); + if (handle == Kernel32.INVALID_HANDLE_VALUE) { + throw new Win32Exception(Kernel32.INSTANCE.GetLastError()); + } + return handle.getPointer().getLong(0); + } + } + + private static void waitForInterruptibly(WinNT.HANDLE handle) { + int result = Kernel32.INSTANCE.WaitForMultipleObjects(1, new WinNT.HANDLE[]{handle}, false, Kernel32.INFINITE); + if (result == Kernel32.WAIT_FAILED) { + throw new Win32Exception(Kernel32.INSTANCE.GetLastError()); + } + } + + private static void waitForTimeoutInterruptibly(WinNT.HANDLE handle, long timeout) { + int result = Kernel32.INSTANCE.WaitForMultipleObjects(1, new WinNT.HANDLE[]{handle}, false, (int) timeout); + if (result == Kernel32.WAIT_FAILED) { + throw new Win32Exception(Kernel32.INSTANCE.GetLastError()); + } + } + +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java new file mode 100644 index 0000000000..3280a9629f --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common; + +import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.junit.Assert; +import org.junit.Test; + +/** + * Constants Test + */ +public class ConstantsTest { + + /** + * Test PID via env + */ + @Test + public void testPID() { + if (OSUtils.isWindows()) { + Assert.assertEquals(Constants.PID, "handle"); + } else { + Assert.assertEquals(Constants.PID, "pid"); + } + } + +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java index 7106804aaf..b955787c69 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java @@ -39,16 +39,20 @@ public class OSUtilsTest { @Test public void testOSMetric(){ - double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); - Assert.assertTrue(availablePhysicalMemorySize > 0.0f); - double totalMemorySize = OSUtils.totalMemorySize(); - Assert.assertTrue(totalMemorySize > 0.0f); - double loadAverage = OSUtils.loadAverage(); - logger.info("loadAverage {}", loadAverage); - double memoryUsage = OSUtils.memoryUsage(); - Assert.assertTrue(memoryUsage > 0.0f); - double cpuUsage = OSUtils.cpuUsage(); - Assert.assertTrue(cpuUsage > 0.0f); + if (!OSUtils.isWindows()) { + double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); + Assert.assertTrue(availablePhysicalMemorySize > 0.0f); + double totalMemorySize = OSUtils.totalMemorySize(); + Assert.assertTrue(totalMemorySize > 0.0f); + double loadAverage = OSUtils.loadAverage(); + logger.info("loadAverage {}", loadAverage); + double memoryUsage = OSUtils.memoryUsage(); + Assert.assertTrue(memoryUsage > 0.0f); + double cpuUsage = OSUtils.cpuUsage(); + Assert.assertTrue(cpuUsage > 0.0f); + } else { + // TODO window ut + } } @Test diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32Test.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32Test.java new file mode 100644 index 0000000000..ce04346743 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32Test.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils.process; + +import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +@RunWith(PowerMockRunner.class) +@PrepareForTest(OSUtils.class) +public class ProcessBuilderForWin32Test { + + private static final Logger logger = LoggerFactory.getLogger(ProcessBuilderForWin32Test.class); + + @Before + public void before() { + PowerMockito.mockStatic(OSUtils.class); + PowerMockito.when(OSUtils.isWindows()).thenReturn(true); + } + + @Test + public void testCreateProcessBuilderForWin32() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + Assert.assertNotNull(builder); + + builder = new ProcessBuilderForWin32("net"); + Assert.assertNotNull(builder); + + builder = new ProcessBuilderForWin32(Collections.singletonList("net")); + Assert.assertNotNull(builder); + + builder = new ProcessBuilderForWin32((List) null); + Assert.assertNotNull(builder); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testBuildUser() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + builder.user("test", StringUtils.EMPTY); + Assert.assertNotNull(builder); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testBuildCommand() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + builder.command(Collections.singletonList("net")); + Assert.assertNotEquals(0, builder.command().size()); + + builder = new ProcessBuilderForWin32(); + builder.command("net"); + Assert.assertNotEquals(0, builder.command().size()); + + builder = new ProcessBuilderForWin32(); + builder.command((List) null); + Assert.assertNotEquals(0, builder.command().size()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testEnvironment() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + Assert.assertNotNull(builder.environment()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + builder.environment(new String[]{ "a=123" }); + Assert.assertNotEquals(0, builder.environment().size()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testDirectory() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + builder.directory(new File("/tmp")); + Assert.assertNotNull(builder.directory()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testStream() { + try { + InputStream in = ProcessBuilderForWin32.NullInputStream.INSTANCE; + Assert.assertNotNull(in); + Assert.assertEquals(-1, in.read()); + Assert.assertEquals(0, in.available()); + + OutputStream out = ProcessBuilderForWin32.NullOutputStream.INSTANCE; + Assert.assertNotNull(out); + out.write(new byte[] {1}); + } catch (Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testRedirect() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + + builder.redirectInput(new File("/tmp")); + Assert.assertNotNull(builder.redirectInput()); + Assert.assertNotNull(builder.redirectInput().file()); + + builder.redirectOutput(new File("/tmp")); + Assert.assertNotNull(builder.redirectOutput()); + Assert.assertNotNull(builder.redirectOutput().file()); + + builder.redirectError(new File("/tmp")); + Assert.assertNotNull(builder.redirectError()); + Assert.assertNotNull(builder.redirectError().file()); + + builder.redirectInput(builder.redirectOutput()); + builder.redirectOutput(builder.redirectInput()); + builder.redirectError(builder.redirectInput()); + + Assert.assertNotNull(ProcessBuilderForWin32.Redirect.PIPE.type()); + Assert.assertNotNull(ProcessBuilderForWin32.Redirect.PIPE.toString()); + Assert.assertNotNull(ProcessBuilderForWin32.Redirect.INHERIT.type()); + Assert.assertNotNull(ProcessBuilderForWin32.Redirect.INHERIT.toString()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testRedirectErrorStream() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + builder.redirectErrorStream(true); + Assert.assertTrue(builder.redirectErrorStream()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void runCmdViaUser() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + builder.user("test123", StringUtils.EMPTY); + + List commands = new ArrayList<>(); + commands.add("cmd.exe"); + commands.add("/c"); + commands.add("net user"); + builder.command(commands); + + Process process = builder.start(); + BufferedReader inReader = new BufferedReader(new InputStreamReader(process.getInputStream(), Charset.forName("GBK"))); + String line; + StringBuilder sb = new StringBuilder(); + while ((line = inReader.readLine()) != null) { + sb.append(line); + } + logger.info("net user: {}", sb.toString()); + Assert.assertNotEquals(StringUtils.EMPTY, sb.toString()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32Test.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32Test.java new file mode 100644 index 0000000000..00c54c0164 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32Test.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils.process; + +import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.Map; +import java.util.Set; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({OSUtils.class, ProcessEnvironmentForWin32.class}) +public class ProcessEnvironmentForWin32Test { + + private static final Logger logger = LoggerFactory.getLogger(ProcessBuilderForWin32Test.class); + + @Before + public void before() { + try { + PowerMockito.mockStatic(OSUtils.class); + PowerMockito.when(OSUtils.isWindows()).thenReturn(true); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testPutAndGet() { + try { + ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0); + processEnvironmentForWin32.put("a", "123"); + Assert.assertEquals("123", processEnvironmentForWin32.get("a")); + Assert.assertTrue(processEnvironmentForWin32.containsKey("a")); + Assert.assertTrue(processEnvironmentForWin32.containsValue("123")); + Assert.assertEquals("123", processEnvironmentForWin32.remove("a")); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + + try { + ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0); + processEnvironmentForWin32.put("b=", "123"); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + + try { + ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0); + processEnvironmentForWin32.put("b", "\u0000"); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + + try { + ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0); + processEnvironmentForWin32.get(null); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testEntrySet() { + try { + ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0); + processEnvironmentForWin32.clear(); + processEnvironmentForWin32.put("a", "123"); + Assert.assertEquals(0, processEnvironmentForWin32.entrySet().size()); + Assert.assertTrue(processEnvironmentForWin32.entrySet().isEmpty()); + for (Map.Entry entry : processEnvironmentForWin32.entrySet()) { + Assert.assertNotNull(entry); + Assert.assertNotNull(entry.getKey()); + Assert.assertNotNull(entry.getValue()); + Assert.assertNotNull(entry.setValue("123")); + } + + processEnvironmentForWin32.clear(); + Set keys = processEnvironmentForWin32.keySet(); + Assert.assertEquals(0, keys.size()); + Assert.assertTrue(keys.isEmpty()); + + processEnvironmentForWin32.clear(); + Collection values = processEnvironmentForWin32.values(); + Assert.assertEquals(0, keys.size()); + Assert.assertTrue(keys.isEmpty()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testToEnvironmentBlock() { + try { + ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0); + Assert.assertNotNull(processEnvironmentForWin32.toEnvironmentBlock()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32Test.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32Test.java new file mode 100644 index 0000000000..3f8bcbfb66 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32Test.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils.process; + +import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sun.security.action.GetPropertyAction; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({OSUtils.class, GetPropertyAction.class}) +public class ProcessImplForWin32Test { + + private static final Logger logger = LoggerFactory.getLogger(ProcessBuilderForWin32Test.class); + + @Before + public void before() { + PowerMockito.mockStatic(OSUtils.class); + PowerMockito.mockStatic(GetPropertyAction.class); + PowerMockito.when(OSUtils.isWindows()).thenReturn(true); + } + + @Test + public void testStart() { + try { + Process process = ProcessImplForWin32.start( + "test123", StringUtils.EMPTY, new String[]{"net"}, + null, null, null, false); + Assert.assertNotNull(process); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + + try { + Process process = ProcessImplForWin32.start( + "test123", StringUtils.EMPTY, new String[]{"net"}, + null, null, new ProcessBuilderForWin32.Redirect[]{ + ProcessBuilderForWin32.Redirect.PIPE, + ProcessBuilderForWin32.Redirect.PIPE, + ProcessBuilderForWin32.Redirect.PIPE + }, false); + Assert.assertNotNull(process); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + +} diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index bac498c150..8e0ccee16c 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -16,21 +16,29 @@ */ package org.apache.dolphinscheduler.server.worker.task; +import com.sun.jna.platform.win32.Kernel32; +import com.sun.jna.platform.win32.WinNT; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils; +import org.apache.dolphinscheduler.common.utils.LoggerUtils; +import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.utils.process.ProcessBuilderForWin32; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.server.utils.ProcessUtils; import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import java.io.*; import java.lang.reflect.Field; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -194,26 +202,49 @@ public abstract class AbstractCommandExecutor { * @throws IOException IO Exception */ private void buildProcess(String commandFile) throws IOException { + // command list + List command = new ArrayList<>(); + //init process builder - ProcessBuilder processBuilder = new ProcessBuilder(); - // setting up a working directory - processBuilder.directory(new File(taskDir)); - // merge error information to standard output stream - processBuilder.redirectErrorStream(true); - // setting up user to run commands - List command = new LinkedList<>(); - command.add("sudo"); - command.add("-u"); - command.add(tenantCode); - command.add(commandInterpreter()); - command.addAll(commandOptions()); - command.add(commandFile); - processBuilder.command(command); - - process = processBuilder.start(); + if (OSUtils.isWindows()) { + ProcessBuilderForWin32 processBuilder = new ProcessBuilderForWin32(); + // setting up a working directory + processBuilder.directory(new File(taskDir)); + processBuilder.user(tenantCode, StringUtils.EMPTY); + // merge error information to standard output stream + processBuilder.redirectErrorStream(true); + + // setting up user to run commands + command.add(commandInterpreter()); + command.add("/c"); + command.addAll(commandOptions()); + command.add(commandFile); + + // setting commands + processBuilder.command(command); + process = processBuilder.start(); + } else { + ProcessBuilder processBuilder = new ProcessBuilder(); + // setting up a working directory + processBuilder.directory(new File(taskDir)); + // merge error information to standard output stream + processBuilder.redirectErrorStream(true); + + // setting up user to run commands + command.add("sudo"); + command.add("-u"); + command.add(tenantCode); + command.add(commandInterpreter()); + command.addAll(commandOptions()); + command.add(commandFile); + + // setting commands + processBuilder.command(command); + process = processBuilder.start(); + } // print command - printCommand(processBuilder); + printCommand(command); } /** @@ -320,13 +351,13 @@ public abstract class AbstractCommandExecutor { /** * print command - * @param processBuilder process builder + * @param command command */ - private void printCommand(ProcessBuilder processBuilder) { + private void printCommand(List command) { String cmdStr; try { - cmdStr = ProcessUtils.buildCommandStr(processBuilder.command()); + cmdStr = ProcessUtils.buildCommandStr(command); logger.info("task run command:\n{}", cmdStr); } catch (IOException e) { logger.error(e.getMessage(), e); @@ -358,7 +389,11 @@ public abstract class AbstractCommandExecutor { BufferedReader inReader = null; try { - inReader = new BufferedReader(new InputStreamReader(process.getInputStream())); + if (OSUtils.isWindows()) { + inReader = new BufferedReader(new InputStreamReader(process.getInputStream(), Charset.forName("GBK"))); + } else { + inReader = new BufferedReader(new InputStreamReader(process.getInputStream())); + } String line; long lastFlushTime = System.currentTimeMillis(); @@ -406,7 +441,7 @@ public abstract class AbstractCommandExecutor { } Thread.sleep(Constants.SLEEP_TIME_MILLIS); } - } + } } catch (Exception e) { logger.error("yarn applications: {} status failed ", appIds,e); result = false; @@ -510,12 +545,15 @@ public abstract class AbstractCommandExecutor { */ private int getProcessId(Process process) { int processId = 0; - try { Field f = process.getClass().getDeclaredField(Constants.PID); f.setAccessible(true); - - processId = f.getInt(process); + if (OSUtils.isWindows()) { + WinNT.HANDLE handle = (WinNT.HANDLE) f.get(process); + processId = Kernel32.INSTANCE.GetProcessId(handle); + } else { + processId = f.getInt(process); + } } catch (Throwable e) { logger.error(e.getMessage(), e); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java index db46d0d856..5d14e6b2a7 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java @@ -17,11 +17,12 @@ package org.apache.dolphinscheduler.server.worker.task; import org.apache.commons.io.FileUtils; +import org.apache.dolphinscheduler.common.utils.OSUtils; import org.slf4j.Logger; import java.io.File; import java.io.IOException; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Date; @@ -34,10 +35,15 @@ import java.util.function.Consumer; public class ShellCommandExecutor extends AbstractCommandExecutor { /** - * sh + * For Unix-like, using sh */ public static final String SH = "sh"; + /** + * For Windows, using cmd.exe + */ + public static final String CMD = "cmd.exe"; + /** * constructor * @param logHandler log handler @@ -66,7 +72,7 @@ public class ShellCommandExecutor extends AbstractCommandExecutor { @Override protected String buildCommandFilePath() { // command file - return String.format("%s/%s.command", taskDir, taskAppId); + return String.format("%s/%s.%s", taskDir, taskAppId, OSUtils.isWindows() ? "bat" : "command"); } /** @@ -75,7 +81,7 @@ public class ShellCommandExecutor extends AbstractCommandExecutor { */ @Override protected String commandInterpreter() { - return SH; + return OSUtils.isWindows() ? CMD : SH; } /** @@ -103,21 +109,26 @@ public class ShellCommandExecutor extends AbstractCommandExecutor { logger.info("create command file:{}", commandFile); StringBuilder sb = new StringBuilder(); - sb.append("#!/bin/sh\n"); - sb.append("BASEDIR=$(cd `dirname $0`; pwd)\n"); - sb.append("cd $BASEDIR\n"); - - if (envFile != null) { - sb.append("source " + envFile + "\n"); + if (OSUtils.isWindows()) { + sb.append("@echo off\n"); + sb.append("cd /d %~dp0\n"); + if (envFile != null) { + sb.append("call ").append(envFile).append("\n"); + } + } else { + sb.append("#!/bin/sh\n"); + sb.append("BASEDIR=$(cd `dirname $0`; pwd)\n"); + sb.append("cd $BASEDIR\n"); + if (envFile != null) { + sb.append("source ").append(envFile).append("\n"); + } } - sb.append("\n\n"); sb.append(execCommand); - logger.info("command : {}",sb.toString()); + logger.info("command : {}", sb.toString()); // write data to file - FileUtils.writeStringToFile(new File(commandFile), sb.toString(), - Charset.forName("UTF-8")); + FileUtils.writeStringToFile(new File(commandFile), sb.toString(), StandardCharsets.UTF_8); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java index ef941cd062..7537ca2edc 100755 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java @@ -44,6 +44,7 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.datax.DataxParameters; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; @@ -338,7 +339,7 @@ public class DataxTask extends AbstractTask { private String buildShellCommandFile(String jobConfigFilePath) throws Exception { // generate scripts - String fileName = String.format("%s/%s_node.sh", taskDir, taskProps.getTaskAppId()); + String fileName = String.format("%s/%s_node.%s", taskDir, taskProps.getTaskAppId(), OSUtils.isWindows() ? "bat" : "sh"); Path path = new File(fileName).toPath(); if (Files.exists(path)) { @@ -370,7 +371,13 @@ public class DataxTask extends AbstractTask { // create shell command file Set perms = PosixFilePermissions.fromString(Constants.RWXR_XR_X); FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); - Files.createFile(path, attr); + + if (OSUtils.isWindows()) { + Files.createFile(path); + } else { + Files.createFile(path, attr); + } + Files.write(path, dataxCommand.getBytes(), StandardOpenOption.APPEND); return fileName; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java index 5704c8052e..90661a690a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.shell.ShellParameters; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; @@ -123,7 +124,7 @@ public class ShellTask extends AbstractTask { */ private String buildCommand() throws Exception { // generate scripts - String fileName = String.format("%s/%s_node.sh", taskDir, taskProps.getTaskAppId()); + String fileName = String.format("%s/%s_node.%s", taskDir, taskProps.getTaskAppId(), OSUtils.isWindows() ? "bat" : "sh"); Path path = new File(fileName).toPath(); if (Files.exists(path)) { @@ -154,7 +155,11 @@ public class ShellTask extends AbstractTask { Set perms = PosixFilePermissions.fromString(Constants.RWXR_XR_X); FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); - Files.createFile(path, attr); + if (OSUtils.isWindows()) { + Files.createFile(path); + } else { + Files.createFile(path, attr); + } Files.write(path, shellParameters.getRawScript().getBytes(), StandardOpenOption.APPEND); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java index 77fc398702..1e0adaad9b 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java @@ -21,7 +21,12 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + public class ProcessUtilsTest { + private static final Logger logger = LoggerFactory.getLogger(ProcessUtilsTest.class); @Test @@ -30,4 +35,16 @@ public class ProcessUtilsTest { Assert.assertNotEquals("The child process of process 1 should not be empty", pidList, ""); logger.info("Sub process list : {}", pidList); } + + @Test + public void testBuildCommandStr() { + List commands = new ArrayList<>(); + commands.add("sudo"); + try { + Assert.assertEquals(ProcessUtils.buildCommandStr(commands), "sudo"); + } catch (IOException e) { + Assert.fail(e.getMessage()); + } + } + } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java new file mode 100644 index 0000000000..5536665e26 --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.server.worker.task.shell; + +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.dao.entity.DataSource; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor; +import org.apache.dolphinscheduler.server.worker.task.TaskProps; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationContext; + +import java.util.Date; + +/** + * shell task test + */ +@RunWith(PowerMockRunner.class) +@PrepareForTest(OSUtils.class) +public class ShellTaskTest { + + private static final Logger logger = LoggerFactory.getLogger(ShellTaskTest.class); + + private ShellTask shellTask; + + private ProcessService processService; + + private ShellCommandExecutor shellCommandExecutor; + + private ApplicationContext applicationContext; + + @Before + public void before() throws Exception { + PowerMockito.mockStatic(OSUtils.class); + processService = PowerMockito.mock(ProcessService.class); + shellCommandExecutor = PowerMockito.mock(ShellCommandExecutor.class); + + applicationContext = PowerMockito.mock(ApplicationContext.class); + SpringApplicationContext springApplicationContext = new SpringApplicationContext(); + springApplicationContext.setApplicationContext(applicationContext); + PowerMockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService); + + TaskProps props = new TaskProps(); + props.setTaskDir("/tmp"); + props.setTaskAppId(String.valueOf(System.currentTimeMillis())); + props.setTaskInstId(1); + props.setTenantCode("1"); + props.setEnvFile(".dolphinscheduler_env.sh"); + props.setTaskStartTime(new Date()); + props.setTaskTimeout(0); + props.setTaskParams("{\"rawScript\": \" echo 'hello world!'\"}"); + shellTask = new ShellTask(props, logger); + shellTask.init(); + + PowerMockito.when(processService.findDataSourceById(1)).thenReturn(getDataSource()); + PowerMockito.when(processService.findDataSourceById(2)).thenReturn(getDataSource()); + PowerMockito.when(processService.findProcessInstanceByTaskId(1)).thenReturn(getProcessInstance()); + + String fileName = String.format("%s/%s_node.%s", props.getTaskDir(), props.getTaskAppId(), OSUtils.isWindows() ? "bat" : "sh"); + PowerMockito.when(shellCommandExecutor.run(fileName, processService)).thenReturn(0); + } + + private DataSource getDataSource() { + DataSource dataSource = new DataSource(); + dataSource.setType(DbType.MYSQL); + dataSource.setConnectionParams( + "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:mysql://127.0.0.1:3306\",\"database\":\"test\",\"jdbcUrl\":\"jdbc:mysql://127.0.0.1:3306/test\"}"); + dataSource.setUserId(1); + return dataSource; + } + + private ProcessInstance getProcessInstance() { + ProcessInstance processInstance = new ProcessInstance(); + processInstance.setCommandType(CommandType.START_PROCESS); + processInstance.setScheduleTime(new Date()); + return processInstance; + } + + @After + public void after() {} + + /** + * Method: ShellTask() + */ + @Test + public void testShellTask() + throws Exception { + TaskProps props = new TaskProps(); + props.setTaskDir("/tmp"); + props.setTaskAppId(String.valueOf(System.currentTimeMillis())); + props.setTaskInstId(1); + props.setTenantCode("1"); + ShellTask shellTaskTest = new ShellTask(props, logger); + Assert.assertNotNull(shellTaskTest); + } + + /** + * Method: init for Unix-like + */ + @Test + public void testInitForUnix() { + try { + PowerMockito.when(OSUtils.isWindows()).thenReturn(false); + shellTask.init(); + Assert.assertTrue(true); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + /** + * Method: init for Windows + */ + @Test + public void testInitForWindows() { + try { + PowerMockito.when(OSUtils.isWindows()).thenReturn(true); + shellTask.init(); + Assert.assertTrue(true); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + /** + * Method: handle() for Unix-like + */ + @Test + public void testHandleForUnix() throws Exception { + try { + PowerMockito.when(OSUtils.isWindows()).thenReturn(false); + shellTask.handle(); + Assert.assertTrue(true); + } catch (Error | Exception e) { + if (!e.getMessage().contains("process error . exitCode is : -1") + && !System.getProperty("os.name").startsWith("Windows")) { + logger.error(e.getMessage()); + } + } + } + + /** + * Method: handle() for Windows + */ + @Test + public void testHandleForWindows() throws Exception { + try { + PowerMockito.when(OSUtils.isWindows()).thenReturn(true); + shellTask.handle(); + Assert.assertTrue(true); + } catch (Error | Exception e) { + if (!e.getMessage().contains("process error . exitCode is : -1")) { + logger.error(e.getMessage()); + } + } + } + + /** + * Method: cancelApplication() + */ + @Test + public void testCancelApplication() throws Exception { + try { + shellTask.cancelApplication(true); + Assert.assertTrue(true); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + +} diff --git a/pom.xml b/pom.xml index dee1dce8b2..a7feec0e14 100644 --- a/pom.xml +++ b/pom.xml @@ -684,6 +684,9 @@ **/common/utils/*.java + **/common/utils/process/ProcessBuilderForWin32Test.java + **/common/utils/process/ProcessEnvironmentForWin32Test.java + **/common/utils/process/ProcessImplForWin32Test.java **/common/log/*.java **/common/threadutils/*.java **/common/graph/*.java @@ -732,6 +735,7 @@ **/alert/template/AlertTemplateFactoryTest.java **/alert/template/impl/DefaultHTMLTemplateTest.java **/server/worker/task/datax/DataxTaskTest.java + **/server/worker/task/shell/ShellTaskTest.java **/server/worker/task/sqoop/SqoopTaskTest.java **/server/utils/DataxUtilsTest.java **/service/zk/DefaultEnsembleProviderTest.java From 4bb97dfb8b1c4aaee8e61a8d12793b1c337ee5f1 Mon Sep 17 00:00:00 2001 From: dailidong Date: Thu, 5 Mar 2020 23:53:12 +0800 Subject: [PATCH 15/59] fix actions/checkout@v2 problem (#2095) --- .github/workflows/ci_backend.yml | 12 ++++++++++++ .github/workflows/ci_e2e.yml | 6 ++++++ .github/workflows/ci_frontend.yml | 12 ++++++++++++ .github/workflows/ci_ut.yml | 6 ++++++ 4 files changed, 36 insertions(+) diff --git a/.github/workflows/ci_backend.yml b/.github/workflows/ci_backend.yml index 1ca15c2813..0273251e99 100644 --- a/.github/workflows/ci_backend.yml +++ b/.github/workflows/ci_backend.yml @@ -46,6 +46,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: @@ -56,6 +62,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml index 7d9c5e3e72..fe818d0668 100644 --- a/.github/workflows/ci_e2e.yml +++ b/.github/workflows/ci_e2e.yml @@ -30,6 +30,12 @@ jobs: steps: - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - uses: actions/cache@v1 with: path: ~/.m2/repository diff --git a/.github/workflows/ci_frontend.yml b/.github/workflows/ci_frontend.yml index 1f959b40eb..494d12dbae 100644 --- a/.github/workflows/ci_frontend.yml +++ b/.github/workflows/ci_frontend.yml @@ -35,6 +35,12 @@ jobs: os: [ubuntu-latest, macos-latest] steps: - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - name: Set up Node.js uses: actions/setup-node@v1 with: @@ -50,6 +56,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index e7e47437d4..48cd1e16ce 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -30,6 +30,12 @@ jobs: steps: - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - uses: actions/cache@v1 with: path: ~/.m2/repository From 0e1dd8d7d95b945c8982f18b802c88b216cf0179 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Sat, 7 Mar 2020 18:07:53 +0800 Subject: [PATCH 16/59] remove incorrect or unnecessary URL tag (#2099) --- dolphinscheduler-common/pom.xml | 2 +- dolphinscheduler-dao/pom.xml | 2 +- dolphinscheduler-remote/pom.xml | 2 -- dolphinscheduler-server/pom.xml | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index e7789f724b..ca75a84a62 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -25,7 +25,7 @@ dolphinscheduler-common dolphinscheduler-common - http://maven.apache.org + jar UTF-8 diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml index fd43458f87..3aea888f94 100644 --- a/dolphinscheduler-dao/pom.xml +++ b/dolphinscheduler-dao/pom.xml @@ -25,7 +25,7 @@ dolphinscheduler-dao ${project.artifactId} - http://maven.apache.org + UTF-8 diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml index b67b033ffa..39c7c6a7c0 100644 --- a/dolphinscheduler-remote/pom.xml +++ b/dolphinscheduler-remote/pom.xml @@ -12,8 +12,6 @@ dolphinscheduler-remote dolphinscheduler-remote - - http://www.example.com UTF-8 diff --git a/dolphinscheduler-server/pom.xml b/dolphinscheduler-server/pom.xml index 86490197b6..e8e84297e2 100644 --- a/dolphinscheduler-server/pom.xml +++ b/dolphinscheduler-server/pom.xml @@ -25,7 +25,7 @@ dolphinscheduler-server dolphinscheduler-server - http://maven.apache.org + jar UTF-8 From 89b42db3b4da98295c898d4243577ef1e0a0291d Mon Sep 17 00:00:00 2001 From: liwenhe1993 <32166572+liwenhe1993@users.noreply.github.com> Date: Sun, 8 Mar 2020 15:56:24 +0800 Subject: [PATCH 17/59] Add docker internal server check script (#2092) * add environment variables add checkpoint.sh file to docker image 1. add `POSTGRESQL_DATABASE` environment variable 2. add `DOLPHINSCHEDULER_DATA_BASEDIR_PATH` environment variable 3. add `DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH` environment variable 4. add `DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH` environment variable 5. add checkpoint.sh file to check server process 6. modify `README.md` and `README_zh_CN.md` * add environment variables add checkpoint.sh file to docker image 1. add `POSTGRESQL_DATABASE` environment variable 2. add `DOLPHINSCHEDULER_DATA_BASEDIR_PATH` environment variable 3. add `DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH` environment variable 4. add `DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH` environment variable 5. add checkpoint.sh file to check server process 6. modify `README.md` and `README_zh_CN.md` * add logger port and modify dolphinscheduler_env to dolphinscheduler_env.sh --- dockerfile/Dockerfile | 13 +++--- dockerfile/README.md | 40 ++++++++++++++----- dockerfile/README_zh_CN.md | 38 ++++++++++++++---- dockerfile/checkpoint.sh | 27 +++++++++++++ .../application.properties.tpl | 2 +- .../dolphinscheduler/common.properties.tpl | 12 +++--- ...nscheduler_env => dolphinscheduler_env.sh} | 0 .../dolphinscheduler/quartz.properties.tpl | 2 +- dockerfile/startup-init-conf.sh | 4 ++ 9 files changed, 108 insertions(+), 30 deletions(-) create mode 100644 dockerfile/checkpoint.sh rename dockerfile/conf/dolphinscheduler/env/{dolphinscheduler_env => dolphinscheduler_env.sh} (100%) diff --git a/dockerfile/Dockerfile b/dockerfile/Dockerfile index 1fc064c489..8fa4886f31 100644 --- a/dockerfile/Dockerfile +++ b/dockerfile/Dockerfile @@ -63,19 +63,22 @@ RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \ ADD ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d #7. add configuration and modify permissions and set soft links +ADD ./checkpoint.sh /root/checkpoint.sh ADD ./startup-init-conf.sh /root/startup-init-conf.sh ADD ./startup.sh /root/startup.sh ADD ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/ -ADD ./conf/dolphinscheduler/env/dolphinscheduler_env /opt/dolphinscheduler/conf/env/ -RUN chmod +x /root/startup-init-conf.sh && \ +ADD conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/ +RUN chmod +x /root/checkpoint.sh && \ + chmod +x /root/startup-init-conf.sh && \ chmod +x /root/startup.sh && \ - chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env && \ + chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \ chmod +x /opt/dolphinscheduler/script/*.sh && \ chmod +x /opt/dolphinscheduler/bin/*.sh && \ chmod +x /opt/zookeeper/bin/*.sh && \ + dos2unix /root/checkpoint.sh && \ dos2unix /root/startup-init-conf.sh && \ dos2unix /root/startup.sh && \ - dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env && \ + dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \ dos2unix /opt/dolphinscheduler/script/*.sh && \ dos2unix /opt/dolphinscheduler/bin/*.sh && \ dos2unix /opt/zookeeper/bin/*.sh && \ @@ -87,6 +90,6 @@ RUN chmod +x /root/startup-init-conf.sh && \ RUN rm -rf /var/cache/apk/* #9. expose port -EXPOSE 2181 2888 3888 5432 12345 8888 +EXPOSE 2181 2888 3888 5432 12345 50051 8888 ENTRYPOINT ["/root/startup.sh"] \ No newline at end of file diff --git a/dockerfile/README.md b/dockerfile/README.md index b5a9d0d3aa..60af7fad8f 100644 --- a/dockerfile/README.md +++ b/dockerfile/README.md @@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org #### You can start a dolphinscheduler instance ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test \ +-e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \ -p 8888:8888 \ dolphinscheduler all ``` @@ -25,13 +25,13 @@ The default postgres user `root`, postgres password `root` and database `dolphin The default zookeeper is created in the `startup.sh`. -#### Or via Environment Variables **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`ZOOKEEPER_QUORUM`** +#### Or via Environment Variables **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`POSTGRESQL_DATABASE`** **`ZOOKEEPER_QUORUM`** You can specify **existing postgres service**. Example: ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -p 8888:8888 \ dolphinscheduler all @@ -42,7 +42,7 @@ You can specify **existing zookeeper service**. Example: ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -p 8888:8888 \ dolphinscheduler all ``` @@ -56,7 +56,7 @@ You can start a standalone dolphinscheduler server. ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler master-server ``` @@ -66,7 +66,7 @@ dolphinscheduler master-server ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler worker-server ``` @@ -75,7 +75,7 @@ dolphinscheduler worker-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -p 12345:12345 \ dolphinscheduler api-server @@ -85,7 +85,7 @@ dolphinscheduler api-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler alert-server ``` @@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \ dolphinscheduler frontend ``` -**Note**: You must be specify `POSTGRESQL_HOST` `POSTGRESQL_PORT` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server. +**Note**: You must be specify `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server. ## How to build a docker image @@ -140,14 +140,36 @@ This environment variable sets the port for PostgreSQL. The default value is `54 This environment variable sets the username for PostgreSQL. The default value is `root`. +**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. + **`POSTGRESQL_PASSWORD`** This environment variable sets the password for PostgreSQL. The default value is `root`. +**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. + +**`POSTGRESQL_DATABASE`** + +This environment variable sets the database for PostgreSQL. The default value is `dolphinscheduler`. + +**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. + **`DOLPHINSCHEDULER_ENV_PATH`** This environment variable sets the runtime environment for task. The default value is `/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh`. +**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`** + +User data directory path, self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler` + +**`DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH`** + +Directory path for user data download. self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler/download` + +**`DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH`** + +Process execute directory. self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler/exec` + **`TASK_QUEUE`** This environment variable sets the task queue for `master-server` and `worker-serverr`. The default value is `zookeeper`. diff --git a/dockerfile/README_zh_CN.md b/dockerfile/README_zh_CN.md index 5830e30b07..900c8b50d9 100644 --- a/dockerfile/README_zh_CN.md +++ b/dockerfile/README_zh_CN.md @@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org #### 你可以运行一个dolphinscheduler实例 ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test \ +-e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \ -p 8888:8888 \ dolphinscheduler all ``` @@ -31,7 +31,7 @@ dolphinscheduler all ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -p 8888:8888 \ dolphinscheduler all @@ -42,7 +42,7 @@ dolphinscheduler all ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -p 8888:8888 \ dolphinscheduler all ``` @@ -56,7 +56,7 @@ dolphinscheduler all ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler master-server ``` @@ -66,7 +66,7 @@ dolphinscheduler master-server ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler worker-server ``` @@ -75,7 +75,7 @@ dolphinscheduler worker-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -p 12345:12345 \ dolphinscheduler api-server @@ -85,7 +85,7 @@ dolphinscheduler api-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler alert-server ``` @@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \ dolphinscheduler frontend ``` -**注意**: 当你运行dolphinscheduler中的部分服务时，你必须指定这些环境变量 `POSTGRESQL_HOST` `POSTGRESQL_PORT` `ZOOKEEPER_QUORUM`。 +**注意**: 当你运行dolphinscheduler中的部分服务时，你必须指定这些环境变量 `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM`。 ## 如何构建一个docker镜像 @@ -140,14 +140,36 @@ Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些 配置`PostgreSQL`的`USERNAME`， 默认值 `root`。 +**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 + **`POSTGRESQL_PASSWORD`** 配置`PostgreSQL`的`PASSWORD`， 默认值 `root`。 +**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 + +**`POSTGRESQL_DATABASE`** + +配置`PostgreSQL`的`DATABASE`， 默认值 `dolphinscheduler`。 + +**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 + **`DOLPHINSCHEDULER_ENV_PATH`** 任务执行时的环境变量配置文件， 默认值 `/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh`。 +**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`** + +用户数据目录, 用户自己配置, 请确保这个目录存在并且用户读写权限， 默认值 `/tmp/dolphinscheduler`。 + +**`DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH`** + +用户数据下载目录, 用户自己配置, 请确保这个目录存在并且用户读写权限， 默认值 `/tmp/dolphinscheduler/download`。 + +**`DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH`** + +任务执行目录, 用户自己配置, 请确保这个目录存在并且用户读写权限， 默认值 `/tmp/dolphinscheduler/exec`。 + **`TASK_QUEUE`** 配置`master-server`和`worker-serverr`的`Zookeeper`任务队列名， 默认值 `zookeeper`。 diff --git a/dockerfile/checkpoint.sh b/dockerfile/checkpoint.sh new file mode 100644 index 0000000000..cd2774f9ce --- /dev/null +++ b/dockerfile/checkpoint.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +if [ "$(ps -ef | grep java | grep -c $1)" -eq 0 ]; then + echo "[ERROR] $1 process not exits." + exit 1 +else + echo "[INFO] $1 process exits." + exit 0 +fi diff --git a/dockerfile/conf/dolphinscheduler/application.properties.tpl b/dockerfile/conf/dolphinscheduler/application.properties.tpl index c643c414cd..6dd8a18e11 100644 --- a/dockerfile/conf/dolphinscheduler/application.properties.tpl +++ b/dockerfile/conf/dolphinscheduler/application.properties.tpl @@ -19,7 +19,7 @@ spring.datasource.type=com.alibaba.druid.pool.DruidDataSource # postgre spring.datasource.driver-class-name=org.postgresql.Driver -spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/dolphinscheduler?characterEncoding=utf8 +spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8 # mysql #spring.datasource.driver-class-name=com.mysql.jdbc.Driver #spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 diff --git a/dockerfile/conf/dolphinscheduler/common.properties.tpl b/dockerfile/conf/dolphinscheduler/common.properties.tpl index ea03e0b78e..8134fc7a9b 100644 --- a/dockerfile/conf/dolphinscheduler/common.properties.tpl +++ b/dockerfile/conf/dolphinscheduler/common.properties.tpl @@ -38,6 +38,12 @@ dolphinscheduler.env.path=${DOLPHINSCHEDULER_ENV_PATH} resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties # is development state? default "false" development.state=true +# user data directory path, self configuration, please make sure the directory exists and have read write permissions +data.basedir.path=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH} +# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions +data.download.basedir.path=${DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH} +# process execute directory. self configuration, please make sure the directory exists and have read write permissions +process.exec.basepath=${DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH} # resource upload startup type : HDFS,S3,NONE res.upload.startup.type=NONE @@ -49,12 +55,6 @@ res.upload.startup.type=NONE hdfs.root.user=hdfs # data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended data.store2hdfs.basepath=/dolphinscheduler -# user data directory path, self configuration, please make sure the directory exists and have read write permissions -data.basedir.path=/tmp/dolphinscheduler -# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions -data.download.basedir.path=/tmp/dolphinscheduler/download -# process execute directory. self configuration, please make sure the directory exists and have read write permissions -process.exec.basepath=/tmp/dolphinscheduler/exec # whether kerberos starts hadoop.security.authentication.startup.state=false # java.security.krb5.conf path diff --git a/dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env b/dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh similarity index 100% rename from dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env rename to dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh diff --git a/dockerfile/conf/dolphinscheduler/quartz.properties.tpl b/dockerfile/conf/dolphinscheduler/quartz.properties.tpl index de5496bc3d..7c7c92e425 100644 --- a/dockerfile/conf/dolphinscheduler/quartz.properties.tpl +++ b/dockerfile/conf/dolphinscheduler/quartz.properties.tpl @@ -22,7 +22,7 @@ org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.PostgreSQLDelegate # postgre org.quartz.dataSource.myDs.driver = org.postgresql.Driver -org.quartz.dataSource.myDs.URL = jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/dolphinscheduler?characterEncoding=utf8 +org.quartz.dataSource.myDs.URL = jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8 org.quartz.dataSource.myDs.user = ${POSTGRESQL_USERNAME} org.quartz.dataSource.myDs.password = ${POSTGRESQL_PASSWORD} org.quartz.scheduler.instanceName = DolphinScheduler diff --git a/dockerfile/startup-init-conf.sh b/dockerfile/startup-init-conf.sh index db37976168..d2b50fa3e7 100644 --- a/dockerfile/startup-init-conf.sh +++ b/dockerfile/startup-init-conf.sh @@ -28,11 +28,15 @@ export POSTGRESQL_HOST=${POSTGRESQL_HOST:-"127.0.0.1"} export POSTGRESQL_PORT=${POSTGRESQL_PORT:-"5432"} export POSTGRESQL_USERNAME=${POSTGRESQL_USERNAME:-"root"} export POSTGRESQL_PASSWORD=${POSTGRESQL_PASSWORD:-"root"} +export POSTGRESQL_DATABASE=${POSTGRESQL_DATABASE:-"dolphinscheduler"} #============================================================================ # System #============================================================================ export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"} +export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"} +export DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH:-"/tmp/dolphinscheduler/download"} +export DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH=${DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH:-"/tmp/dolphinscheduler/exec"} #============================================================================ # Zookeeper From b2268b1af9bdab3162a4c6378c85a936ce00827f Mon Sep 17 00:00:00 2001 From: tswstarplanet Date: Sun, 8 Mar 2020 15:57:06 +0800 Subject: [PATCH 18/59] use stringbuilder to append string (#2108) --- .../service/zk/AbstractZKClient.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java index 135bfdabc6..fa1a0bfced 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java @@ -65,14 +65,16 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator { if (splits.length != Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH){ return; } - String str = splits[0] + Constants.COMMA - + splits[1] + Constants.COMMA - + OSUtils.cpuUsage() + Constants.COMMA - + OSUtils.memoryUsage() + Constants.COMMA - + OSUtils.loadAverage() + Constants.COMMA - + splits[5] + Constants.COMMA - + DateUtils.dateToString(new Date()); - zkClient.setData().forPath(znode,str.getBytes()); + StringBuilder sb = new StringBuilder(); + sb.append(splits[0]).append(Constants.COMMA) + .append(splits[1]).append(Constants.COMMA) + .append(OSUtils.cpuUsage()).append(Constants.COMMA) + .append(OSUtils.memoryUsage()).append(Constants.COMMA) + .append(OSUtils.loadAverage()).append(Constants.COMMA) + .append(splits[5]).append(Constants.COMMA) + .append(DateUtils.dateToString(new Date())); + + zkClient.setData().forPath(znode, sb.toString().getBytes()); } catch (Exception e) { logger.error("heartbeat for zk failed", e); From 275df85250683b22691c658591ed2094d8a32ac3 Mon Sep 17 00:00:00 2001 From: Rubik-W <39549317+Rubik-W@users.noreply.github.com> Date: Sun, 8 Mar 2020 16:00:47 +0800 Subject: [PATCH 19/59] fix: remove redundant import class (#2110) --- .../org/apache/dolphinscheduler/remote/command/Command.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java index 86ba79c884..ed46e1ff51 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java @@ -16,8 +16,6 @@ */ package org.apache.dolphinscheduler.remote.command; -import com.sun.org.apache.regexp.internal.RE; - import java.io.Serializable; import java.util.concurrent.atomic.AtomicLong; From ad381fcb716632deb966fd8ca0627ff8c17d5641 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Sun, 8 Mar 2020 21:22:07 +0800 Subject: [PATCH 20/59] move UT class to reasonable package (#2116) --- .../dolphinscheduler/common}/utils/PreconditionsTest.java | 3 +-- .../dolphinscheduler/service/quartz}/cron/CronUtilsTest.java | 3 +-- .../dolphinscheduler/service}/queue/BaseTaskQueueTest.java | 2 +- .../dolphinscheduler/service}/queue/TaskQueueZKImplTest.java | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) rename {dolphinscheduler-service/src/test/java => dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common}/utils/PreconditionsTest.java (98%) rename dolphinscheduler-service/src/test/java/{ => org/apache/dolphinscheduler/service/quartz}/cron/CronUtilsTest.java (99%) rename dolphinscheduler-service/src/test/java/{ => org/apache/dolphinscheduler/service}/queue/BaseTaskQueueTest.java (96%) rename dolphinscheduler-service/src/test/java/{ => org/apache/dolphinscheduler/service}/queue/TaskQueueZKImplTest.java (99%) diff --git a/dolphinscheduler-service/src/test/java/utils/PreconditionsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java similarity index 98% rename from dolphinscheduler-service/src/test/java/utils/PreconditionsTest.java rename to dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java index a1b85f1b12..47b24bb93c 100644 --- a/dolphinscheduler-service/src/test/java/utils/PreconditionsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java @@ -14,9 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package utils; +package org.apache.dolphinscheduler.common.utils; -import org.apache.dolphinscheduler.common.utils.Preconditions; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-service/src/test/java/cron/CronUtilsTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtilsTest.java similarity index 99% rename from dolphinscheduler-service/src/test/java/cron/CronUtilsTest.java rename to dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtilsTest.java index 6a402b5e67..b4f864c5b4 100644 --- a/dolphinscheduler-service/src/test/java/cron/CronUtilsTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtilsTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package cron; +package org.apache.dolphinscheduler.service.quartz.cron; import com.cronutils.builder.CronBuilder; import com.cronutils.model.Cron; @@ -25,7 +25,6 @@ import com.cronutils.model.field.CronFieldName; import com.cronutils.model.field.expression.*; import org.apache.dolphinscheduler.common.enums.CycleEnum; import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/BaseTaskQueueTest.java similarity index 96% rename from dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java rename to dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/BaseTaskQueueTest.java index 97ab9969a3..17e2ae4056 100644 --- a/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/BaseTaskQueueTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package queue; +package org.apache.dolphinscheduler.service.queue; import org.apache.dolphinscheduler.service.queue.ITaskQueue; import org.apache.dolphinscheduler.service.queue.TaskQueueFactory; diff --git a/dolphinscheduler-service/src/test/java/queue/TaskQueueZKImplTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskQueueZKImplTest.java similarity index 99% rename from dolphinscheduler-service/src/test/java/queue/TaskQueueZKImplTest.java rename to dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskQueueZKImplTest.java index d29c5aa610..5d464ac3c9 100644 --- a/dolphinscheduler-service/src/test/java/queue/TaskQueueZKImplTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskQueueZKImplTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package queue; +package org.apache.dolphinscheduler.service.queue; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.IpUtils; From 450a1f56fc73f088fce89a343a0b008706f2088c Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Sun, 8 Mar 2020 21:28:49 +0800 Subject: [PATCH 21/59] Adapting partial code(file name start with E) to the sonar cloud rule (#1999) * Adapting partial code(file name start with E) to the sonar cloud rule * remove isEmpty invoke * resolve conflicts --- .../manager/EnterpriseWeChatManager.java | 4 +- .../alert/utils/EnterpriseWeChatUtils.java | 38 +++++++++---------- .../utils/EnterpriseWeChatUtilsTest.java | 8 ++-- .../api/controller/ExecutorController.java | 2 +- .../api/service/ExecutorService.java | 15 +++----- .../dao/mapper/ErrorCommandMapperTest.java | 4 +- 6 files changed, 34 insertions(+), 37 deletions(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java index 9bcad56c24..bb06be6561 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java @@ -42,8 +42,8 @@ public class EnterpriseWeChatManager { public Map send(Alert alert, String token){ Map retMap = new HashMap<>(); retMap.put(Constants.STATUS, false); - String agentId = EnterpriseWeChatUtils.enterpriseWeChatAgentId; - String users = EnterpriseWeChatUtils.enterpriseWeChatUsers; + String agentId = EnterpriseWeChatUtils.ENTERPRISE_WE_CHAT_AGENT_ID; + String users = EnterpriseWeChatUtils.ENTERPRISE_WE_CHAT_USERS; List userList = Arrays.asList(users.split(",")); logger.info("send message {}",alert); String msg = EnterpriseWeChatUtils.makeUserSendMsg(userList, agentId,EnterpriseWeChatUtils.markdownByAlert(alert)); diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java index ff8822421a..900c120cd4 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java @@ -43,24 +43,24 @@ public class EnterpriseWeChatUtils { public static final Logger logger = LoggerFactory.getLogger(EnterpriseWeChatUtils.class); - private static final String enterpriseWeChatCorpId = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_CORP_ID); + private static final String ENTERPRISE_WE_CHAT_CORP_ID = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_CORP_ID); - private static final String enterpriseWeChatSecret = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_SECRET); + private static final String ENTERPRISE_WE_CHAT_SECRET = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_SECRET); - private static final String enterpriseWeChatTokenUrl = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TOKEN_URL); - private static String enterpriseWeChatTokenUrlReplace = enterpriseWeChatTokenUrl - .replaceAll("\\$corpId", enterpriseWeChatCorpId) - .replaceAll("\\$secret", enterpriseWeChatSecret); + private static final String ENTERPRISE_WE_CHAT_TOKEN_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TOKEN_URL); + private static final String ENTERPRISE_WE_CHAT_TOKEN_URL_REPLACE = ENTERPRISE_WE_CHAT_TOKEN_URL + .replaceAll("\\$corpId", ENTERPRISE_WE_CHAT_CORP_ID) + .replaceAll("\\$secret", ENTERPRISE_WE_CHAT_SECRET); - private static final String enterpriseWeChatPushUrl = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_PUSH_URL); + private static final String ENTERPRISE_WE_CHAT_PUSH_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_PUSH_URL); - private static final String enterpriseWeChatTeamSendMsg = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TEAM_SEND_MSG); + private static final String ENTERPRISE_WE_CHAT_TEAM_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TEAM_SEND_MSG); - private static final String enterpriseWeChatUserSendMsg = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USER_SEND_MSG); + private static final String ENTERPRISE_WE_CHAT_USER_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USER_SEND_MSG); - public static final String enterpriseWeChatAgentId = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_AGENT_ID); + public static final String ENTERPRISE_WE_CHAT_AGENT_ID = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_AGENT_ID); - public static final String enterpriseWeChatUsers = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USERS); + public static final String ENTERPRISE_WE_CHAT_USERS = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USERS); /** * get Enterprise WeChat is enable @@ -87,7 +87,7 @@ public class EnterpriseWeChatUtils { CloseableHttpClient httpClient = HttpClients.createDefault(); try { - HttpGet httpGet = new HttpGet(enterpriseWeChatTokenUrlReplace); + HttpGet httpGet = new HttpGet(ENTERPRISE_WE_CHAT_TOKEN_URL_REPLACE); CloseableHttpResponse response = httpClient.execute(httpGet); try { HttpEntity entity = response.getEntity(); @@ -114,7 +114,7 @@ public class EnterpriseWeChatUtils { * @return Enterprise WeChat send message */ public static String makeTeamSendMsg(String toParty, String agentId, String msg) { - return enterpriseWeChatTeamSendMsg.replaceAll("\\$toParty", toParty) + return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\$toParty", toParty) .replaceAll("\\$agentId", agentId) .replaceAll("\\$msg", msg); } @@ -128,7 +128,7 @@ public class EnterpriseWeChatUtils { */ public static String makeTeamSendMsg(Collection toParty, String agentId, String msg) { String listParty = FuncUtils.mkString(toParty, "|"); - return enterpriseWeChatTeamSendMsg.replaceAll("\\$toParty", listParty) + return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\$toParty", listParty) .replaceAll("\\$agentId", agentId) .replaceAll("\\$msg", msg); } @@ -141,7 +141,7 @@ public class EnterpriseWeChatUtils { * @return Enterprise WeChat send message */ public static String makeUserSendMsg(String toUser, String agentId, String msg) { - return enterpriseWeChatUserSendMsg.replaceAll("\\$toUser", toUser) + return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\$toUser", toUser) .replaceAll("\\$agentId", agentId) .replaceAll("\\$msg", msg); } @@ -155,7 +155,7 @@ public class EnterpriseWeChatUtils { */ public static String makeUserSendMsg(Collection toUser, String agentId, String msg) { String listUser = FuncUtils.mkString(toUser, "|"); - return enterpriseWeChatUserSendMsg.replaceAll("\\$toUser", listUser) + return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\$toUser", listUser) .replaceAll("\\$agentId", agentId) .replaceAll("\\$msg", msg); } @@ -169,7 +169,7 @@ public class EnterpriseWeChatUtils { * @throws IOException the IOException */ public static String sendEnterpriseWeChat(String charset, String data, String token) throws IOException { - String enterpriseWeChatPushUrlReplace = enterpriseWeChatPushUrl.replaceAll("\\$token", token); + String enterpriseWeChatPushUrlReplace = ENTERPRISE_WE_CHAT_PUSH_URL.replaceAll("\\$token", token); CloseableHttpClient httpClient = HttpClients.createDefault(); try { @@ -184,8 +184,8 @@ public class EnterpriseWeChatUtils { } finally { response.close(); } - logger.info("Enterprise WeChat send [{}], param:{}, resp:{}", - enterpriseWeChatPushUrl, data, resp); + logger.info("Enterprise WeChat send [{}], param:{}, resp:{}", + ENTERPRISE_WE_CHAT_PUSH_URL, data, resp); return resp; } finally { httpClient.close(); diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java index 3471f6efdd..15b92a622e 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java @@ -53,7 +53,7 @@ public class EnterpriseWeChatUtilsTest { String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); String errmsg = JSON.parseObject(resp).getString("errmsg"); - Assert.assertEquals(errmsg, "ok"); + Assert.assertEquals("ok",errmsg); } catch (IOException e) { e.printStackTrace(); } @@ -68,7 +68,7 @@ public class EnterpriseWeChatUtilsTest { String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); String errmsg = JSON.parseObject(resp).getString("errmsg"); - Assert.assertEquals(errmsg, "ok"); + Assert.assertEquals("ok",errmsg); } catch (IOException e) { e.printStackTrace(); } @@ -95,7 +95,7 @@ public class EnterpriseWeChatUtilsTest { String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); String errmsg = JSON.parseObject(resp).getString("errmsg"); - Assert.assertEquals(errmsg, "ok"); + Assert.assertEquals("ok",errmsg); } catch (IOException e) { e.printStackTrace(); } @@ -110,7 +110,7 @@ public class EnterpriseWeChatUtilsTest { String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); String errmsg = JSON.parseObject(resp).getString("errmsg"); - Assert.assertEquals(errmsg, "ok"); + Assert.assertEquals("ok",errmsg); } catch (IOException e) { e.printStackTrace(); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java index cae4993942..ffedd5703c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java @@ -149,7 +149,7 @@ public class ExecutorController extends BaseController { ) { try { logger.info("execute command, login user: {}, project:{}, process instance id:{}, execute type:{}", - loginUser.getUserName(), projectName, processInstanceId, executeType.toString()); + loginUser.getUserName(), projectName, processInstanceId, executeType); Map result = execService.execute(loginUser, projectName, processInstanceId, executeType); return returnDataList(result); } catch (Exception e) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index 152292a21b..86b507f0a0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -259,10 +259,7 @@ public class ExecutorService extends BaseService{ // checkTenantExists(); Tenant tenant = processService.getTenantForProcess(processDefinition.getTenantId(), processDefinition.getUserId()); - if(tenant == null){ - return false; - } - return true; + return tenant != null; } /** @@ -298,6 +295,7 @@ public class ExecutorService extends BaseService{ if (executionStatus.typeIsPause()|| executionStatus.typeIsCancel()) { checkResult = true; } + break; default: break; } @@ -369,7 +367,7 @@ public class ExecutorService extends BaseService{ * @return check result code */ public Map startCheckByProcessDefinedId(int processDefineId) { - Map result = new HashMap(); + Map result = new HashMap<>(); if (processDefineId == 0){ logger.error("process definition id is null"); @@ -378,10 +376,9 @@ public class ExecutorService extends BaseService{ List ids = new ArrayList<>(); processService.recurseFindSubProcessId(processDefineId, ids); Integer[] idArray = ids.toArray(new Integer[ids.size()]); - if (ids.size() > 0){ - List processDefinitionList; - processDefinitionList = processDefinitionMapper.queryDefinitionListByIdList(idArray); - if (processDefinitionList != null && processDefinitionList.size() > 0){ + if (!ids.isEmpty()){ + List processDefinitionList = processDefinitionMapper.queryDefinitionListByIdList(idArray); + if (processDefinitionList != null){ for (ProcessDefinition processDefinition : processDefinitionList){ /** * if there is no online process, exit directly diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java index 3653e6a53d..5fb7dfc09f 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java @@ -67,7 +67,7 @@ public class ErrorCommandMapperTest { //update errorCommand.setUpdateTime(new Date()); int update = errorCommandMapper.updateById(errorCommand); - Assert.assertEquals(update, 1); + Assert.assertEquals(1,update); errorCommandMapper.deleteById(errorCommand.getId()); } @@ -79,7 +79,7 @@ public class ErrorCommandMapperTest { ErrorCommand errorCommand = insertOne(); int delete = errorCommandMapper.deleteById(errorCommand.getId()); - Assert.assertEquals(delete, 1); + Assert.assertEquals(1,delete); } /** From 9224b49b58b756d22c75d8929108f716283282b4 Mon Sep 17 00:00:00 2001 From: tison Date: Mon, 9 Mar 2020 19:06:41 +0800 Subject: [PATCH 22/59] access field handle of FileDescriptor in ProcessImplForWin32 by reflection for portability (#2113) * access field handle of FileDescriptor in ProcessImplForWin32 by reflection for portability Current implementation relies on `sun.misc.JavaIOFileDescriptorAccess` which is only accessible on oraclejdk8. Basically the demand is getting & setting `handle` field of `FileDescriptor`, so we can directly do that with reflection. Though, I suspect the necessity we introduce ProcessImplForWin32. Maybe we could have a better way to support worker server to run bat script. * harden initialization of ProcessImplForWin32 * ignore ShellTaskTest#testHandleForWindows outside Windows --- .../common/utils/OSUtils.java | 16 ++++-- .../utils/process/ProcessImplForWin32.java | 57 +++++++++++++++---- .../worker/task/shell/ShellTaskTest.java | 3 +- 3 files changed, 58 insertions(+), 18 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index acfca77c8c..b011c0bc4e 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -400,8 +400,7 @@ public class OSUtils { * @return true if mac */ public static boolean isMacOS() { - String os = System.getProperty("os.name"); - return os.startsWith("Mac"); + return getOSName().startsWith("Mac"); } @@ -409,9 +408,16 @@ public class OSUtils { * whether is windows * @return true if windows */ - public static boolean isWindows() { - String os = System.getProperty("os.name"); - return os.startsWith("Windows"); + public static boolean isWindows() { ; + return getOSName().startsWith("Windows"); + } + + /** + * get current OS name + * @return current OS name + */ + public static String getOSName() { + return System.getProperty("os.name"); } /** diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java index 4583be8aff..9f2716a096 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java @@ -19,6 +19,8 @@ package org.apache.dolphinscheduler.common.utils.process; import com.sun.jna.Pointer; import com.sun.jna.platform.win32.*; import com.sun.jna.ptr.IntByReference; +import java.lang.reflect.Field; +import org.apache.dolphinscheduler.common.utils.OSUtils; import sun.security.action.GetPropertyAction; import java.io.*; @@ -31,10 +33,25 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import static com.sun.jna.platform.win32.WinBase.STILL_ACTIVE; +import static java.util.Objects.requireNonNull; public class ProcessImplForWin32 extends Process { - private static final sun.misc.JavaIOFileDescriptorAccess fdAccess - = sun.misc.SharedSecrets.getJavaIOFileDescriptorAccess(); + + private static final Field FD_HANDLE; + + static { + if (!OSUtils.isWindows()) { + throw new RuntimeException("ProcessImplForWin32 can be only initialized in " + + "Windows environment, but current OS is " + OSUtils.getOSName()); + } + + try { + FD_HANDLE = requireNonNull(FileDescriptor.class.getDeclaredField("handle")); + FD_HANDLE.setAccessible(true); + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); + } + } private static final int PIPE_SIZE = 4096 + 24; @@ -46,6 +63,22 @@ public class ProcessImplForWin32 extends Process { private static final WinNT.HANDLE JAVA_INVALID_HANDLE_VALUE = new WinNT.HANDLE(Pointer.createConstant(-1)); + private static void setHandle(FileDescriptor obj, long handle) { + try { + FD_HANDLE.set(obj, handle); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + private static long getHandle(FileDescriptor obj) { + try { + return (Long) FD_HANDLE.get(obj); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + /** * Open a file for writing. If {@code append} is {@code true} then the file * is opened for atomic append directly and a FileOutputStream constructed @@ -63,7 +96,7 @@ public class ProcessImplForWin32 extends Process { sm.checkWrite(path); long handle = openForAtomicAppend(path); final FileDescriptor fd = new FileDescriptor(); - fdAccess.setHandle(fd, handle); + setHandle(fd, handle); return AccessController.doPrivileged( new PrivilegedAction() { public FileOutputStream run() { @@ -102,30 +135,30 @@ public class ProcessImplForWin32 extends Process { if (redirects[0] == ProcessBuilderForWin32.Redirect.PIPE) stdHandles[0] = -1L; else if (redirects[0] == ProcessBuilderForWin32.Redirect.INHERIT) - stdHandles[0] = fdAccess.getHandle(FileDescriptor.in); + stdHandles[0] = getHandle(FileDescriptor.in); else { f0 = new FileInputStream(redirects[0].file()); - stdHandles[0] = fdAccess.getHandle(f0.getFD()); + stdHandles[0] = getHandle(f0.getFD()); } if (redirects[1] == ProcessBuilderForWin32.Redirect.PIPE) stdHandles[1] = -1L; else if (redirects[1] == ProcessBuilderForWin32.Redirect.INHERIT) - stdHandles[1] = fdAccess.getHandle(FileDescriptor.out); + stdHandles[1] = getHandle(FileDescriptor.out); else { f1 = newFileOutputStream(redirects[1].file(), redirects[1].append()); - stdHandles[1] = fdAccess.getHandle(f1.getFD()); + stdHandles[1] = getHandle(f1.getFD()); } if (redirects[2] == ProcessBuilderForWin32.Redirect.PIPE) stdHandles[2] = -1L; else if (redirects[2] == ProcessBuilderForWin32.Redirect.INHERIT) - stdHandles[2] = fdAccess.getHandle(FileDescriptor.err); + stdHandles[2] = getHandle(FileDescriptor.err); else { f2 = newFileOutputStream(redirects[2].file(), redirects[2].append()); - stdHandles[2] = fdAccess.getHandle(f2.getFD()); + stdHandles[2] = getHandle(f2.getFD()); } } @@ -442,7 +475,7 @@ public class ProcessImplForWin32 extends Process { stdin_stream = ProcessBuilderForWin32.NullOutputStream.INSTANCE; else { FileDescriptor stdin_fd = new FileDescriptor(); - fdAccess.setHandle(stdin_fd, stdHandles[0]); + setHandle(stdin_fd, stdHandles[0]); stdin_stream = new BufferedOutputStream( new FileOutputStream(stdin_fd)); } @@ -451,7 +484,7 @@ public class ProcessImplForWin32 extends Process { stdout_stream = ProcessBuilderForWin32.NullInputStream.INSTANCE; else { FileDescriptor stdout_fd = new FileDescriptor(); - fdAccess.setHandle(stdout_fd, stdHandles[1]); + setHandle(stdout_fd, stdHandles[1]); stdout_stream = new BufferedInputStream( new FileInputStream(stdout_fd)); } @@ -460,7 +493,7 @@ public class ProcessImplForWin32 extends Process { stderr_stream = ProcessBuilderForWin32.NullInputStream.INSTANCE; else { FileDescriptor stderr_fd = new FileDescriptor(); - fdAccess.setHandle(stderr_fd, stdHandles[2]); + setHandle(stderr_fd, stdHandles[2]); stderr_stream = new FileInputStream(stderr_fd); } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java index 5536665e26..ebe90147d1 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java @@ -27,6 +27,7 @@ import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; import org.junit.After; import org.junit.Assert; +import org.junit.Assume; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -172,7 +173,7 @@ public class ShellTaskTest { @Test public void testHandleForWindows() throws Exception { try { - PowerMockito.when(OSUtils.isWindows()).thenReturn(true); + Assume.assumeTrue(OSUtils.isWindows()); shellTask.handle(); Assert.assertTrue(true); } catch (Error | Exception e) { From ebf7a980d2922a5659d8f0da28f519556aa3a2f0 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Tue, 10 Mar 2020 18:59:45 +0800 Subject: [PATCH 23/59] Adapting partial code(file name start with F) to the sonar cloud rule (#2045) * Adapting partial code(file name start with F) to the sonar cloud rule * add more unit test * add License * add includes configuration to maven-surefire-plugin * fix getResourceFilesList incorrect logic --- .../alert/utils/FuncUtils.java | 2 +- .../alert/utils/FuncUtilsTest.java | 2 +- .../common/task/flink/FlinkParameters.java | 14 +++-- .../common/utils/FileUtils.java | 6 +- .../common/task/FlinkParametersTest.java | 55 +++++++++++++++++++ .../common/utils/FileUtilsTest.java | 17 +++--- .../server/utils/FlinkArgsUtils.java | 16 ++---- .../worker/runner/TaskScheduleThread.java | 4 +- .../server/utils/FlinkArgsUtilsTest.java | 24 ++++---- pom.xml | 1 + 10 files changed, 97 insertions(+), 44 deletions(-) create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java index dd6ca4b8a6..d68532a82b 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java @@ -20,7 +20,7 @@ import org.apache.dolphinscheduler.common.utils.StringUtils; public class FuncUtils { - static public String mkString(Iterable list, String split) { + public static String mkString(Iterable list, String split) { if (null == list || StringUtils.isEmpty(split)){ return null; diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java index e2b25d961d..a4aeea9c0c 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java @@ -46,7 +46,7 @@ public class FuncUtilsTest { logger.info(result); //Expected result string - assertEquals(result, "user1|user2|user3"); + assertEquals("user1|user2|user3", result); //Null list expected return null result = FuncUtils.mkString(null, split); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java index 0638b3858e..1fbd9ab354 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -207,12 +208,15 @@ public class FlinkParameters extends AbstractParameters { @Override public List getResourceFilesList() { - if(resourceList !=null ) { - this.resourceList.add(mainJar); - return resourceList.stream() - .map(p -> p.getRes()).collect(Collectors.toList()); + if(resourceList != null ) { + List resourceFiles = resourceList.stream() + .map(ResourceInfo::getRes).collect(Collectors.toList()); + if(mainJar != null) { + resourceFiles.add(mainJar.getRes()); + } + return resourceFiles; } - return null; + return Collections.emptyList(); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java index c84848fbae..dc60b04c59 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java @@ -44,7 +44,7 @@ public class FileUtils { String fileSuffix = ""; if (StringUtils.isNotEmpty(filename)) { - int lastIndex = filename.lastIndexOf("."); + int lastIndex = filename.lastIndexOf('.'); if (lastIndex > 0) { fileSuffix = filename.substring(lastIndex + 1); } @@ -325,10 +325,8 @@ public class FileUtils { } } else { File parent = file.getParentFile(); - if (parent != null) { - if (!parent.mkdirs() && !parent.isDirectory()) { + if (parent != null && !parent.mkdirs() && !parent.isDirectory()) { throw new IOException("Directory '" + parent + "' could not be created"); - } } } return new FileOutputStream(file, append); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java new file mode 100644 index 0000000000..7ce00e875a --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.task; + +import org.apache.dolphinscheduler.common.process.ResourceInfo; +import org.apache.dolphinscheduler.common.task.flink.FlinkParameters; +import org.junit.Assert; +import org.junit.Test; + +import java.util.LinkedList; +import java.util.List; + +public class FlinkParametersTest { + @Test + public void getResourceFilesList() { + FlinkParameters flinkParameters = new FlinkParameters(); + Assert.assertNotNull(flinkParameters.getResourceFilesList()); + Assert.assertTrue(flinkParameters.getResourceFilesList().isEmpty()); + + ResourceInfo mainResource = new ResourceInfo(); + mainResource.setRes("testFlinkMain-1.0.0-SNAPSHOT.jar"); + flinkParameters.setMainJar(mainResource); + + List resourceInfos = new LinkedList<>(); + ResourceInfo resourceInfo1 = new ResourceInfo(); + resourceInfo1.setRes("testFlinkParameters1.jar"); + resourceInfos.add(resourceInfo1); + + flinkParameters.setResourceList(resourceInfos); + Assert.assertNotNull(flinkParameters.getResourceFilesList()); + Assert.assertEquals(2, flinkParameters.getResourceFilesList().size()); + + ResourceInfo resourceInfo2 = new ResourceInfo(); + resourceInfo2.setRes("testFlinkParameters2.jar"); + resourceInfos.add(resourceInfo2); + + flinkParameters.setResourceList(resourceInfos); + Assert.assertNotNull(flinkParameters.getResourceFilesList()); + Assert.assertEquals(3, flinkParameters.getResourceFilesList().size()); + } +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java index 89458f6f1c..96217842bf 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java @@ -30,29 +30,32 @@ public class FileUtilsTest { @Test public void suffix() { - Assert.assertEquals(FileUtils.suffix("ninfor.java"),"java"); + Assert.assertEquals("java", FileUtils.suffix("ninfor.java")); + Assert.assertEquals("", FileUtils.suffix(null)); + Assert.assertEquals("", FileUtils.suffix("")); + Assert.assertEquals("", FileUtils.suffix("ninfor-java")); } @Test public void testGetDownloadFilename() { PowerMockito.mockStatic(DateUtils.class); PowerMockito.when(DateUtils.getCurrentTime(YYYYMMDDHHMMSS)).thenReturn("20190101101059"); - Assert.assertEquals(FileUtils.getDownloadFilename("test"), - "/tmp/dolphinscheduler/download/20190101101059/test"); + Assert.assertEquals("/tmp/dolphinscheduler/download/20190101101059/test", + FileUtils.getDownloadFilename("test")); } @Test public void testGetUploadFilename() { - Assert.assertEquals(FileUtils.getUploadFilename("aaa","bbb"), - "/tmp/dolphinscheduler/aaa/resources/bbb"); + Assert.assertEquals("/tmp/dolphinscheduler/aaa/resources/bbb", + FileUtils.getUploadFilename("aaa","bbb")); } @Test public void testGetProcessExecDir() { String dir = FileUtils.getProcessExecDir(1,2,3, 4); - Assert.assertEquals(dir, "/tmp/dolphinscheduler/exec/process/1/2/3/4"); + Assert.assertEquals("/tmp/dolphinscheduler/exec/process/1/2/3/4", dir); dir = FileUtils.getProcessExecDir(1,2,3); - Assert.assertEquals(dir, "/tmp/dolphinscheduler/exec/process/1/2/3"); + Assert.assertEquals("/tmp/dolphinscheduler/exec/process/1/2/3", dir); } @Test diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java index 4c33ef8db2..12c7eb2d56 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java @@ -17,12 +17,11 @@ package org.apache.dolphinscheduler.server.utils; +import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.flink.FlinkParameters; -import org.apache.commons.lang.StringUtils; -import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; @@ -32,12 +31,7 @@ import java.util.List; * spark args utils */ public class FlinkArgsUtils { - - /** - * logger of FlinkArgsUtils - */ - private static final org.slf4j.Logger logger = LoggerFactory.getLogger(FlinkArgsUtils.class); - + private static final String LOCAL_DEPLOY_MODE = "local"; /** * build args * @param param flink parameters @@ -52,7 +46,7 @@ public class FlinkArgsUtils { deployMode = tmpDeployMode; } - if (!"local".equals(deployMode)) { + if (!LOCAL_DEPLOY_MODE.equals(deployMode)) { args.add(Constants.FLINK_RUN_MODE); //-m args.add(Constants.FLINK_YARN_CLUSTER); //yarn-cluster @@ -113,12 +107,12 @@ public class FlinkArgsUtils { String queue = param.getQueue(); if (StringUtils.isNotEmpty(others)) { - if (!others.contains(Constants.FLINK_QUEUE) && StringUtils.isNotEmpty(queue) && !deployMode.equals("local")) { + if (!others.contains(Constants.FLINK_QUEUE) && StringUtils.isNotEmpty(queue) && !deployMode.equals(LOCAL_DEPLOY_MODE)) { args.add(Constants.FLINK_QUEUE); args.add(param.getQueue()); } args.add(others); - } else if (StringUtils.isNotEmpty(queue) && !deployMode.equals("local")) { + } else if (StringUtils.isNotEmpty(queue) && !deployMode.equals(LOCAL_DEPLOY_MODE)) { args.add(Constants.FLINK_QUEUE); args.add(param.getQueue()); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java index 5e68acf94e..21ee1dfa40 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java @@ -296,9 +296,7 @@ public class TaskScheduleThread implements Runnable { if (baseParam != null) { List projectResourceFiles = baseParam.getResourceFilesList(); - if (projectResourceFiles != null) { - projectFiles.addAll(projectResourceFiles); - } + projectFiles.addAll(projectResourceFiles); } return new ArrayList<>(projectFiles); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java index 710d2c2505..2e4861e2a2 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java @@ -87,35 +87,35 @@ public class FlinkArgsUtilsTest { } //Expected values and order - assertEquals(result.size(),20); + assertEquals(20, result.size()); - assertEquals(result.get(0),"-m"); - assertEquals(result.get(1),"yarn-cluster"); + assertEquals("-m", result.get(0)); + assertEquals("yarn-cluster", result.get(1)); - assertEquals(result.get(2),"-ys"); + assertEquals("-ys", result.get(2)); assertSame(Integer.valueOf(result.get(3)),slot); - assertEquals(result.get(4),"-ynm"); + assertEquals("-ynm",result.get(4)); assertEquals(result.get(5),appName); - assertEquals(result.get(6),"-yn"); + assertEquals("-yn", result.get(6)); assertSame(Integer.valueOf(result.get(7)),taskManager); - assertEquals(result.get(8),"-yjm"); + assertEquals("-yjm", result.get(8)); assertEquals(result.get(9),jobManagerMemory); - assertEquals(result.get(10),"-ytm"); + assertEquals("-ytm", result.get(10)); assertEquals(result.get(11),taskManagerMemory); - assertEquals(result.get(12),"-d"); + assertEquals("-d", result.get(12)); - assertEquals(result.get(13),"-c"); + assertEquals("-c", result.get(13)); assertEquals(result.get(14),mainClass); assertEquals(result.get(15),mainJar.getRes()); assertEquals(result.get(16),mainArgs); - assertEquals(result.get(17),"--qu"); + assertEquals("--qu", result.get(17)); assertEquals(result.get(18),queue); assertEquals(result.get(19),others); @@ -125,7 +125,7 @@ public class FlinkArgsUtilsTest { param1.setQueue(queue); param1.setDeployMode(mode); result = FlinkArgsUtils.buildArgs(param1); - assertEquals(result.size(),5); + assertEquals(5, result.size()); } } \ No newline at end of file diff --git a/pom.xml b/pom.xml index a7feec0e14..8b91add6d7 100644 --- a/pom.xml +++ b/pom.xml @@ -691,6 +691,7 @@ **/common/threadutils/*.java **/common/graph/*.java **/common/queue/*.java + **/common/task/FlinkParametersTest.java **/common/task/SqoopParameterEntityTest.java **/api/utils/CheckUtilsTest.java **/api/utils/FileUtilsTest.java From ce175310245cf78be91339545cf1f0b8cdd168d2 Mon Sep 17 00:00:00 2001 From: AhahaGe Date: Tue, 10 Mar 2020 22:00:40 +0800 Subject: [PATCH 24/59] add .asf.yaml file (#2136) Co-authored-by: guirong.ggr --- .asf.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .asf.yaml diff --git a/.asf.yaml b/.asf.yaml new file mode 100644 index 0000000000..fa71a46552 --- /dev/null +++ b/.asf.yaml @@ -0,0 +1,10 @@ +staging: + profile: ~ + whoami: dev + foo: trigger + +publish: + whoami: dev + +github: + description: “Dolphin Scheduler is a distributed and easy-to-extend visual DAG workflow scheduling system, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.(分布式易扩展的可视化工作流任务调度)” \ No newline at end of file From 1db4ac6b63afedc2622aa7997cec701c2096e68f Mon Sep 17 00:00:00 2001 From: AhahaGe Date: Tue, 10 Mar 2020 22:28:40 +0800 Subject: [PATCH 25/59] edit README.md modify word from expand to extend (#2138) Co-authored-by: guirong.ggr --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0a9a164b85..ebd620efee 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Dolphin Scheduler Official Website ### Design features: -A distributed and easy-to-expand visual DAG workflow scheduling system. Dedicated to solving the complex dependencies in data processing, making the scheduling system `out of the box` for data processing. +A distributed and easy-to-extend visual DAG workflow scheduling system. Dedicated to solving the complex dependencies in data processing, making the scheduling system `out of the box` for data processing. Its main objectives are as follows: - Associate the Tasks according to the dependencies of the tasks in a DAG graph, which can visualize the running state of task in real time. From b03e0a02c5533ac6c6dc7189cef96b7f686e5497 Mon Sep 17 00:00:00 2001 From: Jave-Chen Date: Thu, 12 Mar 2020 10:09:13 +0800 Subject: [PATCH 26/59] fix wrong sonar analysis with checkout V2. (#2148) --- .github/workflows/ci_ut.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 48cd1e16ce..12f7c04ed6 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -15,7 +15,7 @@ # limitations under the License. # -on: ["pull_request"] +on: ["pull_request", "push"] env: DOCKER_DIR: ./docker LOG_DIR: /tmp/dolphinscheduler @@ -52,7 +52,15 @@ jobs: run: | export MAVEN_OPTS='-Dmaven.repo.local=.m2/repository -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:-UseGCOverheadLimit -Xmx3g' mvn test -B -Dmaven.test.skip=false + - name: Upload coverage report to codecov + if: github.event_name == 'pull_request' + run: | CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash) + - name: Git fetch unshallow + run: | + git fetch --unshallow + git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" + git fetch origin - name: Run SonarCloud Analysis run: > mvn verify --batch-mode From ebfacbfcfa73d3dbbace14a539f8fce0fd7a894d Mon Sep 17 00:00:00 2001 From: liwenhe1993 <32166572+liwenhe1993@users.noreply.github.com> Date: Thu, 12 Mar 2020 10:11:07 +0800 Subject: [PATCH 27/59] docker run failed when docker command was `docker run -d` (#2125) * docker run failed when docker command was `docker run -d` 1. add tini 2. replace `tee` command by `tail` command in startup.sh 3. api-server need to support zookeeper * modify `check.sh` in dockerfile * add init directories and log files --- dockerfile/Dockerfile | 6 +++--- dockerfile/hooks/check | 2 +- dockerfile/startup.sh | 8 ++++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/dockerfile/Dockerfile b/dockerfile/Dockerfile index 8fa4886f31..acd2cb99e5 100644 --- a/dockerfile/Dockerfile +++ b/dockerfile/Dockerfile @@ -23,11 +23,11 @@ ENV TZ Asia/Shanghai ENV LANG C.UTF-8 ENV DEBIAN_FRONTEND noninteractive -#1. install dos2unix shadow bash openrc python sudo vim wget iputils net-tools ssh pip kazoo. +#1. install dos2unix shadow bash openrc python sudo vim wget iputils net-tools ssh pip tini kazoo. #If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example: #RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories RUN apk update && \ - apk add dos2unix shadow bash openrc python sudo vim wget iputils net-tools openssh-server py2-pip && \ + apk add dos2unix shadow bash openrc python sudo vim wget iputils net-tools openssh-server py2-pip tini && \ apk add --update procps && \ openrc boot && \ pip install kazoo @@ -92,4 +92,4 @@ RUN rm -rf /var/cache/apk/* #9. expose port EXPOSE 2181 2888 3888 5432 12345 50051 8888 -ENTRYPOINT ["/root/startup.sh"] \ No newline at end of file +ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"] \ No newline at end of file diff --git a/dockerfile/hooks/check b/dockerfile/hooks/check index fbc3f8bd44..21f3f4db12 100644 --- a/dockerfile/hooks/check +++ b/dockerfile/hooks/check @@ -17,7 +17,7 @@ # echo "------ dolphinscheduler check - server - status -------" sleep 20 -server_num=$(docker top `docker container list | grep startup | awk '{print $1}'`| grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) +server_num=$(docker top `docker container list | grep '/sbin/tini' | awk '{print $1}'`| grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) if [ $server_num -eq 5 ] then echo "Server all start successfully" diff --git a/dockerfile/startup.sh b/dockerfile/startup.sh index 610a86f5d3..30439c2321 100644 --- a/dockerfile/startup.sh +++ b/dockerfile/startup.sh @@ -164,6 +164,7 @@ case "$1" in LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log ;; (api-server) + initZK initPostgreSQL initApiServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log @@ -187,6 +188,9 @@ case "$1" in ;; esac -echo "tee begin" -exec tee ${LOGFILE} +# init directories and log files +mkdir -p ${DOLPHINSCHEDULER_LOGS} && mkdir -p /var/log/nginx/ && cat /dev/null >> ${LOGFILE} + +echo "tail begin" +exec bash -c "tail -n 1 -f ${LOGFILE}" From e27ba3fd978006a56d26bbf1f7d15255296af5fe Mon Sep 17 00:00:00 2001 From: tswstarplanet Date: Thu, 12 Mar 2020 17:38:38 +0800 Subject: [PATCH 28/59] use logger to print exception (#2151) --- .../org/apache/dolphinscheduler/common/shell/AbstractShell.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java index 101da18b2c..f846b19741 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java @@ -335,7 +335,7 @@ public abstract class AbstractShell { try{ entry.getValue().destroy(); } catch (Exception e) { - e.printStackTrace(); + logger.error("Destroy All Processes error", e); } } From e3065dbfcc026639420aab91e9fae4246e593c69 Mon Sep 17 00:00:00 2001 From: tswstarplanet Date: Thu, 12 Mar 2020 17:39:25 +0800 Subject: [PATCH 29/59] fix field access modifier (#2150) --- .../org/apache/dolphinscheduler/api/utils/ZooKeeperState.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java index 5aa6be858a..a33754397d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java @@ -121,7 +121,7 @@ public class ZooKeeperState { private class SendThread extends Thread { private String cmd; - public String ret = ""; + private String ret = ""; public SendThread(String cmd) { this.cmd = cmd; From afb907b4b019cbc9f0ed85c25854e9c381561a4f Mon Sep 17 00:00:00 2001 From: tswstarplanet Date: Thu, 12 Mar 2020 17:41:10 +0800 Subject: [PATCH 30/59] fix unequal number of arguments and placeholder (#2149) --- .../java/org/apache/dolphinscheduler/alert/utils/MailUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java index 99efdc8a6a..130ad8ade6 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java @@ -334,7 +334,7 @@ public class MailUtils { * @param e the exception */ private static void handleException(Collection receivers, Map retMap, Exception e) { - logger.error("Send email to {} failed {}", receivers, e); + logger.error("Send email to {} failed", receivers, e); retMap.put(Constants.MESSAGE, "Send email to {" + String.join(",", receivers) + "} failed，" + e.toString()); } From 46e30b1e50727bba7a553baf64e32bb07de14478 Mon Sep 17 00:00:00 2001 From: t1mon <178317391@qq.com> Date: Thu, 12 Mar 2020 17:50:37 +0800 Subject: [PATCH 31/59] Add mail server configuration description information (#2143) * Add mail server configuration description information * Remove mail server configuration note in alert.properties --- dolphinscheduler-alert/src/main/resources/alert.properties | 1 - install.sh | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-alert/src/main/resources/alert.properties b/dolphinscheduler-alert/src/main/resources/alert.properties index 000d0653b7..9f5acea188 100644 --- a/dolphinscheduler-alert/src/main/resources/alert.properties +++ b/dolphinscheduler-alert/src/main/resources/alert.properties @@ -28,7 +28,6 @@ mail.server.port=25 mail.sender=xxx@xxx.com mail.user=xxx@xxx.com mail.passwd=111111 - # TLS mail.smtp.starttls.enable=true # SSL diff --git a/install.sh b/install.sh index 8735543c47..ed66ce5bd9 100644 --- a/install.sh +++ b/install.sh @@ -109,6 +109,7 @@ mailProtocol="SMTP" mailServerHost="smtp.exmail.qq.com" # mail server port +# note: Different protocols and encryption methods correspond to different ports, when SSL/TLS is enabled, make sure the port is correct. mailServerPort="25" # sender @@ -118,6 +119,7 @@ mailSender="xxxxxxxxxx" mailUser="xxxxxxxxxx" # sender password +# note: The mail.passwd is email service authorization code, not the email login password. mailPassword="xxxxxxxxxx" # TLS mail protocol support From 712a1bc126ed32259edaaa9e2c8b6afb71fd2f6e Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Thu, 12 Mar 2020 17:59:16 +0800 Subject: [PATCH 32/59] Limit customization file content to no more than 3000 lines(#2128) (#2141) * Change DOM label * Change name to lowercase * Limit customization file content to no more than 3000 lines --- .../pages/file/pages/create/index.vue | 5 ++- .../resource/pages/file/pages/edit/index.vue | 38 ++++++++++++------- .../src/js/module/i18n/locale/en_US.js | 1 + .../src/js/module/i18n/locale/zh_CN.js | 1 + 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue index caa31ed4d8..df84f0f292 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue @@ -132,7 +132,10 @@ this.$message.warning(`${i18n.$t('Please enter the resource content')}`) return false } - + if (editor.doc.size>3000) { + this.$message.warning(`${i18n.$t('Resource content cannot exceed 3000 lines')}`) + return false + } return true }, /** diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue index a9aee1c9fd..a0d1d7d187 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue @@ -45,6 +45,7 @@ @@ -257,4 +272,4 @@ } } } - \ No newline at end of file + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sqoop.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sqoop.vue index eead745a06..6594ffcf74 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sqoop.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sqoop.vue @@ -65,7 +65,7 @@
@@ -186,8 +186,8 @@
@@ -385,7 +385,7 @@
@@ -556,7 +556,8 @@ targetType:"HDFS", sourceMysqlParams:{ - srcDatasource:-1, + srcType:"MYSQL", + srcDatasource:"", srcTable:"", srcQueryType:"1", srcQuerySql:'', @@ -588,7 +589,8 @@ }, targetMysqlParams:{ - targetDatasource:-1, + targetType:"MYSQL", + targetDatasource:"", targetTable:"", targetColumns:"", fieldsTerminated:"", @@ -680,6 +682,7 @@ * return data source */ _onSourceDsData (o) { + this.sourceMysqlParams.srcType = o.type this.sourceMysqlParams.srcDatasource = o.datasource }, @@ -687,6 +690,7 @@ * return data source */ _onTargetDsData (o) { + this.targetMysqlParams.targetType = o.type this.targetMysqlParams.targetDatasource = o.datasource }, @@ -697,7 +701,7 @@ var params = null switch(this.sourceType){ case "MYSQL": - this.sourceMysqlParams.srcQuerySql = editor.getValue() + this.sourceMysqlParams.srcQuerySql = editor ? editor.getValue() : this.sourceMysqlParams.srcQuerySql params = JSON.stringify(this.sourceMysqlParams) break; case "ORACLE": @@ -879,7 +883,9 @@ * Processing code highlighting */ _handlerEditor () { - editor = codemirror('code-sql-mirror', { + this._destroyEditor() + + editor = codemirror('code-sqoop-mirror', { mode: 'sql', readOnly: this.isDetails }) @@ -892,9 +898,15 @@ } } + this.changes = () => { + this._cacheParams() + } + // Monitor keyboard editor.on('keypress', this.keypress) + editor.on('changes', this.changes) + editor.setValue(this.sourceMysqlParams.srcQuerySql) return editor @@ -906,6 +918,27 @@ _onLocalParams (a) { this.localParams = a }, + + _cacheParams () { + this.$emit('on-cache-params', { + concurrency:this.concurrency, + modelType:this.modelType, + sourceType:this.sourceType, + targetType:this.targetType, + sourceParams:this._handleSourceParams(), + targetParams:this._handleTargetParams(), + localParams:this.localParams + }); + }, + + _destroyEditor () { + if (editor) { + editor.toTextArea() // Uninstall + editor.off($('.code-sqoop-mirror'), 'keypress', this.keypress) + editor.off($('.code-sqoop-mirror'), 'changes', this.changes) + editor = null + } + }, }, watch: { // Listening to sqlType @@ -927,11 +960,12 @@ }, //Watch the cacheParams cacheParams (val) { - this.$emit('on-cache-params', val); + this._cacheParams() } }, created () { + this._destroyEditor() let o = this.backfillItem // Non-null objects represent backfill @@ -963,11 +997,28 @@ */ if (editor) { editor.toTextArea() // Uninstall - editor.off($('.code-sql-mirror'), 'keypress', this.keypress) + editor.off($('.code-sqoop-mirror'), 'keypress', this.keypress) + editor.off($('.code-sqoop-mirror'), 'changes', this.changes) + editor = null } }, computed: { + cacheParams () { + return { + concurrency:this.concurrency, + modelType:this.modelType, + sourceType:this.sourceType, + targetType:this.targetType, + localParams:this.localParams, + sourceMysqlParams:this.sourceMysqlParams, + sourceHdfsParams:this.sourceHdfsParams, + sourceHiveParams:this.sourceHiveParams, + targetHdfsParams:this.targetHdfsParams, + targetMysqlParams:this.targetMysqlParams, + targetHiveParams:this.targetHiveParams + } + } }, components: { mListBox, mDatasource, mLocalParams} } From 91ecc549a1d7cfae2ead6eb16b5e5559d13f54e0 Mon Sep 17 00:00:00 2001 From: Yelli Date: Thu, 12 Mar 2020 21:52:48 +0800 Subject: [PATCH 36/59] fix bug: data too long for column 'app_link' (#2146) * fix bug: data to long for app_link field * change app_link field length in 1.2.2 ddl * change app_link field type to text --- .../mysql/dolphinscheduler_ddl.sql | 22 ++++++++++++++++++- .../postgresql/dolphinscheduler_ddl.sql | 18 +++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql b/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql index e5785d9997..049484ce3a 100644 --- a/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql @@ -54,4 +54,24 @@ d// delimiter ; CALL uc_dolphin_T_t_ds_task_instance_A_executor_id; -DROP PROCEDURE uc_dolphin_T_t_ds_task_instance_A_executor_id; \ No newline at end of file +DROP PROCEDURE uc_dolphin_T_t_ds_task_instance_A_executor_id; + +-- uc_dolphin_T_t_ds_task_instance_C_app_link +drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_task_instance_C_app_link; +delimiter d// +CREATE PROCEDURE uc_dolphin_T_t_ds_task_instance_C_app_link() + BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_task_instance' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='app_link') + THEN + ALTER TABLE t_ds_task_instance CHANGE COLUMN app_link app_link text COMMENT 'yarn app id'; + END IF; + END; + +d// + +delimiter ; +CALL uc_dolphin_T_t_ds_task_instance_C_app_link; +DROP PROCEDURE uc_dolphin_T_t_ds_task_instance_C_app_link; \ No newline at end of file diff --git a/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql b/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql index 70471cb26d..b1e0fd941c 100644 --- a/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql @@ -49,3 +49,21 @@ d// delimiter ; SELECT uc_dolphin_T_t_ds_task_instance_A_executor_id(); DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_task_instance_A_executor_id(); + +-- uc_dolphin_T_t_ds_task_instance_C_app_link +delimiter d// +CREATE OR REPLACE FUNCTION uc_dolphin_T_t_ds_task_instance_C_app_link() RETURNS void AS $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_task_instance' + AND COLUMN_NAME ='app_link') + THEN + ALTER TABLE t_ds_task_instance ALTER COLUMN app_link type text; + END IF; +END; +$$ LANGUAGE plpgsql; +d// + +delimiter ; +SELECT uc_dolphin_T_t_ds_task_instance_C_app_link(); +DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_task_instance_C_app_link(); \ No newline at end of file From 455d8c7a7c36fca1259d0039910c07a716ff76dd Mon Sep 17 00:00:00 2001 From: dailidong Date: Fri, 13 Mar 2020 11:13:17 +0800 Subject: [PATCH 37/59] Delete .asf.yaml (#2158) --- .asf.yaml | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 .asf.yaml diff --git a/.asf.yaml b/.asf.yaml deleted file mode 100644 index fa71a46552..0000000000 --- a/.asf.yaml +++ /dev/null @@ -1,10 +0,0 @@ -staging: - profile: ~ - whoami: dev - foo: trigger - -publish: - whoami: dev - -github: - description: “Dolphin Scheduler is a distributed and easy-to-extend visual DAG workflow scheduling system, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.(分布式易扩展的可视化工作流任务调度)” \ No newline at end of file From 93aa160698a584bbcb8cdb45953efe1cc2b78915 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Fri, 13 Mar 2020 13:02:51 +0800 Subject: [PATCH 38/59] Adapting partial code(file name start with I) to the sonar cloud rule (#2155) --- .../common/utils/IOUtils.java | 19 +++++-------------- .../common/utils/IpUtils.java | 5 ----- .../common/utils/IpUtilsTest.java | 6 +++--- 3 files changed, 8 insertions(+), 22 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java index 73df158aa3..ce551d8405 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java @@ -19,26 +19,17 @@ package org.apache.dolphinscheduler.common.utils; +import java.io.Closeable; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; public class IOUtils { - public static void closeQuietly(InputStream fis){ - if(fis != null){ + public static void closeQuietly(Closeable closeable){ + if(closeable != null){ try { - fis.close(); - } catch (IOException ignore) { - } - } - } - - public static void closeQuietly(InputStreamReader reader){ - if(reader != null){ - try { - reader.close(); + closeable.close(); } catch (IOException ignore) { + // nothing need to do } } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java index e7e0b34bdd..3b068c60d2 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java @@ -17,16 +17,11 @@ package org.apache.dolphinscheduler.common.utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - /** * http utils */ public class IpUtils { - private static final Logger logger = LoggerFactory.getLogger(IpUtils.class); public static final String DOT = "."; /** diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/IpUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/IpUtilsTest.java index e65bcd219b..ec6ffa35a7 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/IpUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/IpUtilsTest.java @@ -29,13 +29,13 @@ public class IpUtilsTest { long longNumber = IpUtils.ipToLong(ip); long longNumber2 = IpUtils.ipToLong(ip2); System.out.println(longNumber); - Assert.assertEquals(longNumber, 3232263681L); - Assert.assertEquals(longNumber2, 0L); + Assert.assertEquals(3232263681L, longNumber); + Assert.assertEquals(0L, longNumber2); String ip3 = "255.255.255.255"; long longNumber3 = IpUtils.ipToLong(ip3); System.out.println(longNumber3); - Assert.assertEquals(longNumber3, 4294967295L); + Assert.assertEquals(4294967295L, longNumber3); } From a0088c113726506b3b1c74ee2411a39ab184d2be Mon Sep 17 00:00:00 2001 From: gaojun2048 <32193458+gaojun2048@users.noreply.github.com> Date: Fri, 13 Mar 2020 13:04:33 +0800 Subject: [PATCH 39/59] fix rpmbuild and DS Ambari plugin bug (#2144) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update quick-start.md * 更新demo的地址 * add rpm plugin * delete the git ignore files * 测试git ignore * 测试ignore * optimize the .gitignore file * add rpm package, delete unuse copy jar action in assembly * delete docs dir * add rpm plugin support * add rpm plugin support * add rpm plugin support * dolphinscheduler-common have not bin dir * dolphinscheduler-common have not bin dir * delete unuse config about assembly * add defineSatement * add rpm plugin support * add rpm plugin support * add rpm plugin support * add rpm support * update install dirname , make front and backend together * update rpm name * update rpm name * update rpm config * add jars excludes * add jars excludes * add rpm plugion * add rpm plugion * add rpm plugion * add auto create /opt/soft * The install dir can not be fixed to apache-dolphinscheduler-incubating-1.2.1 , its version is 1.2.2 * install from rpm will create a link * the ui install thrown rpm only support jetty not nginx * the ui install thrown rpm only support jetty not nginx * sql目录里加上soft_version文件 * env dir need in conf dir * remove no use module * add module * updaate * rpmbuild add dolphinscheduler-service module --- .../DOLPHIN/1.2.1/package/scripts/params.py | 2 +- dolphinscheduler-dist/pom.xml | 53 +++++++++--------- dolphinscheduler-ui/pom.xml | 55 +++++++++++++++++++ pom.xml | 2 +- 4 files changed, 83 insertions(+), 29 deletions(-) diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py b/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py index 230ad14565..3780f6c27e 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py +++ b/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py @@ -30,7 +30,7 @@ sys.setdefaultencoding('utf-8') config = Script.get_config() # conf_dir = "/etc/" -dolphin_home = "/opt/soft/apache-dolphinscheduler-incubating-1.2.1" +dolphin_home = "/opt/soft/dolphinscheduler" dolphin_conf_dir = dolphin_home + "/conf" dolphin_log_dir = dolphin_home + "/logs" dolphin_bin_dir = dolphin_home + "/bin" diff --git a/dolphinscheduler-dist/pom.xml b/dolphinscheduler-dist/pom.xml index 80ddfec9b0..0ead2b9880 100644 --- a/dolphinscheduler-dist/pom.xml +++ b/dolphinscheduler-dist/pom.xml @@ -220,10 +220,7 @@ ${basedir}/../dolphinscheduler-alert/src/main/resources - **/*.properties - **/*.xml - **/*.json - **/*.ftl + **/*.* @@ -232,9 +229,7 @@ ${basedir}/../dolphinscheduler-common/src/main/resources - **/*.properties - **/*.xml - **/*.json + **/*.* @@ -243,10 +238,7 @@ ${basedir}/../dolphinscheduler-dao/src/main/resources - **/*.properties - **/*.xml - **/*.json - **/*.yml + **/*.* @@ -255,9 +247,7 @@ ${basedir}/../dolphinscheduler-api/src/main/resources - **/*.properties - **/*.xml - **/*.json + **/*.* @@ -266,13 +256,19 @@ ${basedir}/../dolphinscheduler-server/src/main/resources - **/*.properties - **/*.xml - **/*.json config/*.* + + + ${basedir}/../dolphinscheduler-service/src/main/resources + + + *.* + + + ${basedir}/../script @@ -342,14 +338,6 @@ - - - ${basedir}/../dolphinscheduler-ui - - - install-dolphinscheduler-ui.sh - - ${basedir}/release-docs @@ -362,7 +350,7 @@ - /opt/soft/${project.build.finalName}/dist + /opt/soft/${project.build.finalName}/ui 755 root root @@ -391,6 +379,14 @@ **/*.* + + + ${basedir}/../sql + + + soft_version + + @@ -405,7 +401,7 @@ ${basedir}/../script - **/*.* + *.sh @@ -416,6 +412,9 @@ + + + diff --git a/dolphinscheduler-ui/pom.xml b/dolphinscheduler-ui/pom.xml index 3fd9aa6650..78869ffbc4 100644 --- a/dolphinscheduler-ui/pom.xml +++ b/dolphinscheduler-ui/pom.xml @@ -89,6 +89,61 @@ + + rpmbuild + + + + com.github.eirslett + frontend-maven-plugin + ${frontend-maven-plugin.version} + + + install node and npm + + install-node-and-npm + + + ${node.version} + ${npm.version} + + + + npm install node-sass --unsafe-perm + + npm + + generate-resources + + install node-sass --unsafe-perm + + + + npm install + + npm + + generate-resources + + install + + + + npm run build:release + + npm + + + run build:release + + + + + + + + + nginx diff --git a/pom.xml b/pom.xml index 8b91add6d7..8910723117 100644 --- a/pom.xml +++ b/pom.xml @@ -885,4 +885,4 @@ dolphinscheduler-service - + \ No newline at end of file From dd2c50a10d13811f7eb71a3c1bb9d5ab712f8dcd Mon Sep 17 00:00:00 2001 From: liwenhe1993 <32166572+liwenhe1993@users.noreply.github.com> Date: Fri, 13 Mar 2020 23:38:38 +0800 Subject: [PATCH 40/59] Toolbar of the DAG has not prompt (#2161) * Toolbar of the DAG has not prompt * Remove duplication code --- dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue | 2 +- dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js | 1 - dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue index 7d6f95d753..a1ccd39260 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue @@ -25,7 +25,7 @@ :key="v" v-for="(item,v) in tasksTypeList" @mousedown="_getDagId(v)"> -
+
diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js index 4507250aba..64ab298437 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js @@ -567,7 +567,6 @@ export default { 'Data Target': 'Data Target', 'All Columns': 'All Columns', 'Some Columns': 'Some Columns', - 'Modify User': 'Modify User', 'Branch flow': 'Branch flow', 'Cannot select the same node for successful branch flow and failed branch flow': 'Cannot select the same node for successful branch flow and failed branch flow' } diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js index 132b4d8745..3363eea198 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -567,7 +567,6 @@ export default { 'Data Target': '数据目的', 'All Columns': '全表导入', 'Some Columns': '选择列', - 'Modify User': '修改用户', 'Branch flow': '分支流转', 'Cannot select the same node for successful branch flow and failed branch flow': '成功分支流转和失败分支流转不能选择同一个节点' } From daad5efbabe15f3d37193f60083a2f4a29da54b7 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Fri, 13 Mar 2020 23:53:22 +0800 Subject: [PATCH 41/59] use static method in static class JSON (#2156) --- .../alert/utils/JSONUtils.java | 6 ++--- .../api/service/DataSourceService.java | 25 ++++++++++--------- .../controller/ResourcesControllerTest.java | 23 +++++++++-------- .../common/model/TaskNode.java | 4 +-- .../common/utils/JSONUtils.java | 15 +++++------ .../common/utils/ParameterUtils.java | 4 +-- .../common/utils/JSONUtilsTest.java | 4 +-- .../common/utils/ParameterUtilsTest.java | 8 +++--- .../dao/entity/ProcessDefinition.java | 8 +++--- .../master/runner/MasterExecThread.java | 4 +-- .../master/runner/MasterTaskExecThread.java | 4 +-- .../worker/runner/TaskScheduleThread.java | 6 ++--- .../task/processdure/ProcedureTask.java | 4 +-- .../server/worker/task/sql/SqlTask.java | 9 ++++--- .../server/master/MasterExecThreadTest.java | 4 +-- .../shell/ShellCommandExecutorTest.java | 4 +-- .../server/worker/sql/SqlExecutorTest.java | 4 +-- .../worker/task/sqoop/SqoopTaskTest.java | 10 ++++---- .../service/process/ProcessService.java | 7 +++--- 19 files changed, 79 insertions(+), 74 deletions(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java index 1cd9f490b2..8d66255fc4 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java @@ -16,8 +16,8 @@ */ package org.apache.dolphinscheduler.alert.utils; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; -import com.alibaba.fastjson.JSONObject; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,7 +38,7 @@ public class JSONUtils { */ public static String toJsonString(Object object) { try{ - return JSONObject.toJSONString(object,false); + return JSON.toJSONString(object,false); } catch (Exception e) { throw new RuntimeException("Json deserialization exception.", e); } @@ -57,7 +57,7 @@ public class JSONUtils { return null; } try { - return JSONArray.parseArray(json, clazz); + return JSON.parseArray(json, clazz); } catch (Exception e) { logger.error("JSONArray.parseArray exception!",e); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java index f6d8903dd8..6a732fed0e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.api.service; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; @@ -303,7 +304,7 @@ public class DataSourceService extends BaseService{ for (DataSource dataSource : dataSourceList) { String connectionParams = dataSource.getConnectionParams(); - JSONObject object = JSONObject.parseObject(connectionParams); + JSONObject object = JSON.parseObject(connectionParams); object.put(Constants.PASSWORD, Constants.XXXXXX); dataSource.setConnectionParams(JSONUtils.toJson(object)); @@ -367,11 +368,11 @@ public class DataSourceService extends BaseService{ try { switch (dbType) { case POSTGRESQL: - datasource = JSONObject.parseObject(parameter, PostgreDataSource.class); + datasource = JSON.parseObject(parameter, PostgreDataSource.class); Class.forName(Constants.ORG_POSTGRESQL_DRIVER); break; case MYSQL: - datasource = JSONObject.parseObject(parameter, MySQLDataSource.class); + datasource = JSON.parseObject(parameter, MySQLDataSource.class); Class.forName(Constants.COM_MYSQL_JDBC_DRIVER); break; case HIVE: @@ -386,26 +387,26 @@ public class DataSourceService extends BaseService{ getString(org.apache.dolphinscheduler.common.Constants.LOGIN_USER_KEY_TAB_PATH)); } if (dbType == DbType.HIVE){ - datasource = JSONObject.parseObject(parameter, HiveDataSource.class); + datasource = JSON.parseObject(parameter, HiveDataSource.class); }else if (dbType == DbType.SPARK){ - datasource = JSONObject.parseObject(parameter, SparkDataSource.class); + datasource = JSON.parseObject(parameter, SparkDataSource.class); } Class.forName(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER); break; case CLICKHOUSE: - datasource = JSONObject.parseObject(parameter, ClickHouseDataSource.class); + datasource = JSON.parseObject(parameter, ClickHouseDataSource.class); Class.forName(Constants.COM_CLICKHOUSE_JDBC_DRIVER); break; case ORACLE: - datasource = JSONObject.parseObject(parameter, OracleDataSource.class); + datasource = JSON.parseObject(parameter, OracleDataSource.class); Class.forName(Constants.COM_ORACLE_JDBC_DRIVER); break; case SQLSERVER: - datasource = JSONObject.parseObject(parameter, SQLServerDataSource.class); + datasource = JSON.parseObject(parameter, SQLServerDataSource.class); Class.forName(Constants.COM_SQLSERVER_JDBC_DRIVER); break; case DB2: - datasource = JSONObject.parseObject(parameter, DB2ServerDataSource.class); + datasource = JSON.parseObject(parameter, DB2ServerDataSource.class); Class.forName(Constants.COM_DB2_JDBC_DRIVER); break; default: @@ -507,7 +508,7 @@ public class DataSourceService extends BaseService{ parameterMap.put(Constants.PRINCIPAL,principal); } if (other != null && !"".equals(other)) { - LinkedHashMap map = JSONObject.parseObject(other, new TypeReference>() { + LinkedHashMap map = JSON.parseObject(other, new TypeReference>() { }); if (map.size() > 0) { StringBuilder otherSb = new StringBuilder(); @@ -523,9 +524,9 @@ public class DataSourceService extends BaseService{ } if(logger.isDebugEnabled()){ - logger.info("parameters map-----" + JSONObject.toJSONString(parameterMap)); + logger.info("parameters map-----" + JSON.toJSONString(parameterMap)); } - return JSONObject.toJSONString(parameterMap); + return JSON.toJSONString(parameterMap); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java index 40bdd5490d..a56e3f83ef 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.api.controller; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.ResourceType; @@ -54,7 +55,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -78,7 +79,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -281,7 +282,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -303,7 +304,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -324,7 +325,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -344,7 +345,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -365,7 +366,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -386,7 +387,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -406,7 +407,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -427,7 +428,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -446,7 +447,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java index b45bd8aeb8..c0ad907dca 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.common.model; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; @@ -23,7 +24,6 @@ import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import com.alibaba.fastjson.JSONObject; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -294,7 +294,7 @@ public class TaskNode { if(StringUtils.isNotEmpty(this.getTimeout())){ String formatStr = String.format("%s,%s", TaskTimeoutStrategy.WARN.name(), TaskTimeoutStrategy.FAILED.name()); String timeout = this.getTimeout().replace(formatStr,TaskTimeoutStrategy.WARNFAILED.name()); - return JSONObject.parseObject(timeout,TaskTimeoutParameter.class); + return JSON.parseObject(timeout,TaskTimeoutParameter.class); } return new TaskTimeoutParameter(false); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java index ec523b1ff2..b6ed71f402 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.TypeReference; @@ -59,7 +60,7 @@ public class JSONUtils { */ public static String toJson(Object object) { try{ - return JSONObject.toJSONString(object,false); + return JSON.toJSONString(object,false); } catch (Exception e) { logger.error("object to json exception!",e); } @@ -89,7 +90,7 @@ public class JSONUtils { } try { - return JSONObject.parseObject(json, clazz); + return JSON.parseObject(json, clazz); } catch (Exception e) { logger.error("parse object exception!",e); } @@ -178,7 +179,7 @@ public class JSONUtils { } try { - return JSONObject.parseObject(json, new TypeReference>(){}); + return JSON.parseObject(json, new TypeReference>(){}); } catch (Exception e) { logger.error("json to map exception!",e); } @@ -203,7 +204,7 @@ public class JSONUtils { } try { - return JSONObject.parseObject(json, new TypeReference>() {}); + return JSON.parseObject(json, new TypeReference>() {}); } catch (Exception e) { logger.error("json to map exception!",e); } @@ -218,7 +219,7 @@ public class JSONUtils { */ public static String toJsonString(Object object) { try{ - return JSONObject.toJSONString(object,false); + return JSON.toJSONString(object,false); } catch (Exception e) { throw new RuntimeException("Json deserialization exception.", e); } @@ -226,7 +227,7 @@ public class JSONUtils { public static JSONObject parseObject(String text) { try{ - return JSONObject.parseObject(text); + return JSON.parseObject(text); } catch (Exception e) { throw new RuntimeException("Json deserialization exception.", e); } @@ -234,7 +235,7 @@ public class JSONUtils { public static JSONArray parseArray(String text) { try{ - return JSONObject.parseArray(text); + return JSON.parseArray(text); } catch (Exception e) { throw new RuntimeException("Json deserialization exception.", e); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java index 9492b49cb1..7a4553aaf5 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DataType; @@ -23,7 +24,6 @@ import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils; import org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils; -import com.alibaba.fastjson.JSONObject; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.time.DateUtils; import org.slf4j.Logger; @@ -157,7 +157,7 @@ public class ParameterUtils { property.setValue(val); } } - return JSONObject.toJSONString(globalParamList); + return JSON.toJSONString(globalParamList); } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java index bd924e4852..d249314819 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java @@ -16,10 +16,10 @@ */ package org.apache.dolphinscheduler.common.utils; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.Direct; import org.apache.dolphinscheduler.common.process.Property; -import com.alibaba.fastjson.JSONObject; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import org.junit.Assert; @@ -53,7 +53,7 @@ public class JSONUtilsTest { property.setType(DataType.VARCHAR); property.setValue("sssssss"); String str = "{\"direct\":\"IN\",\"prop\":\"ds\",\"type\":\"VARCHAR\",\"value\":\"sssssss\"}"; - Property property1 = JSONObject.parseObject(str, Property.class); + Property property1 = JSON.parseObject(str, Property.class); Direct direct = property1.getDirect(); Assert.assertEquals(direct , Direct.IN); } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java index 8bb64b03c8..abdc15cc6e 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import org.apache.commons.lang.time.DateUtils; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DataType; @@ -91,13 +91,13 @@ public class ParameterUtilsTest { globalParamList.add(property); String result2 = ParameterUtils.curingGlobalParams(null,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,scheduleTime); - Assert.assertEquals(result2, JSONObject.toJSONString(globalParamList)); + Assert.assertEquals(result2, JSON.toJSONString(globalParamList)); String result3 = ParameterUtils.curingGlobalParams(globalParamMap,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,null); - Assert.assertEquals(result3, JSONObject.toJSONString(globalParamList)); + Assert.assertEquals(result3, JSON.toJSONString(globalParamList)); String result4 = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime); - Assert.assertEquals(result4, JSONObject.toJSONString(globalParamList)); + Assert.assertEquals(result4, JSON.toJSONString(globalParamList)); //test var $ startsWith globalParamMap.put("bizDate","${system.biz.date}"); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java index dbb880c025..f59d11f3fe 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java @@ -16,10 +16,10 @@ */ package org.apache.dolphinscheduler.dao.entity; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.process.Property; -import com.alibaba.fastjson.JSONObject; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; @@ -266,7 +266,7 @@ public class ProcessDefinition { } public void setGlobalParams(String globalParams) { - this.globalParamList = JSONObject.parseArray(globalParams, Property.class); + this.globalParamList = JSON.parseArray(globalParams, Property.class); this.globalParams = globalParams; } @@ -275,7 +275,7 @@ public class ProcessDefinition { } public void setGlobalParamList(List globalParamList) { - this.globalParams = JSONObject.toJSONString(globalParamList); + this.globalParams = JSON.toJSONString(globalParamList); this.globalParamList = globalParamList; } @@ -283,7 +283,7 @@ public class ProcessDefinition { List propList; if (globalParamMap == null && StringUtils.isNotEmpty(globalParams)) { - propList = JSONObject.parseArray(globalParams, Property.class); + propList = JSON.parseArray(globalParams, Property.class); globalParamMap = propList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 0b81a30191..849bbd5680 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.server.master.runner; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import com.google.common.collect.Lists; import org.apache.commons.io.FileUtils; import org.apache.dolphinscheduler.common.Constants; @@ -454,7 +454,7 @@ public class MasterExecThread implements Runnable { // process instance id taskInstance.setProcessInstanceId(processInstance.getId()); // task instance node json - taskInstance.setTaskJson(JSONObject.toJSONString(taskNode)); + taskInstance.setTaskJson(JSON.toJSONString(taskNode)); // task instance type taskInstance.setTaskType(taskNode.getType()); // task instance whether alert diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java index 37262c05e2..3cdb8a0271 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.server.master.runner; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; @@ -25,7 +26,6 @@ import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import com.alibaba.fastjson.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -176,7 +176,7 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { */ private TaskTimeoutParameter getTaskTimeoutParameter(){ String taskJson = taskInstance.getTaskJson(); - TaskNode taskNode = JSONObject.parseObject(taskJson, TaskNode.class); + TaskNode taskNode = JSON.parseObject(taskJson, TaskNode.class); return taskNode.getTaskTimeoutParameter(); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java index 21ee1dfa40..d36d4de5b4 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.server.worker.runner; import ch.qos.logback.classic.LoggerContext; import ch.qos.logback.classic.sift.SiftingAppender; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; @@ -93,7 +93,7 @@ public class TaskScheduleThread implements Runnable { logger.info("script path : {}", taskInstance.getExecutePath()); // task node - TaskNode taskNode = JSONObject.parseObject(taskInstance.getTaskJson(), TaskNode.class); + TaskNode taskNode = JSON.parseObject(taskInstance.getTaskJson(), TaskNode.class); // get resource files List resourceFiles = createProjectResFiles(taskNode); @@ -176,7 +176,7 @@ public class TaskScheduleThread implements Runnable { String globalParamsStr = taskInstance.getProcessInstance().getGlobalParams(); if (globalParamsStr != null) { - List globalParamsList = JSONObject.parseArray(globalParamsStr, Property.class); + List globalParamsList = JSON.parseArray(globalParamsStr, Property.class); globalParamsMap.putAll(globalParamsList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue))); } return globalParamsMap; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java index fb881453e9..fd00e517b5 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.server.worker.task.processdure; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import com.cronutils.utils.StringUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DataType; @@ -75,7 +75,7 @@ public class ProcedureTask extends AbstractTask { logger.info("procedure task params {}", taskProps.getTaskParams()); - this.procedureParameters = JSONObject.parseObject(taskProps.getTaskParams(), ProcedureParameters.class); + this.procedureParameters = JSON.parseObject(taskProps.getTaskParams(), ProcedureParameters.class); // check parameters if (!procedureParameters.checkParameters()) { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index aae11f5530..82034740fc 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.server.worker.task.sql; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.serializer.SerializerFeature; @@ -92,7 +93,7 @@ public class SqlTask extends AbstractTask { super(taskProps, logger); logger.info("sql task params {}", taskProps.getTaskParams()); - this.sqlParameters = JSONObject.parseObject(taskProps.getTaskParams(), SqlParameters.class); + this.sqlParameters = JSON.parseObject(taskProps.getTaskParams(), SqlParameters.class); if (!sqlParameters.checkParameters()) { throw new RuntimeException("sql task params is not valid"); @@ -308,16 +309,16 @@ public class SqlTask extends AbstractTask { } resultJSONArray.add(mapOfColValues); } - logger.debug("execute sql : {}", JSONObject.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); + logger.debug("execute sql : {}", JSON.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); // if there is a result set if ( !resultJSONArray.isEmpty() ) { if (StringUtils.isNotEmpty(sqlParameters.getTitle())) { sendAttachment(sqlParameters.getTitle(), - JSONObject.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); + JSON.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); }else{ sendAttachment(taskProps.getNodeName() + " query resultsets ", - JSONObject.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); + JSON.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java index d2a0fb2407..af312d9601 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.server.master; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.enums.*; import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.utils.DateUtils; @@ -85,7 +85,7 @@ public class MasterExecThreadTest { Map cmdParam = new HashMap<>(); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, "2020-01-01 00:00:00"); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, "2020-01-31 23:00:00"); - Mockito.when(processInstance.getCommandParam()).thenReturn(JSONObject.toJSONString(cmdParam)); + Mockito.when(processInstance.getCommandParam()).thenReturn(JSON.toJSONString(cmdParam)); ProcessDefinition processDefinition = new ProcessDefinition(); processDefinition.setGlobalParamMap(Collections.EMPTY_MAP); processDefinition.setGlobalParamList(Collections.EMPTY_LIST); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index 5d4263644b..250c8a2680 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.server.worker.shell; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.TaskNode; @@ -68,7 +68,7 @@ public class ShellCommandExecutorTest { TaskInstance taskInstance = processService.findTaskInstanceById(7657); String taskJson = taskInstance.getTaskJson(); - TaskNode taskNode = JSONObject.parseObject(taskJson, TaskNode.class); + TaskNode taskNode = JSON.parseObject(taskJson, TaskNode.class); taskProps.setTaskParams(taskNode.getParams()); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java index c395eabe51..07b700239b 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.server.worker.sql; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; @@ -112,7 +112,7 @@ public class SqlExecutorTest { TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId); String taskJson = taskInstance.getTaskJson(); - TaskNode taskNode = JSONObject.parseObject(taskJson, TaskNode.class); + TaskNode taskNode = JSON.parseObject(taskJson, TaskNode.class); taskProps.setTaskParams(taskNode.getParams()); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java index 511102e4b5..5c7afa8155 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.server.worker.task.sqoop; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; import org.apache.dolphinscheduler.dao.entity.DataSource; @@ -74,7 +74,7 @@ public class SqoopTaskTest { @Test public void testGenerator(){ String data1 = "{\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HDFS\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"0\\\",\\\"srcQuerySql\\\":\\\"\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[]}\",\"targetParams\":\"{\\\"targetPath\\\":\\\"/ods/tmp/test/person7\\\",\\\"deleteTargetDir\\\":true,\\\"fileType\\\":\\\"--as-textfile\\\",\\\"compressionCodec\\\":\\\"\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; - SqoopParameters sqoopParameters1 = JSONObject.parseObject(data1,SqoopParameters.class); + SqoopParameters sqoopParameters1 = JSON.parseObject(data1,SqoopParameters.class); SqoopJobGenerator generator = new SqoopJobGenerator(); String script = generator.generateSqoopJob(sqoopParameters1); @@ -82,21 +82,21 @@ public class SqoopTaskTest { Assert.assertEquals(expected, script); String data2 = "{\"concurrency\":1,\"modelType\":\"export\",\"sourceType\":\"HDFS\",\"targetType\":\"MYSQL\",\"sourceParams\":\"{\\\"exportDir\\\":\\\"/ods/tmp/test/person7\\\"}\",\"targetParams\":\"{\\\"targetDatasource\\\":2,\\\"targetTable\\\":\\\"person_3\\\",\\\"targetColumns\\\":\\\"id,name,age,sex,create_time\\\",\\\"preQuery\\\":\\\"\\\",\\\"isUpdate\\\":true,\\\"targetUpdateKey\\\":\\\"id\\\",\\\"targetUpdateMode\\\":\\\"allowinsert\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; - SqoopParameters sqoopParameters2 = JSONObject.parseObject(data2,SqoopParameters.class); + SqoopParameters sqoopParameters2 = JSON.parseObject(data2,SqoopParameters.class); String script2 = generator.generateSqoopJob(sqoopParameters2); String expected2 = "sqoop export -m 1 --export-dir /ods/tmp/test/person7 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_3 --columns id,name,age,sex,create_time --fields-terminated-by '@' --lines-terminated-by '\\n' --update-key id --update-mode allowinsert"; Assert.assertEquals(expected2, script2); String data3 = "{\"concurrency\":1,\"modelType\":\"export\",\"sourceType\":\"HIVE\",\"targetType\":\"MYSQL\",\"sourceParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-17\\\"}\",\"targetParams\":\"{\\\"targetDatasource\\\":2,\\\"targetTable\\\":\\\"person_3\\\",\\\"targetColumns\\\":\\\"\\\",\\\"preQuery\\\":\\\"\\\",\\\"isUpdate\\\":false,\\\"targetUpdateKey\\\":\\\"\\\",\\\"targetUpdateMode\\\":\\\"allowinsert\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; - SqoopParameters sqoopParameters3 = JSONObject.parseObject(data3,SqoopParameters.class); + SqoopParameters sqoopParameters3 = JSON.parseObject(data3,SqoopParameters.class); String script3 = generator.generateSqoopJob(sqoopParameters3); String expected3 = "sqoop export -m 1 --hcatalog-database stg --hcatalog-table person_internal --hcatalog-partition-keys date --hcatalog-partition-values 2020-02-17 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_3 --fields-terminated-by '@' --lines-terminated-by '\\n'"; Assert.assertEquals(expected3, script3); String data4 = "{\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HIVE\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"1\\\",\\\"srcQuerySql\\\":\\\"SELECT * FROM person_2\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[{\\\"prop\\\":\\\"id\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"Integer\\\"}]}\",\"targetParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal_2\\\",\\\"createHiveTable\\\":true,\\\"dropDelimiter\\\":false,\\\"hiveOverWrite\\\":true,\\\"replaceDelimiter\\\":\\\"\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-16\\\"}\",\"localParams\":[]}"; - SqoopParameters sqoopParameters4 = JSONObject.parseObject(data4,SqoopParameters.class); + SqoopParameters sqoopParameters4 = JSON.parseObject(data4,SqoopParameters.class); String script4 = generator.generateSqoopJob(sqoopParameters4); String expected4 = "sqoop import -m 1 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --query 'SELECT * FROM person_2 WHERE $CONDITIONS' --map-column-java id=Integer --hive-import --hive-table stg.person_internal_2 --create-hive-table --hive-overwrite -delete-target-dir --hive-partition-key date --hive-partition-value 2020-02-16"; diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index 8bcd64f1fd..c848ec5197 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.service.process; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.cronutils.model.Cron; import org.apache.commons.lang.ArrayUtils; @@ -207,7 +208,7 @@ public class ProcessService { CommandType commandType = command.getCommandType(); if(cmdTypeMap.containsKey(commandType)){ - JSONObject cmdParamObj = (JSONObject) JSONObject.parse(command.getCommandParam()); + JSONObject cmdParamObj = (JSONObject) JSON.parse(command.getCommandParam()); JSONObject tempObj; int processInstanceId = cmdParamObj.getInteger(CMDPARAM_RECOVER_PROCESS_ID_STRING); @@ -215,7 +216,7 @@ public class ProcessService { // for all commands for (Command tmpCommand:commands){ if(cmdTypeMap.containsKey(tmpCommand.getCommandType())){ - tempObj = (JSONObject) JSONObject.parse(tmpCommand.getCommandParam()); + tempObj = (JSONObject) JSON.parse(tmpCommand.getCommandParam()); if(tempObj != null && processInstanceId == tempObj.getInteger(CMDPARAM_RECOVER_PROCESS_ID_STRING)){ isNeedCreate = false; break; @@ -309,7 +310,7 @@ public class ProcessService { for (TaskNode taskNode : taskNodeList){ String parameter = taskNode.getParams(); if (parameter.contains(CMDPARAM_SUB_PROCESS_DEFINE_ID)){ - SubProcessParameters subProcessParam = JSONObject.parseObject(parameter, SubProcessParameters.class); + SubProcessParameters subProcessParam = JSON.parseObject(parameter, SubProcessParameters.class); ids.add(subProcessParam.getProcessDefinitionId()); recurseFindSubProcessId(subProcessParam.getProcessDefinitionId(),ids); } From 205af16d55f6e16a7db9f20e8076b4658934eed3 Mon Sep 17 00:00:00 2001 From: DS Date: Sat, 14 Mar 2020 08:30:41 +0800 Subject: [PATCH 42/59] Create .asf.yaml (#2166) --- .asf.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .asf.yaml diff --git a/.asf.yaml b/.asf.yaml new file mode 100644 index 0000000000..650e3c7e75 --- /dev/null +++ b/.asf.yaml @@ -0,0 +1,4 @@ +# Web site staging services: + +github: + description: "Dolphin Scheduler is a distributed and easy-to-extend visual workflow scheduling platform, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.(分布式易扩展的可视化工作流任务调度)" From e98c6f3008930e7f96943d55636a50e3ff2c05d3 Mon Sep 17 00:00:00 2001 From: DS Date: Sat, 14 Mar 2020 08:42:55 +0800 Subject: [PATCH 43/59] Update .asf.yaml (#2167) --- .asf.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.asf.yaml b/.asf.yaml index 650e3c7e75..7e950a0a53 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -1,4 +1,7 @@ -# Web site staging services: +staging: + profile: ~ + whoami: master + foo: trigger github: description: "Dolphin Scheduler is a distributed and easy-to-extend visual workflow scheduling platform, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.(分布式易扩展的可视化工作流任务调度)" From 2e63dc9c96b1fe68407ba7adf2cb237f785f7831 Mon Sep 17 00:00:00 2001 From: DS Date: Sat, 14 Mar 2020 08:47:23 +0800 Subject: [PATCH 44/59] Delete .asf.yaml (#2168) --- .asf.yaml | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 .asf.yaml diff --git a/.asf.yaml b/.asf.yaml deleted file mode 100644 index 7e950a0a53..0000000000 --- a/.asf.yaml +++ /dev/null @@ -1,7 +0,0 @@ -staging: - profile: ~ - whoami: master - foo: trigger - -github: - description: "Dolphin Scheduler is a distributed and easy-to-extend visual workflow scheduling platform, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.(分布式易扩展的可视化工作流任务调度)" From 80c6ce5711ab77e104df83ea70853d86916135b7 Mon Sep 17 00:00:00 2001 From: Rubik-W <39549317+Rubik-W@users.noreply.github.com> Date: Sat, 14 Mar 2020 18:10:27 +0800 Subject: [PATCH 45/59] The master and worker server exit exception #2163 (#2176) * fix: #2163 * fix: format --- .../common/thread/ThreadUtils.java | 17 +++++-- .../server/master/MasterServer.java | 33 +++++++------- .../server/worker/WorkerServer.java | 45 +++++++------------ 3 files changed, 46 insertions(+), 49 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java index 0a4ed9b5ac..d8ef0bb38d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java @@ -120,12 +120,24 @@ public class ThreadUtils { /** * Wrapper over ScheduledThreadPoolExecutor + * @param threadName * @param corePoolSize * @return */ - public static ScheduledExecutorService newDaemonThreadScheduledExecutor(String threadName,int corePoolSize) { + public static ScheduledExecutorService newDaemonThreadScheduledExecutor(String threadName, int corePoolSize) { + return newThreadScheduledExecutor(threadName, corePoolSize, true); + } + + /** + * Wrapper over ScheduledThreadPoolExecutor + * @param threadName + * @param corePoolSize + * @param isDaemon + * @return + */ + public static ScheduledExecutorService newThreadScheduledExecutor(String threadName, int corePoolSize, boolean isDaemon) { ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setDaemon(true) + .setDaemon(isDaemon) .setNameFormat(threadName) .build(); ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(corePoolSize, threadFactory); @@ -135,7 +147,6 @@ public class ThreadUtils { return executor; } - public static ThreadInfo getThreadInfo(Thread t) { long tid = t.getId(); return threadBean.getThreadInfo(tid, STACK_DEPTH); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index 6b5063cba4..9512b1a1c2 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.runner.MasterSchedulerThread; +import org.apache.dolphinscheduler.server.worker.WorkerServer; import org.apache.dolphinscheduler.server.zk.ZKMasterClient; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; @@ -37,8 +38,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.WebApplicationType; import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.FilterType; import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -46,7 +49,9 @@ import java.util.concurrent.TimeUnit; /** * master server */ -@ComponentScan("org.apache.dolphinscheduler") +@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = { + @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, classes = {WorkerServer.class}) +}) public class MasterServer implements IStoppable { /** @@ -112,7 +117,7 @@ public class MasterServer implements IStoppable { masterSchedulerService = ThreadUtils.newDaemonSingleThreadExecutor("Master-Scheduler-Thread"); - heartbeatMasterService = ThreadUtils.newDaemonThreadScheduledExecutor("Master-Main-Thread",Constants.DEFAULT_MASTER_HEARTBEAT_THREAD_NUM); + heartbeatMasterService = ThreadUtils.newThreadScheduledExecutor("Master-Main-Thread",Constants.DEFAULT_MASTER_HEARTBEAT_THREAD_NUM, false); // heartbeat thread implement Runnable heartBeatThread = heartBeatThread(); @@ -147,23 +152,17 @@ public class MasterServer implements IStoppable { } logger.error("start Quartz failed", e); } - - - /** - * register hooks, which are called before the process exits - */ - Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { - @Override - public void run() { - if (zkMasterClient.getActiveMasterNum() <= 1) { - zkMasterClient.getAlertDao().sendServerStopedAlert( - 1, OSUtils.getHost(), "Master-Server"); - } - stop("shutdownhook"); - } - })); } + @PreDestroy + public void destroy() { + // master server exit alert + if (zkMasterClient.getActiveMasterNum() <= 1) { + zkMasterClient.getAlertDao().sendServerStopedAlert( + 1, OSUtils.getHost(), "Master-Server"); + } + stop("shutdownhook"); + } /** * gracefully stop diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index ace93079ff..86bb7d3e07 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -29,6 +29,7 @@ import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.server.master.MasterServer; import org.apache.dolphinscheduler.server.utils.ProcessUtils; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.runner.FetchTaskThread; @@ -43,10 +44,13 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.FilterType; import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -56,7 +60,10 @@ import java.util.concurrent.TimeUnit; /** * worker server */ -@ComponentScan("org.apache.dolphinscheduler") +@SpringBootApplication +@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = { + @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, classes = {MasterServer.class}) +}) public class WorkerServer implements IStoppable { /** @@ -104,11 +111,6 @@ public class WorkerServer implements IStoppable { */ private ExecutorService fetchTaskExecutorService; - /** - * CountDownLatch latch - */ - private CountDownLatch latch; - @Value("${server.is-combined-server:false}") private Boolean isCombinedServer; @@ -149,7 +151,7 @@ public class WorkerServer implements IStoppable { this.fetchTaskExecutorService = ThreadUtils.newDaemonSingleThreadExecutor("Worker-Fetch-Thread-Executor"); - heartbeatWorkerService = ThreadUtils.newDaemonThreadScheduledExecutor("Worker-Heartbeat-Thread-Executor", Constants.DEFAUL_WORKER_HEARTBEAT_THREAD_NUM); + heartbeatWorkerService = ThreadUtils.newThreadScheduledExecutor("Worker-Heartbeat-Thread-Executor", Constants.DEFAUL_WORKER_HEARTBEAT_THREAD_NUM, false); // heartbeat thread implement Runnable heartBeatThread = heartBeatThread(); @@ -171,29 +173,15 @@ public class WorkerServer implements IStoppable { // submit fetch task thread fetchTaskExecutorService.execute(fetchTaskThread); + } - /** - * register hooks, which are called before the process exits - */ - Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { - @Override - public void run() { - // worker server exit alert - if (zkWorkerClient.getActiveMasterNum() <= 1) { - alertDao.sendServerStopedAlert(1, OSUtils.getHost(), "Worker-Server"); - } - stop("shutdownhook"); - } - })); - - //let the main thread await - latch = new CountDownLatch(1); - if (!isCombinedServer) { - try { - latch.await(); - } catch (InterruptedException ignore) { - } + @PreDestroy + public void destroy() { + // worker server exit alert + if (zkWorkerClient.getActiveMasterNum() <= 1) { + alertDao.sendServerStopedAlert(1, OSUtils.getHost(), "Worker-Server"); } + stop("shutdownhook"); } @Override @@ -251,7 +239,6 @@ public class WorkerServer implements IStoppable { }catch (Exception e){ logger.warn("zookeeper service stopped exception:{}",e.getMessage()); } - latch.countDown(); logger.info("zookeeper service stopped"); } catch (Exception e) { From a076ae4c3658b23afee1e5b516b073cac0866756 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Sat, 14 Mar 2020 18:12:14 +0800 Subject: [PATCH 46/59] Adapting partial code(file name start with J) to the sonar cloud rule (#2165) * Adapting partial code(file name start with J) to the sonar cloud rule * fix unit failure --- .../alert/utils/JSONUtils.java | 7 ++--- .../alert/utils/JSONUtilsTest.java | 27 ++++++++++--------- .../common/utils/JSONUtils.java | 10 ++----- .../common/utils/JSONUtilsTest.java | 18 ++++++------- 4 files changed, 29 insertions(+), 33 deletions(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java index 8d66255fc4..5f8371de2d 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collections; import java.util.List; /** @@ -50,11 +51,11 @@ public class JSONUtils { * @param json the json * @param clazz c * @param the generic clazz - * @return the result list + * @return the result list or empty list */ public static List toList(String json, Class clazz) { if (StringUtils.isEmpty(json)) { - return null; + return Collections.emptyList(); } try { return JSON.parseArray(json, clazz); @@ -62,7 +63,7 @@ public class JSONUtils { logger.error("JSONArray.parseArray exception!",e); } - return null; + return Collections.emptyList(); } } diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java index 277c42b5bd..a151abc714 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java @@ -26,8 +26,7 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.Assert.*; public class JSONUtilsTest { @@ -73,7 +72,7 @@ public class JSONUtilsTest { result = JSONUtils.toJsonString(null); logger.info(result); - assertEquals(result,"null"); + assertEquals("null", result); } @@ -86,25 +85,27 @@ public class JSONUtilsTest { //Invoke toList List result = JSONUtils.toList(expected ,LinkedHashMap.class); //Equal list size=1 - assertEquals(result.size(),1); + assertEquals(1,result.size()); //Transform entity to LinkedHashMap LinkedHashMap entity = result.get(0); //Equal expected values - assertEquals(entity.get("mysql service name"),"mysql200"); - assertEquals(entity.get("mysql address"),"192.168.xx.xx"); - assertEquals(entity.get("port"),"3306"); - assertEquals(entity.get("no index of number"),"80"); - assertEquals(entity.get("database client connections"),"190"); + assertEquals("mysql200",entity.get("mysql service name")); + assertEquals("192.168.xx.xx", entity.get("mysql address")); + assertEquals("3306", entity.get("port")); + assertEquals("80", entity.get("no index of number")); + assertEquals("190", entity.get("database client connections")); - //If param is null, then return null + //If param is null, then return empty list result = JSONUtils.toList(null ,LinkedHashMap.class); - assertNull(result); + assertNotNull(result); + assertTrue(result.isEmpty()); - //If param is incorrect, then return null and log error message + //If param is incorrect, then return empty list and log error message result = JSONUtils.toList("}{" ,LinkedHashMap.class); - assertNull(result); + assertNotNull(result); + assertTrue(result.isEmpty()); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java index b6ed71f402..f0aed91a0d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java @@ -42,12 +42,6 @@ public class JSONUtils { */ private static final ObjectMapper objectMapper = new ObjectMapper(); - /** - * init - */ - private static final JSONUtils instance = new JSONUtils(); - - private JSONUtils() { //Feature that determines whether encountering of unknown properties, false means not analyzer unknown properties objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false).setTimeZone(TimeZone.getDefault()); @@ -221,7 +215,7 @@ public class JSONUtils { try{ return JSON.toJSONString(object,false); } catch (Exception e) { - throw new RuntimeException("Json deserialization exception.", e); + throw new RuntimeException("Object json deserialization exception.", e); } } @@ -229,7 +223,7 @@ public class JSONUtils { try{ return JSON.parseObject(text); } catch (Exception e) { - throw new RuntimeException("Json deserialization exception.", e); + throw new RuntimeException("String json deserialization exception.", e); } } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java index d249314819..8ce60349ed 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java @@ -40,8 +40,8 @@ public class JSONUtilsTest { String jsonStr = "{\"id\":\"1001\",\"name\":\"Jobs\"}"; Map models = JSONUtils.toMap(jsonStr); - Assert.assertEquals(models.get("id"), "1001"); - Assert.assertEquals(models.get("name"), "Jobs"); + Assert.assertEquals("1001", models.get("id")); + Assert.assertEquals("Jobs", models.get("name")); } @@ -55,7 +55,7 @@ public class JSONUtilsTest { String str = "{\"direct\":\"IN\",\"prop\":\"ds\",\"type\":\"VARCHAR\",\"value\":\"sssssss\"}"; Property property1 = JSON.parseObject(str, Property.class); Direct direct = property1.getDirect(); - Assert.assertEquals(direct , Direct.IN); + Assert.assertEquals(Direct.IN, direct); } @@ -66,12 +66,12 @@ public class JSONUtilsTest { List maps = JSONUtils.toList(str, LinkedHashMap.class); - Assert.assertEquals(maps.size(), 1); - Assert.assertEquals(maps.get(0).get("mysql service name"), "mysql200"); - Assert.assertEquals(maps.get(0).get("mysql address"), "192.168.xx.xx"); - Assert.assertEquals(maps.get(0).get("port"), "3306"); - Assert.assertEquals(maps.get(0).get("no index of number"), "80"); - Assert.assertEquals(maps.get(0).get("database client connections"), "190"); + Assert.assertEquals(1, maps.size()); + Assert.assertEquals("mysql200", maps.get(0).get("mysql service name")); + Assert.assertEquals("192.168.xx.xx", maps.get(0).get("mysql address")); + Assert.assertEquals("3306", maps.get(0).get("port")); + Assert.assertEquals("80", maps.get(0).get("no index of number")); + Assert.assertEquals("190", maps.get(0).get("database client connections")); } public String list2String(){ From e0389aebf6416c90f31ec0f6a4ff5a5d94cfecfb Mon Sep 17 00:00:00 2001 From: Rubik-W <39549317+Rubik-W@users.noreply.github.com> Date: Sat, 14 Mar 2020 19:20:46 +0800 Subject: [PATCH 47/59] fix: NullPointException in the case of task queuing (#2175) --- .../dolphinscheduler/server/master/runner/MasterExecThread.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 849bbd5680..0c44b7a77e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -1062,7 +1062,7 @@ public class MasterExecThread implements Runnable { TaskInstance taskInstance = taskExecThread.getTaskInstance(); taskInstance = processService.findTaskInstanceById(taskInstance.getId()); - if(taskInstance.getState().typeIsFinished()){ + if(taskInstance != null && taskInstance.getState().typeIsFinished()){ continue; } From 61ebbf29463809449734ff00b00e6d9627170d10 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Sat, 14 Mar 2020 19:22:24 +0800 Subject: [PATCH 48/59] Adapting partial code(file name start with L) to the sonar cloud rule (#2171) --- .../interceptor/LoginHandlerInterceptor.java | 11 ---------- .../common/utils/LoggerUtils.java | 2 +- .../server/log/LoggerRequestProcessor.java | 21 +++++-------------- .../service/log/LogClientService.java | 8 +++---- 4 files changed, 10 insertions(+), 32 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java index 380eea5774..98bac42f72 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java @@ -27,7 +27,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.servlet.HandlerInterceptor; -import org.springframework.web.servlet.ModelAndView; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -90,14 +89,4 @@ public class LoginHandlerInterceptor implements HandlerInterceptor { return true; } - @Override - public void postHandle(HttpServletRequest request, HttpServletResponse response, Object handler, ModelAndView modelAndView) throws Exception { - - } - - @Override - public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) throws Exception { - - } - } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java index fc08eb645b..191df335c5 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java @@ -79,7 +79,7 @@ public class LoggerUtils { */ public static List getAppIds(String log, Logger logger) { - List appIds = new ArrayList(); + List appIds = new ArrayList<>(); Matcher matcher = APPLICATION_REGEX.matcher(log); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java index 4e4404ea1c..4b21d5f4a2 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.log; import io.netty.channel.Channel; +import org.apache.dolphinscheduler.common.utils.IOUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.log.*; @@ -116,16 +117,8 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { }catch (IOException e){ logger.error("get file bytes error",e); }finally { - if (bos != null){ - try { - bos.close(); - } catch (IOException ignore) {} - } - if (in != null){ - try { - in.close(); - } catch (IOException ignore) {} - } + IOUtils.closeQuietly(bos); + IOUtils.closeQuietly(in); } return new byte[0]; } @@ -146,7 +139,7 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { } catch (IOException e) { logger.error("read file error",e); } - return Collections.EMPTY_LIST; + return Collections.emptyList(); } /** @@ -168,11 +161,7 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { }catch (IOException e){ logger.error("read file error",e); }finally { - try { - if (br != null){ - br.close(); - } - } catch (IOException ignore) {} + IOUtils.closeQuietly(br); } return ""; } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java index 01a49910df..163cd3696f 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java @@ -40,7 +40,7 @@ public class LogClientService { /** * request time out */ - private final long logRequestTimeout = 10 * 1000; + private static final long LOG_REQUEST_TIMEOUT = 10 * 1000L; /** * construct client @@ -75,7 +75,7 @@ public class LogClientService { final Address address = new Address(host, port); try { Command command = request.convert2Command(); - Command response = this.client.sendSync(address, command, logRequestTimeout); + Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); if(response != null){ RollViewLogResponseCommand rollReviewLog = FastJsonSerializer.deserialize( response.getBody(), RollViewLogResponseCommand.class); @@ -103,7 +103,7 @@ public class LogClientService { final Address address = new Address(host, port); try { Command command = request.convert2Command(); - Command response = this.client.sendSync(address, command, logRequestTimeout); + Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); if(response != null){ ViewLogResponseCommand viewLog = FastJsonSerializer.deserialize( response.getBody(), ViewLogResponseCommand.class); @@ -131,7 +131,7 @@ public class LogClientService { final Address address = new Address(host, port); try { Command command = request.convert2Command(); - Command response = this.client.sendSync(address, command, logRequestTimeout); + Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); if(response != null){ GetLogBytesResponseCommand getLog = FastJsonSerializer.deserialize( response.getBody(), GetLogBytesResponseCommand.class); From fcc57320d6b281e81fd9068f4dbb3d86ec6978f1 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Sun, 15 Mar 2020 19:05:33 +0800 Subject: [PATCH 49/59] Adapting partial code(file name start with M #1) to the sonar cloud rule (#2173) * Adapting partial code(file name start with M #1) to the sonar cloud rule * remove unused import --- .../alert/utils/MailUtils.java | 52 ++++++++-------- .../common/task/mr/MapreduceParameters.java | 14 +++-- .../dao/entity/ProcessInstance.java | 2 +- .../dao/entity/TaskInstance.java | 4 +- .../master/runner/MasterExecThread.java | 61 +++++++++---------- .../server/worker/task/mr/MapReduceTask.java | 17 +++--- 6 files changed, 72 insertions(+), 78 deletions(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java index 130ad8ade6..66a63dcff4 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java @@ -39,29 +39,29 @@ public class MailUtils { public static final Logger logger = LoggerFactory.getLogger(MailUtils.class); - public static final String mailProtocol = PropertyUtils.getString(Constants.MAIL_PROTOCOL); + public static final String MAIL_PROTOCOL = PropertyUtils.getString(Constants.MAIL_PROTOCOL); - public static final String mailServerHost = PropertyUtils.getString(Constants.MAIL_SERVER_HOST); + public static final String MAIL_SERVER_HOST = PropertyUtils.getString(Constants.MAIL_SERVER_HOST); - public static final Integer mailServerPort = PropertyUtils.getInt(Constants.MAIL_SERVER_PORT); + public static final Integer MAIL_SERVER_PORT = PropertyUtils.getInt(Constants.MAIL_SERVER_PORT); - public static final String mailSender = PropertyUtils.getString(Constants.MAIL_SENDER); + public static final String MAIL_SENDER = PropertyUtils.getString(Constants.MAIL_SENDER); - public static final String mailUser = PropertyUtils.getString(Constants.MAIL_USER); + public static final String MAIL_USER = PropertyUtils.getString(Constants.MAIL_USER); - public static final String mailPasswd = PropertyUtils.getString(Constants.MAIL_PASSWD); + public static final String MAIL_PASSWD = PropertyUtils.getString(Constants.MAIL_PASSWD); - public static final Boolean mailUseStartTLS = PropertyUtils.getBoolean(Constants.MAIL_SMTP_STARTTLS_ENABLE); + public static final Boolean MAIL_USE_START_TLS = PropertyUtils.getBoolean(Constants.MAIL_SMTP_STARTTLS_ENABLE); - public static final Boolean mailUseSSL = PropertyUtils.getBoolean(Constants.MAIL_SMTP_SSL_ENABLE); + public static final Boolean MAIL_USE_SSL = PropertyUtils.getBoolean(Constants.MAIL_SMTP_SSL_ENABLE); - public static final String xlsFilePath = PropertyUtils.getString(Constants.XLS_FILE_PATH); + public static final String XLS_FILE_PATH = PropertyUtils.getString(Constants.XLS_FILE_PATH); - public static final String starttlsEnable = PropertyUtils.getString(Constants.MAIL_SMTP_STARTTLS_ENABLE); + public static final String STARTTLS_ENABLE = PropertyUtils.getString(Constants.MAIL_SMTP_STARTTLS_ENABLE); - public static final String sslEnable = PropertyUtils.getString(Constants.MAIL_SMTP_SSL_ENABLE); + public static final String SSL_ENABLE = PropertyUtils.getString(Constants.MAIL_SMTP_SSL_ENABLE); - public static final String sslTrust = PropertyUtils.getString(Constants.MAIL_SMTP_SSL_TRUST); + public static final String SSL_TRUST = PropertyUtils.getString(Constants.MAIL_SMTP_SSL_TRUST); public static final AlertTemplate alertTemplate = AlertTemplateFactory.getMessageTemplate(); @@ -105,7 +105,7 @@ public class MailUtils { try { Session session = getSession(); email.setMailSession(session); - email.setFrom(mailSender); + email.setFrom(MAIL_SENDER); email.setCharset(Constants.UTF_8); if (CollectionUtils.isNotEmpty(receivers)){ // receivers mail @@ -199,10 +199,10 @@ public class MailUtils { // 2. creating mail: Creating a MimeMessage MimeMessage msg = new MimeMessage(session); // 3. set sender - msg.setFrom(new InternetAddress(mailSender)); + msg.setFrom(new InternetAddress(MAIL_SENDER)); // 4. set receivers for (String receiver : receivers) { - msg.addRecipients(MimeMessage.RecipientType.TO, InternetAddress.parse(receiver)); + msg.addRecipients(Message.RecipientType.TO, InternetAddress.parse(receiver)); } return msg; } @@ -213,19 +213,19 @@ public class MailUtils { */ private static Session getSession() { Properties props = new Properties(); - props.setProperty(Constants.MAIL_HOST, mailServerHost); - props.setProperty(Constants.MAIL_PORT, String.valueOf(mailServerPort)); + props.setProperty(Constants.MAIL_HOST, MAIL_SERVER_HOST); + props.setProperty(Constants.MAIL_PORT, String.valueOf(MAIL_SERVER_PORT)); props.setProperty(Constants.MAIL_SMTP_AUTH, Constants.STRING_TRUE); - props.setProperty(Constants.MAIL_TRANSPORT_PROTOCOL, mailProtocol); - props.setProperty(Constants.MAIL_SMTP_STARTTLS_ENABLE, starttlsEnable); - props.setProperty(Constants.MAIL_SMTP_SSL_ENABLE, sslEnable); - props.setProperty(Constants.MAIL_SMTP_SSL_TRUST, sslTrust); + props.setProperty(Constants.MAIL_TRANSPORT_PROTOCOL, MAIL_PROTOCOL); + props.setProperty(Constants.MAIL_SMTP_STARTTLS_ENABLE, STARTTLS_ENABLE); + props.setProperty(Constants.MAIL_SMTP_SSL_ENABLE, SSL_ENABLE); + props.setProperty(Constants.MAIL_SMTP_SSL_TRUST, SSL_TRUST); Authenticator auth = new Authenticator() { @Override protected PasswordAuthentication getPasswordAuthentication() { // mail username and password - return new PasswordAuthentication(mailUser, mailPasswd); + return new PasswordAuthentication(MAIL_USER, MAIL_PASSWD); } }; @@ -248,12 +248,10 @@ public class MailUtils { */ if(CollectionUtils.isNotEmpty(receiversCc)){ for (String receiverCc : receiversCc){ - msg.addRecipients(MimeMessage.RecipientType.CC, InternetAddress.parse(receiverCc)); + msg.addRecipients(Message.RecipientType.CC, InternetAddress.parse(receiverCc)); } } - // set receivers type to cc - // msg.addRecipients(MimeMessage.RecipientType.CC, InternetAddress.parse(propMap.get("${CC}"))); // set subject msg.setSubject(title); MimeMultipart partList = new MimeMultipart(); @@ -263,8 +261,8 @@ public class MailUtils { // set attach file MimeBodyPart part2 = new MimeBodyPart(); // make excel file - ExcelUtils.genExcelFile(content,title,xlsFilePath); - File file = new File(xlsFilePath + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS); + ExcelUtils.genExcelFile(content,title, XLS_FILE_PATH); + File file = new File(XLS_FILE_PATH + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS); part2.attachFile(file); part2.setFileName(MimeUtility.encodeText(title + Constants.EXCEL_SUFFIX_XLS,Constants.UTF_8,"B")); // add components to collection diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/mr/MapreduceParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/mr/MapreduceParameters.java index b8fd6ebcbf..31c9c7292f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/mr/MapreduceParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/mr/MapreduceParameters.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -125,12 +126,15 @@ public class MapreduceParameters extends AbstractParameters { @Override public List getResourceFilesList() { - if (resourceList != null) { - this.resourceList.add(mainJar); - return resourceList.stream() - .map(p -> p.getRes()).collect(Collectors.toList()); + if(resourceList != null ) { + List resourceFiles = resourceList.stream() + .map(ResourceInfo::getRes).collect(Collectors.toList()); + if(mainJar != null) { + resourceFiles.add(mainJar.getRes()); + } + return resourceFiles; } - return null; + return Collections.emptyList(); } @Override diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java index fb5d0cda42..fd9653a36c 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java @@ -506,7 +506,7 @@ public class ProcessInstance { * check this process is start complement data * @return whether complement data */ - public Boolean isComplementData(){ + public boolean isComplementData(){ if(!StringUtils.isNotEmpty(this.historyCmd)){ return false; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java index c692575e3a..702897fba9 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java @@ -373,7 +373,7 @@ public class TaskInstance { } - public Boolean isSubProcess(){ + public boolean isSubProcess(){ return TaskType.SUB_PROCESS.getDescp().equals(this.taskType); } @@ -442,7 +442,7 @@ public class TaskInstance { this.executorName = executorName; } - public Boolean isTaskComplete() { + public boolean isTaskComplete() { return this.getState().typeIsPause() || this.getState().typeIsSuccess() diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 0c44b7a77e..26457a386a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -68,7 +68,7 @@ public class MasterExecThread implements Runnable { /** * runing TaskNode */ - private final Map> activeTaskNode = new ConcurrentHashMap>(); + private final Map> activeTaskNode = new ConcurrentHashMap<>(); /** * task exec service @@ -78,7 +78,7 @@ public class MasterExecThread implements Runnable { /** * submit failure nodes */ - private Boolean taskFailedSubmit = false; + private boolean taskFailedSubmit = false; /** * recover node id list @@ -652,7 +652,7 @@ public class MasterExecThread implements Runnable { continue; } if(task.getState().typeIsPause() || task.getState().typeIsCancel()){ - logger.info("task {} stopped, the state is {}", task.getName(), task.getState().toString()); + logger.info("task {} stopped, the state is {}", task.getName(), task.getState()); }else{ addTaskToStandByList(task); } @@ -685,11 +685,12 @@ public class MasterExecThread implements Runnable { } ExecutionStatus depTaskState = completeTaskList.get(depsNode).getState(); // conditions task would not return failed. - if(depTaskState.typeIsFailure()){ - if(!haveConditionsAfterNode(depsNode) && !dag.getNode(depsNode).isConditionsTask()){ - return DependResult.FAILED; - } + if(depTaskState.typeIsFailure() + && !haveConditionsAfterNode(depsNode) + && !dag.getNode(depsNode).isConditionsTask()){ + return DependResult.FAILED; } + if(depTaskState.typeIsPause() || depTaskState.typeIsCancel()){ return DependResult.WAITING; } @@ -737,7 +738,7 @@ public class MasterExecThread implements Runnable { * * @return Boolean whether has failed task */ - private Boolean hasFailedTask(){ + private boolean hasFailedTask(){ if(this.taskFailedSubmit){ return true; @@ -753,7 +754,7 @@ public class MasterExecThread implements Runnable { * * @return Boolean whether process instance failed */ - private Boolean processFailed(){ + private boolean processFailed(){ if(hasFailedTask()) { if(processInstance.getFailureStrategy() == FailureStrategy.END){ return true; @@ -769,9 +770,9 @@ public class MasterExecThread implements Runnable { * whether task for waiting thread * @return Boolean whether has waiting thread task */ - private Boolean hasWaitingThreadTask(){ + private boolean hasWaitingThreadTask(){ List waitingList = getCompleteTaskByState(ExecutionStatus.WAITTING_THREAD); - return waitingList.size() > 0; + return CollectionUtils.isNotEmpty(waitingList); } /** @@ -787,7 +788,7 @@ public class MasterExecThread implements Runnable { } List pauseList = getCompleteTaskByState(ExecutionStatus.PAUSE); - if(pauseList.size() > 0 + if(CollectionUtils.isNotEmpty(pauseList) || !isComplementEnd() || readyToSubmitTaskList.size() > 0){ return ExecutionStatus.PAUSE; @@ -827,7 +828,8 @@ public class MasterExecThread implements Runnable { if(state == ExecutionStatus.READY_STOP){ List stopList = getCompleteTaskByState(ExecutionStatus.STOP); List killList = getCompleteTaskByState(ExecutionStatus.KILL); - if(stopList.size() > 0 || killList.size() > 0 || !isComplementEnd()){ + if(CollectionUtils.isNotEmpty(stopList) + || CollectionUtils.isNotEmpty(killList) || !isComplementEnd()){ return ExecutionStatus.STOP; }else{ return ExecutionStatus.SUCCESS; @@ -852,7 +854,7 @@ public class MasterExecThread implements Runnable { * whether complement end * @return Boolean whether is complement end */ - private Boolean isComplementEnd() { + private boolean isComplementEnd() { if(!processInstance.isComplementData()){ return true; } @@ -877,8 +879,8 @@ public class MasterExecThread implements Runnable { logger.info( "work flow process instance [id: {}, name:{}], state change from {} to {}, cmd type: {}", processInstance.getId(), processInstance.getName(), - processInstance.getState().toString(), state.toString(), - processInstance.getCommandType().toString()); + processInstance.getState(), state, + processInstance.getCommandType()); processInstance.setState(state); ProcessInstance instance = processService.findProcessInstanceById(processInstance.getId()); instance.setState(state); @@ -894,8 +896,7 @@ public class MasterExecThread implements Runnable { * @return DependResult */ private DependResult getDependResultForTask(TaskInstance taskInstance){ - DependResult inner = isTaskDepsComplete(taskInstance.getName()); - return inner; + return isTaskDepsComplete(taskInstance.getName()); } /** @@ -920,7 +921,7 @@ public class MasterExecThread implements Runnable { * has retry task in standby * @return Boolean whether has retry task in standby */ - private Boolean hasRetryTaskInStandBy(){ + private boolean hasRetryTaskInStandBy(){ for (Map.Entry entry: readyToSubmitTaskList.entrySet()) { if(entry.getValue().getState().typeIsFailure()){ return true; @@ -958,7 +959,7 @@ public class MasterExecThread implements Runnable { continue; } logger.info("task :{}, id:{} complete, state is {} ", - task.getName(), task.getId(), task.getState().toString()); + task.getName(), task.getId(), task.getState()); // node success , post node submit if(task.getState() == ExecutionStatus.SUCCESS){ completeTaskList.put(task.getName(), task); @@ -990,7 +991,7 @@ public class MasterExecThread implements Runnable { completeTaskList.put(task.getName(), task); } // send alert - if(this.recoverToleranceFaultTaskList.size() > 0){ + if(CollectionUtils.isNotEmpty(this.recoverToleranceFaultTaskList)){ alertManager.sendAlertWorkerToleranceFault(processInstance, recoverToleranceFaultTaskList); this.recoverToleranceFaultTaskList.clear(); } @@ -1034,10 +1035,7 @@ public class MasterExecThread implements Runnable { Date now = new Date(); long runningTime = DateUtils.diffMin(now, processInstance.getStartTime()); - if(runningTime > processInstance.getTimeout()){ - return true; - } - return false; + return runningTime > processInstance.getTimeout(); } /** @@ -1081,22 +1079,19 @@ public class MasterExecThread implements Runnable { * @param taskInstance task instance * @return Boolean */ - private Boolean retryTaskIntervalOverTime(TaskInstance taskInstance){ + private boolean retryTaskIntervalOverTime(TaskInstance taskInstance){ if(taskInstance.getState() != ExecutionStatus.FAILURE){ - return Boolean.TRUE; + return true; } if(taskInstance.getId() == 0 || taskInstance.getMaxRetryTimes() ==0 || taskInstance.getRetryInterval() == 0 ){ - return Boolean.TRUE; + return true; } Date now = new Date(); long failedTimeInterval = DateUtils.differSec(now, taskInstance.getEndTime()); // task retry does not over time, return false - if(taskInstance.getRetryInterval() * SEC_2_MINUTES_TIME_UNIT >= failedTimeInterval){ - return Boolean.FALSE; - } - return Boolean.TRUE; + return taskInstance.getRetryInterval() * SEC_2_MINUTES_TIME_UNIT < failedTimeInterval; } /** @@ -1189,7 +1184,7 @@ public class MasterExecThread implements Runnable { */ private List getRecoveryNodeNameList(){ List recoveryNodeNameList = new ArrayList<>(); - if(recoverNodeIdList.size() > 0) { + if(CollectionUtils.isNotEmpty(recoverNodeIdList)) { for (TaskInstance task : recoverNodeIdList) { recoveryNodeNameList.add(task.getName()); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java index b86ff9952e..7f6baad427 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java @@ -122,22 +122,19 @@ public class MapReduceTask extends AbstractYarnTask { } // main class - if(mapreduceParameters.getProgramType() !=null ){ - if(mapreduceParameters.getProgramType()!= ProgramType.PYTHON){ - if(StringUtils.isNotEmpty(mapreduceParameters.getMainClass())){ - result.add(mapreduceParameters.getMainClass()); - } - } + if(!ProgramType.PYTHON.equals(mapreduceParameters.getProgramType()) + && StringUtils.isNotEmpty(mapreduceParameters.getMainClass())){ + result.add(mapreduceParameters.getMainClass()); } // others if (StringUtils.isNotEmpty(mapreduceParameters.getOthers())) { String others = mapreduceParameters.getOthers(); - if(!others.contains(Constants.MR_QUEUE)){ - if (StringUtils.isNotEmpty(mapreduceParameters.getQueue())) { - result.add(String.format("%s %s=%s", Constants.D, Constants.MR_QUEUE, mapreduceParameters.getQueue())); - } + if (!others.contains(Constants.MR_QUEUE) + && StringUtils.isNotEmpty(mapreduceParameters.getQueue())) { + result.add(String.format("%s %s=%s", Constants.D, Constants.MR_QUEUE, mapreduceParameters.getQueue())); } + result.add(mapreduceParameters.getOthers()); }else if (StringUtils.isNotEmpty(mapreduceParameters.getQueue())) { result.add(String.format("%s %s=%s", Constants.D, Constants.MR_QUEUE, mapreduceParameters.getQueue())); From e0fc3171da3b731433a4e8da3403070323007cce Mon Sep 17 00:00:00 2001 From: tswstarplanet Date: Sun, 15 Mar 2020 21:14:43 +0800 Subject: [PATCH 50/59] Cache HadoopUtils instance with specific days expire time (#2181) * Cache HadoopUtils instance with 7 days expire time * solve sonar issue * add kerberos expire time config * move KERBEROS_EXPIRE_TIME to Constants.java --- .../dolphinscheduler/common/Constants.java | 5 + .../common/utils/HadoopUtils.java | 234 +++++++++--------- .../src/main/resources/common.properties | 2 +- 3 files changed, 124 insertions(+), 117 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 6af0e6445f..b0a7b74d39 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -813,6 +813,11 @@ public final class Constants { */ public static final String KERBEROS = "kerberos"; + /** + * kerberos expire time + */ + public static final String KERBEROS_EXPIRE_TIME = "kerberos.expire.time"; + /** * java.security.krb5.conf */ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 6cb58a4324..e767911580 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -16,6 +16,9 @@ */ package org.apache.dolphinscheduler.common.utils; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ResUploadType; @@ -37,6 +40,7 @@ import java.security.PrivilegedExceptionAction; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -48,30 +52,37 @@ public class HadoopUtils implements Closeable { private static final Logger logger = LoggerFactory.getLogger(HadoopUtils.class); - private static HadoopUtils instance = new HadoopUtils(); + private static final String HADOOP_UTILS_KEY = "HADOOP_UTILS_KEY"; + + private static final LoadingCache cache = CacheBuilder + .newBuilder() + .expireAfterWrite(PropertyUtils.getInt(Constants.KERBEROS_EXPIRE_TIME, 7), TimeUnit.DAYS) + .build(new CacheLoader() { + @Override + public HadoopUtils load(String key) throws Exception { + return new HadoopUtils(); + } + }); + private static Configuration configuration; private static FileSystem fs; - private String hdfsUser; + private static String hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER); - private HadoopUtils(){ - hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER); + private HadoopUtils() { init(); initHdfsPath(); } - public static HadoopUtils getInstance(){ - // if kerberos startup , renew HadoopUtils - if (CommonUtils.getKerberosStartupState()){ - return new HadoopUtils(); - } - return instance; + public static HadoopUtils getInstance() { + + return cache.getUnchecked(HADOOP_UTILS_KEY); } /** * init dolphinscheduler root path in hdfs */ - private void initHdfsPath(){ + private void initHdfsPath() { String hdfsPath = PropertyUtils.getString(Constants.DATA_STORE_2_HDFS_BASEPATH); Path path = new Path(hdfsPath); @@ -80,7 +91,7 @@ public class HadoopUtils implements Closeable { fs.mkdirs(path); } } catch (Exception e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); } } @@ -88,82 +99,75 @@ public class HadoopUtils implements Closeable { /** * init hadoop configuration */ - private void init() { - if (configuration == null) { - synchronized (HadoopUtils.class) { - if (configuration == null) { - try { - configuration = new Configuration(); - - String resUploadStartupType = PropertyUtils.getString(Constants.RES_UPLOAD_STARTUP_TYPE); - ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); - - if (resUploadType == ResUploadType.HDFS){ - if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE)){ - System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, - PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)); - configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION,"kerberos"); - UserGroupInformation.setConfiguration(configuration); - UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), - PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)); - } + private static void init() { + try { + configuration = new Configuration(); + + String resUploadStartupType = PropertyUtils.getString(Constants.RES_UPLOAD_STARTUP_TYPE); + ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); + + if (resUploadType == ResUploadType.HDFS) { + if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE)) { + System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, + PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)); + configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + UserGroupInformation.setConfiguration(configuration); + UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), + PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)); + } - String defaultFS = configuration.get(Constants.FS_DEFAULTFS); - //first get key from core-site.xml hdfs-site.xml ,if null ,then try to get from properties file - // the default is the local file system - if(defaultFS.startsWith("file")){ - String defaultFSProp = PropertyUtils.getString(Constants.FS_DEFAULTFS); - if(StringUtils.isNotBlank(defaultFSProp)){ - Map fsRelatedProps = PropertyUtils.getPrefixedProperties("fs."); - configuration.set(Constants.FS_DEFAULTFS,defaultFSProp); - fsRelatedProps.forEach((key, value) -> configuration.set(key, value)); - }else{ - logger.error("property:{} can not to be empty, please set!", Constants.FS_DEFAULTFS ); - throw new RuntimeException( - String.format("property: %s can not to be empty, please set!", Constants.FS_DEFAULTFS) - ); - } - }else{ - logger.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", Constants.FS_DEFAULTFS, defaultFS); - } + String defaultFS = configuration.get(Constants.FS_DEFAULTFS); + //first get key from core-site.xml hdfs-site.xml ,if null ,then try to get from properties file + // the default is the local file system + if (defaultFS.startsWith("file")) { + String defaultFSProp = PropertyUtils.getString(Constants.FS_DEFAULTFS); + if (StringUtils.isNotBlank(defaultFSProp)) { + Map fsRelatedProps = PropertyUtils.getPrefixedProperties("fs."); + configuration.set(Constants.FS_DEFAULTFS, defaultFSProp); + fsRelatedProps.forEach((key, value) -> configuration.set(key, value)); + } else { + logger.error("property:{} can not to be empty, please set!", Constants.FS_DEFAULTFS); + throw new RuntimeException( + String.format("property: %s can not to be empty, please set!", Constants.FS_DEFAULTFS) + ); + } + } else { + logger.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", Constants.FS_DEFAULTFS, defaultFS); + } - if (fs == null) { - if(StringUtils.isNotEmpty(hdfsUser)){ - UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser); - ugi.doAs(new PrivilegedExceptionAction() { - @Override - public Boolean run() throws Exception { - fs = FileSystem.get(configuration); - return true; - } - }); - }else{ - logger.warn("hdfs.root.user is not set value!"); - fs = FileSystem.get(configuration); - } + if (fs == null) { + if (StringUtils.isNotEmpty(hdfsUser)) { + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser); + ugi.doAs(new PrivilegedExceptionAction() { + @Override + public Boolean run() throws Exception { + fs = FileSystem.get(configuration); + return true; } - }else if (resUploadType == ResUploadType.S3){ - configuration.set(Constants.FS_DEFAULTFS, PropertyUtils.getString(Constants.FS_DEFAULTFS)); - configuration.set(Constants.FS_S3A_ENDPOINT, PropertyUtils.getString(Constants.FS_S3A_ENDPOINT)); - configuration.set(Constants.FS_S3A_ACCESS_KEY, PropertyUtils.getString(Constants.FS_S3A_ACCESS_KEY)); - configuration.set(Constants.FS_S3A_SECRET_KEY, PropertyUtils.getString(Constants.FS_S3A_SECRET_KEY)); - fs = FileSystem.get(configuration); - } - - - String rmHaIds = PropertyUtils.getString(Constants.YARN_RESOURCEMANAGER_HA_RM_IDS); - String appAddress = PropertyUtils.getString(Constants.YARN_APPLICATION_STATUS_ADDRESS); - if (!StringUtils.isEmpty(rmHaIds)) { - appAddress = getAppAddress(appAddress, rmHaIds); - logger.info("appAddress : {}", appAddress); - } - configuration.set(Constants.YARN_APPLICATION_STATUS_ADDRESS, appAddress); - } catch (Exception e) { - logger.error(e.getMessage(), e); + }); + } else { + logger.warn("hdfs.root.user is not set value!"); + fs = FileSystem.get(configuration); } - } + } else if (resUploadType == ResUploadType.S3) { + configuration.set(Constants.FS_DEFAULTFS, PropertyUtils.getString(Constants.FS_DEFAULTFS)); + configuration.set(Constants.FS_S3A_ENDPOINT, PropertyUtils.getString(Constants.FS_S3A_ENDPOINT)); + configuration.set(Constants.FS_S3A_ACCESS_KEY, PropertyUtils.getString(Constants.FS_S3A_ACCESS_KEY)); + configuration.set(Constants.FS_S3A_SECRET_KEY, PropertyUtils.getString(Constants.FS_S3A_SECRET_KEY)); + fs = FileSystem.get(configuration); + } + + + String rmHaIds = PropertyUtils.getString(Constants.YARN_RESOURCEMANAGER_HA_RM_IDS); + String appAddress = PropertyUtils.getString(Constants.YARN_APPLICATION_STATUS_ADDRESS); + if (!StringUtils.isEmpty(rmHaIds)) { + appAddress = getAppAddress(appAddress, rmHaIds); + logger.info("appAddress : {}", appAddress); } + configuration.set(Constants.YARN_APPLICATION_STATUS_ADDRESS, appAddress); + } catch (Exception e) { + logger.error(e.getMessage(), e); } } @@ -187,14 +191,14 @@ public class HadoopUtils implements Closeable { /** * cat file on hdfs * - * @param hdfsFilePath hdfs file path + * @param hdfsFilePath hdfs file path * @return byte[] byte array * @throws IOException errors */ public byte[] catFile(String hdfsFilePath) throws IOException { - if(StringUtils.isBlank(hdfsFilePath)){ - logger.error("hdfs file path:{} is blank",hdfsFilePath); + if (StringUtils.isBlank(hdfsFilePath)) { + logger.error("hdfs file path:{} is blank", hdfsFilePath); return new byte[0]; } @@ -203,29 +207,28 @@ public class HadoopUtils implements Closeable { } - /** * cat file on hdfs * - * @param hdfsFilePath hdfs file path - * @param skipLineNums skip line numbers - * @param limit read how many lines + * @param hdfsFilePath hdfs file path + * @param skipLineNums skip line numbers + * @param limit read how many lines * @return content of file * @throws IOException errors */ public List catFile(String hdfsFilePath, int skipLineNums, int limit) throws IOException { - if (StringUtils.isBlank(hdfsFilePath)){ - logger.error("hdfs file path:{} is blank",hdfsFilePath); + if (StringUtils.isBlank(hdfsFilePath)) { + logger.error("hdfs file path:{} is blank", hdfsFilePath); return Collections.emptyList(); } - try (FSDataInputStream in = fs.open(new Path(hdfsFilePath))){ + try (FSDataInputStream in = fs.open(new Path(hdfsFilePath))) { BufferedReader br = new BufferedReader(new InputStreamReader(in)); Stream stream = br.lines().skip(skipLineNums).limit(limit); return stream.collect(Collectors.toList()); } - + } /** @@ -258,17 +261,17 @@ public class HadoopUtils implements Closeable { /** * the src file is on the local disk. Add it to FS at * the given dst name. - - * @param srcFile local file - * @param dstHdfsPath destination hdfs path - * @param deleteSource whether to delete the src - * @param overwrite whether to overwrite an existing file + * + * @param srcFile local file + * @param dstHdfsPath destination hdfs path + * @param deleteSource whether to delete the src + * @param overwrite whether to overwrite an existing file * @return if success or not * @throws IOException errors */ public boolean copyLocalToHdfs(String srcFile, String dstHdfsPath, boolean deleteSource, boolean overwrite) throws IOException { Path srcPath = new Path(srcFile); - Path dstPath= new Path(dstHdfsPath); + Path dstPath = new Path(dstHdfsPath); fs.copyFromLocalFile(deleteSource, overwrite, srcPath, dstPath); @@ -278,10 +281,10 @@ public class HadoopUtils implements Closeable { /** * copy hdfs file to local * - * @param srcHdfsFilePath source hdfs file path - * @param dstFile destination file - * @param deleteSource delete source - * @param overwrite overwrite + * @param srcHdfsFilePath source hdfs file path + * @param dstFile destination file + * @param deleteSource delete source + * @param overwrite overwrite * @return result of copy hdfs file to local * @throws IOException errors */ @@ -299,7 +302,7 @@ public class HadoopUtils implements Closeable { } } - if(!dstPath.getParentFile().exists()){ + if (!dstPath.getParentFile().exists()) { dstPath.getParentFile().mkdirs(); } @@ -307,14 +310,13 @@ public class HadoopUtils implements Closeable { } /** - * * delete a file * * @param hdfsFilePath the path to delete. - * @param recursive if path is a directory and set to - * true, the directory is deleted else throws an exception. In - * case of a file the recursive can be set to either true or false. - * @return true if delete is successful else false. + * @param recursive if path is a directory and set to + * true, the directory is deleted else throws an exception. In + * case of a file the recursive can be set to either true or false. + * @return true if delete is successful else false. * @throws IOException errors */ public boolean delete(String hdfsFilePath, boolean recursive) throws IOException { @@ -339,7 +341,7 @@ public class HadoopUtils implements Closeable { * @return {@link FileStatus} file status * @throws Exception errors */ - public FileStatus[] listFileStatus(String filePath)throws Exception{ + public FileStatus[] listFileStatus(String filePath) throws Exception { try { return fs.listStatus(new Path(filePath)); } catch (IOException e) { @@ -351,10 +353,11 @@ public class HadoopUtils implements Closeable { /** * Renames Path src to Path dst. Can take place on local fs * or remote DFS. + * * @param src path to be renamed * @param dst new path after rename - * @throws IOException on failure * @return true if rename is successful + * @throws IOException on failure */ public boolean rename(String src, String dst) throws IOException { return fs.rename(new Path(src), new Path(dst)); @@ -400,7 +403,6 @@ public class HadoopUtils implements Closeable { } /** - * * @return data hdfs path */ public static String getHdfsDataBasePath() { @@ -427,11 +429,11 @@ public class HadoopUtils implements Closeable { * hdfs user dir * * @param tenantCode tenant code - * @param userId user id + * @param userId user id * @return hdfs resource dir */ - public static String getHdfsUserDir(String tenantCode,int userId) { - return String.format("%s/home/%d", getHdfsTenantDir(tenantCode),userId); + public static String getHdfsUserDir(String tenantCode, int userId) { + return String.format("%s/home/%d", getHdfsTenantDir(tenantCode), userId); } /** @@ -479,7 +481,7 @@ public class HadoopUtils implements Closeable { * getAppAddress * * @param appAddress app address - * @param rmHa resource manager ha + * @param rmHa resource manager ha * @return app address */ public static String getAppAddress(String appAddress, String rmHa) { diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index 5a4aa1441f..8391a9e2d6 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -91,4 +91,4 @@ yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx # If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s - +kerberos.expire.time=7 From 1658176ef52e5a31414525f59d3d1fa2f1319bac Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Sun, 15 Mar 2020 21:25:52 +0800 Subject: [PATCH 51/59] Adapting partial code(file name start with P #1) to the sonar cloud rule (#2171) (#2187) --- .../utils/placeholder/PlaceholderUtils.java | 6 +-- .../utils/process/ProcessBuilderForWin32.java | 10 ++++- .../dao/datasource/PostgreDataSource.java | 2 +- .../dao/upgrade/PostgresqlUpgradeDao.java | 39 +++---------------- .../dao/utils/PostgrePerformance.java | 3 +- .../mapper/ProcessDefinitionMapperTest.java | 6 +-- .../dolphinscheduler/remote/command/Ping.java | 4 +- .../dolphinscheduler/remote/command/Pong.java | 4 +- .../server/utils/ParamUtils.java | 22 +++++------ .../task/processdure/ProcedureTask.java | 27 ++++++------- .../service/permission/PermissionCheck.java | 2 +- 11 files changed, 49 insertions(+), 76 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/PlaceholderUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/PlaceholderUtils.java index 0c756cb0b3..53a0d5af5b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/PlaceholderUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/PlaceholderUtils.java @@ -31,12 +31,12 @@ public class PlaceholderUtils { /** * Prefix of the position to be replaced */ - public static final String placeholderPrefix = "${"; + public static final String PLACEHOLDER_PREFIX = "${"; /** * The suffix of the position to be replaced */ - public static final String placeholderSuffix = "}"; + public static final String PLACEHOLDER_SUFFIX = "}"; /** @@ -68,7 +68,7 @@ public class PlaceholderUtils { */ public static PropertyPlaceholderHelper getPropertyPlaceholderHelper(boolean ignoreUnresolvablePlaceholders) { - return new PropertyPlaceholderHelper(placeholderPrefix, placeholderSuffix, null, ignoreUnresolvablePlaceholders); + return new PropertyPlaceholderHelper(PLACEHOLDER_PREFIX, PLACEHOLDER_SUFFIX, null, ignoreUnresolvablePlaceholders); } /** diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32.java index 4fb5f94616..eee456d019 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32.java @@ -426,6 +426,7 @@ public class ProcessBuilderForWin32 { static final ProcessBuilderForWin32.NullInputStream INSTANCE = new ProcessBuilderForWin32.NullInputStream(); private NullInputStream() {} public int read() { return -1; } + @Override public int available() { return 0; } } @@ -462,7 +463,7 @@ public class ProcessBuilderForWin32 { * * @since 1.7 */ - public static abstract class Redirect { + public abstract static class Redirect { /** * The type of a {@link ProcessBuilderForWin32.Redirect}. */ @@ -494,7 +495,7 @@ public class ProcessBuilderForWin32 { * {@link ProcessBuilderForWin32.Redirect#appendTo Redirect.appendTo(File)}. */ APPEND - }; + } /** * Returns the type of this {@code Redirect}. @@ -568,6 +569,7 @@ public class ProcessBuilderForWin32 { throw new NullPointerException(); return new ProcessBuilderForWin32.Redirect() { public Type type() { return Type.READ; } + @Override public File file() { return file; } public String toString() { return "redirect to read from file \"" + file + "\""; @@ -595,10 +597,12 @@ public class ProcessBuilderForWin32 { throw new NullPointerException(); return new ProcessBuilderForWin32.Redirect() { public Type type() { return Type.WRITE; } + @Override public File file() { return file; } public String toString() { return "redirect to write to file \"" + file + "\""; } + @Override boolean append() { return false; } }; } @@ -626,10 +630,12 @@ public class ProcessBuilderForWin32 { throw new NullPointerException(); return new ProcessBuilderForWin32.Redirect() { public Type type() { return Type.APPEND; } + @Override public File file() { return file; } public String toString() { return "redirect to append to file \"" + file + "\""; } + @Override boolean append() { return true; } }; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java index 176cba2587..c01dfa4dc0 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java @@ -40,7 +40,7 @@ public class PostgreDataSource extends BaseDataSource { @Override public String getJdbcUrl() { String jdbcUrl = getAddress(); - if (jdbcUrl.lastIndexOf("/") != (jdbcUrl.length() - 1)) { + if (jdbcUrl.lastIndexOf('/') != (jdbcUrl.length() - 1)) { jdbcUrl += "/"; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java index 5db273642a..c58521de1d 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java @@ -30,16 +30,8 @@ import java.sql.SQLException; */ public class PostgresqlUpgradeDao extends UpgradeDao { - public static final Logger logger = LoggerFactory.getLogger(UpgradeDao.class); - private static final String schema = getSchema(); - - /** - * init - */ - @Override - protected void init() { - - } + public static final Logger logger = LoggerFactory.getLogger(PostgresqlUpgradeDao.class); + private static final String SCHEMA = getSchema(); /** * postgresql upgrade dao holder @@ -58,16 +50,6 @@ public class PostgresqlUpgradeDao extends UpgradeDao { return PostgresqlUpgradeDaoHolder.INSTANCE; } - - /** - * init schema - * @param initSqlPath initSqlPath - */ - @Override - public void initSchema(String initSqlPath) { - super.initSchema(initSqlPath); - } - /** * getSchema * @return schema @@ -107,13 +89,9 @@ public class PostgresqlUpgradeDao extends UpgradeDao { try { conn = dataSource.getConnection(); - rs = conn.getMetaData().getTables(null, schema, tableName, null); - if (rs.next()) { - return true; - } else { - return false; - } + rs = conn.getMetaData().getTables(null, SCHEMA, tableName, null); + return rs.next(); } catch (SQLException e) { logger.error(e.getMessage(),e); throw new RuntimeException(e.getMessage(),e); @@ -135,13 +113,8 @@ public class PostgresqlUpgradeDao extends UpgradeDao { ResultSet rs = null; try { conn = dataSource.getConnection(); - rs = conn.getMetaData().getColumns(null,schema,tableName,columnName); - if (rs.next()) { - return true; - } else { - return false; - } - + rs = conn.getMetaData().getColumns(null, SCHEMA,tableName,columnName); + return rs.next(); } catch (SQLException e) { logger.error(e.getMessage(),e); throw new RuntimeException(e.getMessage(),e); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgrePerformance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgrePerformance.java index 031fd00681..b1cdf6f179 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgrePerformance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgrePerformance.java @@ -24,7 +24,6 @@ import java.util.Date; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.Flag; -import org.apache.dolphinscheduler.dao.MonitorDBDao; import org.apache.dolphinscheduler.dao.entity.MonitorRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,7 +33,7 @@ import org.slf4j.LoggerFactory; */ public class PostgrePerformance extends BaseDBPerformance { - private static Logger logger = LoggerFactory.getLogger(MonitorDBDao.class); + private static Logger logger = LoggerFactory.getLogger(PostgrePerformance.class); /** * get monitor record diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java index 1b3db55608..775d10656a 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java @@ -77,7 +77,7 @@ public class ProcessDefinitionMapperTest { //update processDefinition.setUpdateTime(new Date()); int update = processDefinitionMapper.updateById(processDefinition); - Assert.assertEquals(update, 1); + Assert.assertEquals(1, update); processDefinitionMapper.deleteById(processDefinition.getId()); } @@ -88,7 +88,7 @@ public class ProcessDefinitionMapperTest { public void testDelete(){ ProcessDefinition processDefinition = insertOne(); int delete = processDefinitionMapper.deleteById(processDefinition.getId()); - Assert.assertEquals(delete, 1); + Assert.assertEquals(1, delete); } /** @@ -189,7 +189,7 @@ public class ProcessDefinitionMapperTest { List processDefinitions = processDefinitionMapper.queryDefinitionListByIdList(array); processDefinitionMapper.deleteById(processDefinition.getId()); processDefinitionMapper.deleteById(processDefinition1.getId()); - Assert.assertEquals(processDefinitions.size(), 2); + Assert.assertEquals(2, processDefinitions.size()); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java index c5e4d075af..f90d3fff18 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java @@ -30,12 +30,12 @@ public class Ping implements Serializable { /** * ping body */ - protected static ByteBuf EMPTY_BODY = Unpooled.EMPTY_BUFFER; + protected static final ByteBuf EMPTY_BODY = Unpooled.EMPTY_BUFFER; /** * request command body */ - private static byte[] EMPTY_BODY_ARRAY = new byte[0]; + private static final byte[] EMPTY_BODY_ARRAY = new byte[0]; private static final ByteBuf PING_BUF; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Pong.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Pong.java index e52cef6d92..1b51373bff 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Pong.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Pong.java @@ -30,12 +30,12 @@ public class Pong implements Serializable { /** * pong body */ - protected static ByteBuf EMPTY_BODY = Unpooled.EMPTY_BUFFER; + protected static final ByteBuf EMPTY_BODY = Unpooled.EMPTY_BUFFER; /** * pong command body */ - private static byte[] EMPTY_BODY_ARRAY = new byte[0]; + private static final byte[] EMPTY_BODY_ARRAY = new byte[0]; /** * ping byte buffer diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ParamUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ParamUtils.java index 1d7a80daf0..3040cd198d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ParamUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ParamUtils.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.server.utils; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; import java.util.Date; @@ -70,17 +71,16 @@ public class ParamUtils { Map.Entry en = iter.next(); Property property = en.getValue(); - if (property.getValue() != null && property.getValue().length() > 0){ - if (property.getValue().startsWith("$")){ - /** - * local parameter refers to global parameter with the same name - * note: the global parameters of the process instance here are solidified parameters, - * and there are no variables in them. - */ - String val = property.getValue(); - val = ParameterUtils.convertParameterPlaceholders(val, timeParams); - property.setValue(val); - } + if (StringUtils.isNotEmpty(property.getValue()) + && property.getValue().startsWith("$")){ + /** + * local parameter refers to global parameter with the same name + * note: the global parameters of the process instance here are solidified parameters, + * and there are no variables in them. + */ + String val = property.getValue(); + val = ParameterUtils.convertParameterPlaceholders(val, timeParams); + property.setValue(val); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java index fd00e517b5..5a8afdb7bb 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java @@ -60,11 +60,6 @@ public class ProcedureTask extends AbstractTask { */ private ProcessService processService; - /** - * base datasource - */ - private BaseDataSource baseDataSource; - /** * constructor * @param taskProps task props @@ -117,7 +112,7 @@ public class ProcedureTask extends AbstractTask { // load class DataSourceFactory.loadClass(dataSource.getType()); // get datasource - baseDataSource = DataSourceFactory.getDatasource(dataSource.getType(), + BaseDataSource baseDataSource = DataSourceFactory.getDatasource(dataSource.getType(), dataSource.getConnectionParams()); // get jdbc connection @@ -163,7 +158,7 @@ public class ProcedureTask extends AbstractTask { stmt.setQueryTimeout(taskProps.getTaskTimeout()); } Map outParameterMap = new HashMap<>(); - if (userDefParamsList != null && userDefParamsList.size() > 0){ + if (CollectionUtils.isNotEmpty(userDefParamsList)){ int index = 1; for (Property property : userDefParamsList){ logger.info("localParams : prop : {} , dirct : {} , type : {} , value : {}" @@ -237,31 +232,31 @@ public class ProcedureTask extends AbstractTask { private void getOutputParameter(CallableStatement stmt, int index, String prop, DataType dataType) throws SQLException { switch (dataType){ case VARCHAR: - logger.info("out prameter key : {} , value : {}",prop,stmt.getString(index)); + logger.info("out prameter varchar key : {} , value : {}",prop,stmt.getString(index)); break; case INTEGER: - logger.info("out prameter key : {} , value : {}", prop, stmt.getInt(index)); + logger.info("out prameter integer key : {} , value : {}", prop, stmt.getInt(index)); break; case LONG: - logger.info("out prameter key : {} , value : {}",prop,stmt.getLong(index)); + logger.info("out prameter long key : {} , value : {}",prop,stmt.getLong(index)); break; case FLOAT: - logger.info("out prameter key : {} , value : {}",prop,stmt.getFloat(index)); + logger.info("out prameter float key : {} , value : {}",prop,stmt.getFloat(index)); break; case DOUBLE: - logger.info("out prameter key : {} , value : {}",prop,stmt.getDouble(index)); + logger.info("out prameter double key : {} , value : {}",prop,stmt.getDouble(index)); break; case DATE: - logger.info("out prameter key : {} , value : {}",prop,stmt.getDate(index)); + logger.info("out prameter date key : {} , value : {}",prop,stmt.getDate(index)); break; case TIME: - logger.info("out prameter key : {} , value : {}",prop,stmt.getTime(index)); + logger.info("out prameter time key : {} , value : {}",prop,stmt.getTime(index)); break; case TIMESTAMP: - logger.info("out prameter key : {} , value : {}",prop,stmt.getTimestamp(index)); + logger.info("out prameter timestamp key : {} , value : {}",prop,stmt.getTimestamp(index)); break; case BOOLEAN: - logger.info("out prameter key : {} , value : {}",prop, stmt.getBoolean(index)); + logger.info("out prameter boolean key : {} , value : {}",prop, stmt.getBoolean(index)); break; default: break; diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java index 027666f053..e53fae6e86 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java @@ -147,7 +147,7 @@ public class PermissionCheck { List unauthorizedList = processService.listUnauthorized(userId,needChecks,authorizationType); // if exist unauthorized resource if(CollectionUtils.isNotEmpty(unauthorizedList)){ - logger.error("user {} didn't has permission of {}: {}", user.getUserName(), authorizationType.getDescp(),unauthorizedList.toString()); + logger.error("user {} didn't has permission of {}: {}", user.getUserName(), authorizationType.getDescp(),unauthorizedList); throw new RuntimeException(String.format("user %s didn't has permission of %s %s", user.getUserName(), authorizationType.getDescp(), unauthorizedList.get(0))); } } From f6f29f4abb9f8034441a05ea4b7f85eef4f9f100 Mon Sep 17 00:00:00 2001 From: t1mon <178317391@qq.com> Date: Mon, 16 Mar 2020 17:10:55 +0800 Subject: [PATCH 52/59] Replace Charset.forName("UTF-8") with StandardCharsets.UTF_8. (#2193) --- .../dolphinscheduler/api/service/ProcessInstanceService.java | 5 +++-- .../org/apache/dolphinscheduler/remote/utils/Constants.java | 3 ++- .../apache/dolphinscheduler/server/utils/ProcessUtils.java | 4 ++-- .../dolphinscheduler/server/worker/task/datax/DataxTask.java | 4 ++-- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java index 4f81d89505..3530c66ff6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.api.service; +import java.nio.charset.StandardCharsets; import org.apache.dolphinscheduler.api.dto.gantt.GanttDto; import org.apache.dolphinscheduler.api.dto.gantt.Task; import org.apache.dolphinscheduler.api.enums.Status; @@ -49,7 +50,6 @@ import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStreamReader; -import java.nio.charset.Charset; import java.text.ParseException; import java.util.*; import java.util.stream.Collectors; @@ -273,7 +273,8 @@ public class ProcessInstanceService extends BaseDAGService { return resultMap; } - BufferedReader br = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(log.getBytes(Charset.forName("utf8"))), Charset.forName("utf8"))); + BufferedReader br = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(log.getBytes( + StandardCharsets.UTF_8)), StandardCharsets.UTF_8)); String line; while ((line = br.readLine()) != null) { if(line.contains(DEPENDENT_SPLIT)){ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java index 5733b17790..f4791715b8 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.remote.utils; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; /** @@ -31,7 +32,7 @@ public class Constants { /** * charset */ - public static final Charset UTF8 = Charset.forName("UTF-8"); + public static final Charset UTF8 = StandardCharsets.UTF_8; /** * cpus diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java index e0c00c55d9..f3f8a79489 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.server.utils; +import java.nio.charset.StandardCharsets; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.LoggerUtils; @@ -29,7 +30,6 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; -import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; @@ -297,7 +297,7 @@ public class ProcessUtils { File f = new File(commandFile); if (!f.exists()) { - FileUtils.writeStringToFile(new File(commandFile), sb.toString(), Charset.forName("UTF-8")); + FileUtils.writeStringToFile(new File(commandFile), sb.toString(), StandardCharsets.UTF_8); } String runCmd = "sh " + commandFile; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java index 7537ca2edc..952030ea62 100755 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.server.worker.task.datax; import java.io.File; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; @@ -209,7 +209,7 @@ public class DataxTask extends AbstractTask { logger.debug("datax job json : {}", root.toString()); // create datax json file - FileUtils.writeStringToFile(new File(fileName), root.toString(), Charset.forName("UTF-8")); + FileUtils.writeStringToFile(new File(fileName), root.toString(), StandardCharsets.UTF_8); return fileName; } From a3767642a546a4dac11975b0032eb80766871134 Mon Sep 17 00:00:00 2001 From: tswstarplanet Date: Mon, 16 Mar 2020 18:06:13 +0800 Subject: [PATCH 53/59] change static field to instance field (#2190) --- .../apache/dolphinscheduler/common/utils/HadoopUtils.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index e767911580..6c42704b47 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -64,8 +64,8 @@ public class HadoopUtils implements Closeable { } }); - private static Configuration configuration; - private static FileSystem fs; + private Configuration configuration; + private FileSystem fs; private static String hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER); @@ -99,7 +99,7 @@ public class HadoopUtils implements Closeable { /** * init hadoop configuration */ - private static void init() { + private void init() { try { configuration = new Configuration(); From f28ba55014b30c5498bf823dff5259dcd6c25af4 Mon Sep 17 00:00:00 2001 From: tswstarplanet Date: Mon, 16 Mar 2020 18:56:36 +0800 Subject: [PATCH 54/59] fix ResultSet not close and reformat code (#2183) * fix ResultSet not close * fix ResultSet not close * extract code of closing resource to a method * remove redundant if condition * modify e2e timeout --- .../dolphinscheduler/dao/TaskRecordDao.java | 134 ++++++++++-------- .../test/resources/config/config.properties | 8 +- 2 files changed, 79 insertions(+), 63 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java index f7ba1054a5..f89002ebdc 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java @@ -50,25 +50,28 @@ public class TaskRecordDao { static { try { conf = new PropertiesConfiguration(Constants.APPLICATION_PROPERTIES); - }catch (ConfigurationException e){ - logger.error("load configuration exception",e); + } catch (ConfigurationException e) { + logger.error("load configuration exception", e); System.exit(1); } } /** - * get task record flag + * get task record flag + * * @return whether startup taskrecord */ - public static boolean getTaskRecordFlag(){ - return conf.getBoolean(Constants.TASK_RECORD_FLAG); + public static boolean getTaskRecordFlag() { + return conf.getBoolean(Constants.TASK_RECORD_FLAG); } + /** * create connection + * * @return connection */ private static Connection getConn() { - if(!getTaskRecordFlag()){ + if (!getTaskRecordFlag()) { return null; } String driver = "com.mysql.jdbc.Driver"; @@ -90,101 +93,96 @@ public class TaskRecordDao { /** * generate where sql string + * * @param filterMap filterMap * @return sql string */ private static String getWhereString(Map filterMap) { - if(filterMap.size() ==0){ + if (filterMap.size() == 0) { return ""; } String result = " where 1=1 "; Object taskName = filterMap.get("taskName"); - if(taskName != null && StringUtils.isNotEmpty(taskName.toString())){ + if (taskName != null && StringUtils.isNotEmpty(taskName.toString())) { result += " and PROC_NAME like concat('%', '" + taskName.toString() + "', '%') "; } Object taskDate = filterMap.get("taskDate"); - if(taskDate != null && StringUtils.isNotEmpty(taskDate.toString())){ + if (taskDate != null && StringUtils.isNotEmpty(taskDate.toString())) { result += " and PROC_DATE='" + taskDate.toString() + "'"; } Object state = filterMap.get("state"); - if(state != null && StringUtils.isNotEmpty(state.toString())){ + if (state != null && StringUtils.isNotEmpty(state.toString())) { result += " and NOTE='" + state.toString() + "'"; } Object sourceTable = filterMap.get("sourceTable"); - if(sourceTable!= null && StringUtils.isNotEmpty(sourceTable.toString())){ - result += " and SOURCE_TAB like concat('%', '" + sourceTable.toString()+ "', '%')"; + if (sourceTable != null && StringUtils.isNotEmpty(sourceTable.toString())) { + result += " and SOURCE_TAB like concat('%', '" + sourceTable.toString() + "', '%')"; } Object targetTable = filterMap.get("targetTable"); - if(sourceTable!= null && StringUtils.isNotEmpty(targetTable.toString())){ - result += " and TARGET_TAB like concat('%', '"+ targetTable.toString()+"', '%') " ; + if (sourceTable != null && StringUtils.isNotEmpty(targetTable.toString())) { + result += " and TARGET_TAB like concat('%', '" + targetTable.toString() + "', '%') "; } Object start = filterMap.get("startTime"); - if(start != null && StringUtils.isNotEmpty(start.toString())){ + if (start != null && StringUtils.isNotEmpty(start.toString())) { result += " and STARTDATE>='" + start.toString() + "'"; } Object end = filterMap.get("endTime"); - if(end != null && StringUtils.isNotEmpty(end.toString())){ - result += " and ENDDATE>='" + end.toString()+ "'"; + if (end != null && StringUtils.isNotEmpty(end.toString())) { + result += " and ENDDATE>='" + end.toString() + "'"; } return result; } /** * count task record + * * @param filterMap filterMap - * @param table table + * @param table table * @return task record count */ - public static int countTaskRecord(Map filterMap, String table){ + public static int countTaskRecord(Map filterMap, String table) { int count = 0; Connection conn = null; PreparedStatement pstmt = null; + ResultSet rs = null; try { conn = getConn(); - if(conn == null){ + if (conn == null) { return count; } String sql = String.format("select count(1) as count from %s", table); sql += getWhereString(filterMap); pstmt = conn.prepareStatement(sql); - ResultSet rs = pstmt.executeQuery(); - while(rs.next()){ + rs = pstmt.executeQuery(); + while (rs.next()) { count = rs.getInt("count"); break; } } catch (SQLException e) { logger.error("Exception ", e); - }finally { - try { - if(pstmt != null) { - pstmt.close(); - } - if(conn != null){ - conn.close(); - } - } catch (SQLException e) { - logger.error("Exception ", e); - } + } finally { + closeResource(rs, pstmt, conn); } return count; } /** * query task record by filter map paging + * * @param filterMap filterMap - * @param table table + * @param table table * @return task record list */ - public static List queryAllTaskRecord(Map filterMap , String table) { + public static List queryAllTaskRecord(Map filterMap, String table) { String sql = String.format("select * from %s", table); sql += getWhereString(filterMap); @@ -194,9 +192,9 @@ public class TaskRecordDao { sql += String.format(" order by STARTDATE desc limit %d,%d", offset, pageSize); List recordList = new ArrayList<>(); - try{ + try { recordList = getQueryResult(sql); - }catch (Exception e){ + } catch (Exception e) { logger.error("Exception ", e); } return recordList; @@ -204,6 +202,7 @@ public class TaskRecordDao { /** * convert result set to task record + * * @param resultSet resultSet * @return task record * @throws SQLException if error throws SQLException @@ -232,6 +231,7 @@ public class TaskRecordDao { /** * query task list by select sql + * * @param selectSql select sql * @return task record list */ @@ -239,65 +239,81 @@ public class TaskRecordDao { List recordList = new ArrayList<>(); Connection conn = null; PreparedStatement pstmt = null; + ResultSet rs = null; try { conn = getConn(); - if(conn == null){ + if (conn == null) { return recordList; } pstmt = conn.prepareStatement(selectSql); - ResultSet rs = pstmt.executeQuery(); + rs = pstmt.executeQuery(); - while(rs.next()){ + while (rs.next()) { TaskRecord taskRecord = convertToTaskRecord(rs); recordList.add(taskRecord); } } catch (SQLException e) { logger.error("Exception ", e); - }finally { - try { - if(pstmt != null) { - pstmt.close(); - } - if(conn != null){ - conn.close(); - } - } catch (SQLException e) { - logger.error("Exception ", e); - } + } finally { + closeResource(rs, pstmt, conn); } return recordList; } /** * according to procname and procdate query task record + * * @param procName procName * @param procDate procDate * @return task record status */ - public static TaskRecordStatus getTaskRecordState(String procName,String procDate){ + public static TaskRecordStatus getTaskRecordState(String procName, String procDate) { String sql = String.format("SELECT * FROM eamp_hive_log_hd WHERE PROC_NAME='%s' and PROC_DATE like '%s'" - ,procName,procDate + "%"); + , procName, procDate + "%"); List taskRecordList = getQueryResult(sql); // contains no record and sql exception - if (CollectionUtils.isEmpty(taskRecordList)){ + if (CollectionUtils.isEmpty(taskRecordList)) { // exception return TaskRecordStatus.EXCEPTION; - }else if (taskRecordList.size() > 1){ + } else if (taskRecordList.size() > 1) { return TaskRecordStatus.EXCEPTION; - }else { + } else { TaskRecord taskRecord = taskRecordList.get(0); - if (taskRecord == null){ + if (taskRecord == null) { return TaskRecordStatus.EXCEPTION; } Long targetRowCount = taskRecord.getTargetRowCount(); - if (targetRowCount <= 0){ + if (targetRowCount <= 0) { return TaskRecordStatus.FAILURE; - }else { + } else { return TaskRecordStatus.SUCCESS; } } } + private static void closeResource(ResultSet rs, PreparedStatement pstmt, Connection conn) { + if (rs != null) { + try { + rs.close(); + } catch (SQLException e) { + logger.error("Exception ", e); + } + } + if (pstmt != null) { + try { + pstmt.close(); + } catch (SQLException e) { + logger.error("Exception ", e); + } + } + if (conn != null) { + try { + conn.close(); + } catch (SQLException e) { + logger.error("Exception ", e); + } + } + } } diff --git a/e2e/src/test/resources/config/config.properties b/e2e/src/test/resources/config/config.properties index 6a01234d01..352d9375be 100644 --- a/e2e/src/test/resources/config/config.properties +++ b/e2e/src/test/resources/config/config.properties @@ -27,13 +27,13 @@ PASSWORD=dolphinscheduler123 # driver path driver.chromeDriver=/usr/local/bin/chromedriver # implicitly wait(s) -driver.timeouts.implicitlyWait=10 +driver.timeouts.implicitlyWait=30 # show wait(s) -driver.timeouts.webDriverWait=10 +driver.timeouts.webDriverWait=30 # page load timeout(s) -driver.timeouts.pageLoadTimeout=10 +driver.timeouts.pageLoadTimeout=30 # JS wait timeouts(s) -driver.timeouts.setScriptTimeout=10 +driver.timeouts.setScriptTimeout=30 ############### redis ############## From 69633a29c60ac1c98704a419952f4e1cf56ea518 Mon Sep 17 00:00:00 2001 From: Rubik-W <39549317+Rubik-W@users.noreply.github.com> Date: Tue, 17 Mar 2020 11:10:55 +0800 Subject: [PATCH 55/59] fix: Api server startup fail (#2198) --- .../org/apache/dolphinscheduler/api/ApiApplicationServer.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java index 8376c2876d..5998ec5a4d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java @@ -25,7 +25,9 @@ import springfox.documentation.swagger2.annotations.EnableSwagger2; @SpringBootApplication @ServletComponentScan -@ComponentScan("org.apache.dolphinscheduler") +@ComponentScan({"org.apache.dolphinscheduler.api", + "org.apache.dolphinscheduler.dao", + "org.apache.dolphinscheduler.service"}) public class ApiApplicationServer extends SpringBootServletInitializer { public static void main(String[] args) { From c5e1c55c0003e3d5da525314a7af906bda545ccb Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Tue, 17 Mar 2020 14:58:16 +0800 Subject: [PATCH 56/59] Adapting partial code(file name start with M #2) to the sonar cloud rule (#2179) * Adapting partial code(file name start with M #2) to the sonar cloud rule * remove unused import --- .../dolphinscheduler/dao/MonitorDBDao.java | 2 +- .../dao/datasource/MySQLDataSource.java | 2 +- .../dao/upgrade/MysqlUpgradeDao.java | 23 +++---------------- .../dao/utils/MysqlPerformance.java | 3 +-- .../server/master/MasterServer.java | 3 +-- .../master/runner/MasterTaskExecThread.java | 7 +++--- .../server/monitor/MonitorServer.java | 2 +- .../sources/MysqlSourceGenerator.java | 19 ++++++++------- .../targets/MysqlTargetGenerator.java | 11 ++++----- 9 files changed, 25 insertions(+), 47 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java index 51f60666d1..5ea5966238 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java @@ -61,7 +61,7 @@ public class MonitorDBDao { return new PostgrePerformance().getMonitorRecord(conn); } }catch (Exception e) { - logger.error("SQLException " + e); + logger.error("SQLException: {}", e.getMessage(), e); }finally { try { if (conn != null) { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java index 6e2fbe3dd8..969c17b54d 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java @@ -39,7 +39,7 @@ public class MySQLDataSource extends BaseDataSource { @Override public String getJdbcUrl() { String address = getAddress(); - if (address.lastIndexOf("/") != (address.length() - 1)) { + if (address.lastIndexOf('/') != (address.length() - 1)) { address += "/"; } String jdbcUrl = address + getDatabase(); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java index 58bd673fc5..a20a3acb95 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java @@ -29,15 +29,7 @@ import java.sql.SQLException; */ public class MysqlUpgradeDao extends UpgradeDao { - public static final Logger logger = LoggerFactory.getLogger(UpgradeDao.class); - - /** - * init - */ - @Override - protected void init() { - - } + public static final Logger logger = LoggerFactory.getLogger(MysqlUpgradeDao.class); /** * mysql upgrade dao holder @@ -69,12 +61,7 @@ public class MysqlUpgradeDao extends UpgradeDao { try { conn = dataSource.getConnection(); rs = conn.getMetaData().getTables(null, null, tableName, null); - if (rs.next()) { - return true; - } else { - return false; - } - + return rs.next(); } catch (SQLException e) { logger.error(e.getMessage(),e); throw new RuntimeException(e.getMessage(),e); @@ -96,11 +83,7 @@ public class MysqlUpgradeDao extends UpgradeDao { try { conn = dataSource.getConnection(); ResultSet rs = conn.getMetaData().getColumns(null,null,tableName,columnName); - if (rs.next()) { - return true; - } else { - return false; - } + return rs.next(); } catch (SQLException e) { logger.error(e.getMessage(),e); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java index 40d12ab36f..1e1ee78036 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java @@ -27,7 +27,6 @@ import java.util.Date; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.Flag; -import org.apache.dolphinscheduler.dao.MonitorDBDao; import org.apache.dolphinscheduler.dao.entity.MonitorRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +36,7 @@ import org.slf4j.LoggerFactory; */ public class MysqlPerformance extends BaseDBPerformance{ - private static Logger logger = LoggerFactory.getLogger(MonitorDBDao.class); + private static Logger logger = LoggerFactory.getLogger(MysqlPerformance.class); /** diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index 9512b1a1c2..bf58607262 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -243,7 +243,7 @@ public class MasterServer implements IStoppable { */ private Runnable heartBeatThread(){ logger.info("start master heart beat thread..."); - Runnable heartBeatThread = new Runnable() { + return new Runnable() { @Override public void run() { if(Stopper.isRunning()) { @@ -257,7 +257,6 @@ public class MasterServer implements IStoppable { } } }; - return heartBeatThread; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java index 3cdb8a0271..38b5419a92 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java @@ -64,7 +64,7 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { /** * whether already Killed,default false */ - private Boolean alreadyKilled = false; + private boolean alreadyKilled = false; /** * submit task instance and wait complete @@ -98,7 +98,7 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { logger.info("wait task: process id: {}, task id:{}, task name:{} complete", this.taskInstance.getProcessInstanceId(), this.taskInstance.getId(), this.taskInstance.getName()); // task time out - Boolean checkTimeout = false; + boolean checkTimeout = false; TaskTimeoutParameter taskTimeoutParameter = getTaskTimeoutParameter(); if(taskTimeoutParameter.getEnable()){ TaskTimeoutStrategy strategy = taskTimeoutParameter.getStrategy(); @@ -189,7 +189,6 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { private long getRemaintime(long timeoutSeconds) { Date startTime = taskInstance.getStartTime(); long usedTime = (System.currentTimeMillis() - startTime.getTime()) / 1000; - long remainTime = timeoutSeconds - usedTime; - return remainTime; + return timeoutSeconds - usedTime; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/MonitorServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/MonitorServer.java index ac549bc386..a1f43add6e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/MonitorServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/MonitorServer.java @@ -30,7 +30,7 @@ import org.springframework.context.annotation.ComponentScan; @ComponentScan("org.apache.dolphinscheduler") public class MonitorServer implements CommandLineRunner { - private static Integer ARGS_LENGTH = 4; + private static final Integer ARGS_LENGTH = 4; private static final Logger logger = LoggerFactory.getLogger(MonitorServer.class); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java index 050fef7cc7..404f10bf39 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java @@ -69,17 +69,16 @@ public class MysqlSourceGenerator implements ISourceGenerator { result.append(" --columns ").append(sourceMysqlParameter.getSrcColumns()); } - }else if(sourceMysqlParameter.getSrcQueryType() == QueryType.SQL.ordinal()){ - if(StringUtils.isNotEmpty(sourceMysqlParameter.getSrcQuerySql())){ - - String srcQuery = sourceMysqlParameter.getSrcQuerySql(); - if(srcQuery.toLowerCase().contains("where")){ - srcQuery += " AND "+"$CONDITIONS"; - }else{ - srcQuery += " WHERE $CONDITIONS"; - } - result.append(" --query \'"+srcQuery+"\'"); + }else if(sourceMysqlParameter.getSrcQueryType() == QueryType.SQL.ordinal() + && StringUtils.isNotEmpty(sourceMysqlParameter.getSrcQuerySql())){ + String srcQuery = sourceMysqlParameter.getSrcQuerySql(); + if(srcQuery.toLowerCase().contains("where")){ + srcQuery += " AND "+"$CONDITIONS"; + }else{ + srcQuery += " WHERE $CONDITIONS"; } + result.append(" --query \'"+srcQuery+"\'"); + } List mapColumnHive = sourceMysqlParameter.getMapColumnHive(); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java index 0733338812..0e33b176e5 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java @@ -75,12 +75,11 @@ public class MysqlTargetGenerator implements ITargetGenerator { result.append(" --lines-terminated-by '").append(targetMysqlParameter.getLinesTerminated()).append("'"); } - if(targetMysqlParameter.isUpdate()){ - if(StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateKey())&& - StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateMode())){ - result.append(" --update-key ").append(targetMysqlParameter.getTargetUpdateKey()) - .append(" --update-mode ").append(targetMysqlParameter.getTargetUpdateMode()); - } + if(targetMysqlParameter.isUpdate() + && StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateKey()) + && StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateMode())){ + result.append(" --update-key ").append(targetMysqlParameter.getTargetUpdateKey()) + .append(" --update-mode ").append(targetMysqlParameter.getTargetUpdateMode()); } } } From 7dd126da71acc9f2b64cb44f8a53e69f0ee50487 Mon Sep 17 00:00:00 2001 From: t1mon <178317391@qq.com> Date: Tue, 17 Mar 2020 15:01:42 +0800 Subject: [PATCH 57/59] Optimize PropertyUtils instantiation. (#2199) * Optimize PropertyUtils instantiation. * Fix info error. --- .../dolphinscheduler/common/utils/PropertyUtils.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index 5c1011cd8b..ca928c94d5 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -43,13 +43,11 @@ public class PropertyUtils { private static final Properties properties = new Properties(); - private static final PropertyUtils propertyUtils = new PropertyUtils(); - - private PropertyUtils(){ - init(); + private PropertyUtils() { + throw new IllegalStateException("PropertyUtils class"); } - private void init(){ + static { String[] propertyFiles = new String[]{COMMON_PROPERTIES_PATH}; for (String fileName : propertyFiles) { InputStream fis = null; From c82758c5054e0c8af8f2c9ef73d1af8653e2444f Mon Sep 17 00:00:00 2001 From: xingchun-chen <55787491+xingchun-chen@users.noreply.github.com> Date: Tue, 17 Mar 2020 17:57:12 +0800 Subject: [PATCH 58/59] =?UTF-8?q?e2e=20add=20the=20case=20of=20deleting=20?= =?UTF-8?q?user=20and=20tenant=20=C2=A0=20(#2210)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add e2e --- .../dolphinscheduler/base/BaseDriver.java | 8 +- .../dolphinscheduler/base/BaseTest.java | 17 ++-- .../dolphinscheduler/data/LoginData.java | 2 +- .../data/project/CreatWorkflowData.java | 2 +- ...rojectData.java => CreateProjectData.java} | 4 +- .../data/project/CreateWorkflowData.java | 73 +++++++++++++++ .../data/security/TenantManageData.java | 10 +-- .../data/security/UserManageData.java | 5 +- .../project/CreateWorkflowLocator.java | 69 ++++++++++++-- .../locator/security/TenantManageLocator.java | 8 +- .../locator/security/UserManageLocator.java | 3 + .../dolphinscheduler/page/LoginPage.java | 14 ++- .../page/project/CreateProjectPage.java | 11 +-- .../page/project/CreateWorkflowPage.java | 89 +++++++++++++++---- .../page/security/TenantManagePage.java | 26 ++++-- .../page/security/UserManagePage.java | 25 ++++-- .../dolphinscheduler/testcase/LoginTest.java | 35 +++----- .../testcase/deleteData/DeleteTenantTest.java | 37 ++++++++ .../testcase/deleteData/DeleteUserTest.java | 36 ++++++++ .../testcase/project/CreateProjectTest.java | 7 +- .../testcase/project/CreateWorkflowTest.java | 10 ++- .../testcase/security/TenantManageTest.java | 11 ++- .../testcase/security/UserManageTest.java | 8 +- e2e/suite.xml | 18 ---- e2e/testng.xml | 28 +++--- 25 files changed, 415 insertions(+), 141 deletions(-) rename e2e/src/test/java/org/apache/dolphinscheduler/data/project/{CreatProjectData.java => CreateProjectData.java} (91%) create mode 100644 e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreateWorkflowData.java create mode 100644 e2e/src/test/java/org/apache/dolphinscheduler/testcase/deleteData/DeleteTenantTest.java create mode 100644 e2e/src/test/java/org/apache/dolphinscheduler/testcase/deleteData/DeleteUserTest.java delete mode 100644 e2e/suite.xml diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseDriver.java b/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseDriver.java index 7d3ab9b837..044e28f082 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseDriver.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseDriver.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.base; import org.apache.dolphinscheduler.constant.TestConstant; import org.apache.dolphinscheduler.util.PropertiesReader; +import org.openqa.selenium.Cookie; import org.openqa.selenium.JavascriptExecutor; import org.openqa.selenium.WebDriver; import org.openqa.selenium.chrome.ChromeDriver; @@ -35,7 +36,7 @@ public class BaseDriver { /** * driver */ - private WebDriver driver; + private static WebDriver driver; /** * chrome driver path @@ -88,6 +89,7 @@ public class BaseDriver { chromeOptions.setPageLoadStrategy(PageLoadStrategy.NONE); chromeOptions.addArguments("--no-sandbox"); chromeOptions.addArguments("--disable-dev-shm-usage"); + //Browser client running requires annotation --headless chromeOptions.addArguments("--headless"); chromeOptions.addArguments("--disable-gpu"); chromeOptions.addArguments("--whitelisted-ips"); @@ -120,7 +122,7 @@ public class BaseDriver { * * @return driver */ - public WebDriver getDriver() { + public static WebDriver getDriver() { return driver; } @@ -141,7 +143,7 @@ public class BaseDriver { public void closeBrowser() throws InterruptedException { // JS Show a pop-up box to indicate the end of the test Thread.sleep(TestConstant.ONE_THOUSANG); - ((JavascriptExecutor) driver).executeScript("alert('Test completed, browser closes after 3s')"); +// ((JavascriptExecutor) driver).executeScript("alert('Test completed, browser closes after 3s')"); Thread.sleep(TestConstant.THREE_THOUSANG); if (driver != null) { driver.quit(); diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseTest.java index c12c19fc1d..88012e9f42 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseTest.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseTest.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.base; -import org.apache.dolphinscheduler.page.LoginPage; import org.apache.dolphinscheduler.util.PropertiesReader; import org.openqa.selenium.WebDriver; import org.testng.annotations.*; @@ -28,6 +27,7 @@ import java.util.Properties; /** * base test class */ +@Test(groups={"functionTests"}) public class BaseTest { /** * properties @@ -43,7 +43,7 @@ public class BaseTest { /** * driver */ - public WebDriver driver; + public static WebDriver driver; /** * Executed before executing a test suite  @@ -54,7 +54,7 @@ public class BaseTest { */ @BeforeSuite(alwaysRun = true) @Parameters({"propertiesPath"}) - public void beforeSuite(@Optional("src/test/resources/config/config.properties") String propertiesPath) throws IOException { + public void beforeSuite(@Optional("src/test/resources/config/config.properties") String propertiesPath) throws Exception { // read properties properties = PropertiesReader.readProperties(propertiesPath); } @@ -70,14 +70,13 @@ public class BaseTest { driver = baseDriver.getDriver(); } + + /** * Executed before executing a class method in a test case */ @BeforeClass(alwaysRun = true) - public void setUp() throws IOException, InterruptedException { - LoginPage loginPage = new LoginPage(driver); - loginPage.jumpPage(); - loginPage.login(); + public void setUp() throws Exception { } @@ -85,7 +84,7 @@ public class BaseTest { * Execute after executing a class method in a test case */ @AfterClass(alwaysRun = true) - public void afterClass() { + public void afterClass() throws InterruptedException { // logout } @@ -102,6 +101,6 @@ public class BaseTest { * Execute after executing a testsuite */ @AfterSuite(alwaysRun = true) - public void afterSuite() { + public void afterSuite() throws InterruptedException { } } \ No newline at end of file diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/LoginData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/LoginData.java index 532849565c..e56df5a448 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/data/LoginData.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/LoginData.java @@ -39,5 +39,5 @@ public class LoginData { */ public static final String PASSWORD = PropertiesReader.getKey("PASSWORD"); - public static final String TENANT = "Tenant Manage - DolphinScheduler"; + public static final String TENANT = "租户管理 - DolphinScheduler"; } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreatWorkflowData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreatWorkflowData.java index 765a54f406..70f4238610 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreatWorkflowData.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreatWorkflowData.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.data.project; public class CreatWorkflowData { //input shell task name - public static final String SHELL_TASK_NAME = "shell task description test"; + public static final String SHELL_TASK_NAME = "shell task description test1"; //input shell task description public static final String SHELL_TASK_DESCRIPTION = "shell task description test"; diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreatProjectData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreateProjectData.java similarity index 91% rename from e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreatProjectData.java rename to e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreateProjectData.java index 8f6c9c8e97..57d79e87d7 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreatProjectData.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreateProjectData.java @@ -16,9 +16,9 @@ */ package org.apache.dolphinscheduler.data.project; -public class CreatProjectData { +public class CreateProjectData { // create project name - public static final String PROJECT_NAME = "selenium_project"; + public static final String PROJECT_NAME = "selenium_project_3"; // create project description public static final String DESCRIPTION = "test create project description"; // project page title diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreateWorkflowData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreateWorkflowData.java new file mode 100644 index 0000000000..31b5bdd1e0 --- /dev/null +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreateWorkflowData.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.data.project; + +public class CreateWorkflowData { + /** + * create workflow data + */ + //input shell task name + public static final String SHELL_TASK_NAME = "shell_task_selenium_5"; + + //input shell task description + public static final String SHELL_TASK_DESCRIPTION = "shell task description test"; + + //input timeout + public static final String INPUT_TIMEOUT = "60"; + + //input shell script + public static final String SHELL_SCRIPT = "echo 1111111"; + + //input custom parameters + public static final String INPUT_CUSTOM_PARAMETERS = "selenium_parameter"; + + //input custom parameters value + public static final String INPUT_CUSTOM_PARAMETERS_VALUE = "selenium_parameter_123"; + + //input add custom parameters + public static final String INPUT_ADD_CUSTOM_PARAMETERS = "selenium_parameter_delete"; + + //input add custom parameters value + public static final String INPUT_ADD_CUSTOM_PARAMETERS_VALUE = "selenium_parameter_delete_456"; + + //create workflow title + public static final String WORKFLOW_TITLE = "创建流程定义 - DolphinScheduler"; + + /** + * save workflow data + */ + //input workflow name + public static final String INPUT_WORKFLOW_NAME = "selenium_shell_1"; + + //input workflow description + public static final String INPUT_WORKFLOW_DESCRIPTION = "test selenium_shell_1 description"; + + //input workflow timeout + public static final String INPUT_WORKFLOW_TIMEOUT = "30"; + + //input workflow global parameters + public static final String INPUT_WORKFLOW_GLOBAL_PARAMETERS = "selenium_global_parameters_1"; + + //input workflow global parameters value + public static final String INPUT_WORKFLOW_GLOBAL_PARAMETERS_VALUES = "selenium_global_parameters_value_1"; + + //input to add workflow global parameters + public static final String INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS = "selenium_global_parameters_2"; + + //input to add workflow global parameters value + public static final String INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS_VALUES = "selenium_global_parameters_value_2"; +} diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/security/TenantManageData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/security/TenantManageData.java index e6f6ee6b86..100351b600 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/data/security/TenantManageData.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/security/TenantManageData.java @@ -20,18 +20,12 @@ */ package org.apache.dolphinscheduler.data.security; -import org.apache.dolphinscheduler.data.LoginData; public class TenantManageData { - /** - * Tenant URL - */ - public static final String TENANAT_URL = LoginData.URL + "/ui/#/security/tenant"; - /** * Tenant Code */ - public static final String TENANAT_CODE = "dolphinscheduler_tenant_code2"; + public static final String TENANAT_CODE = "dolphinscheduler_tenant_code15"; /** * Tenant Name @@ -48,7 +42,7 @@ public class TenantManageData { */ public static final String DESCRIPTION = "creat tenant test"; - public static final String TENANAT_MANAGE = "Tenant Manage - DolphinScheduler"; + public static final String TENANAT_MANAGE = "租户管理 - DolphinScheduler"; diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/security/UserManageData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/security/UserManageData.java index 03c985fd81..6fcd923c53 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/data/security/UserManageData.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/security/UserManageData.java @@ -16,12 +16,9 @@ */ package org.apache.dolphinscheduler.data.security; -import org.apache.dolphinscheduler.data.LoginData; public class UserManageData { - public static final String USER_URL = LoginData.URL + "/ui/#/security/users"; - - public static final String USERNAME = "selenium111"; + public static final String USERNAME = "selenium_5"; public static final String PASSWORD = "123456qwe"; diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateWorkflowLocator.java b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateWorkflowLocator.java index f063d6ef61..25defcf53f 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateWorkflowLocator.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateWorkflowLocator.java @@ -20,12 +20,20 @@ package org.apache.dolphinscheduler.locator.project; import org.openqa.selenium.By; public class CreateWorkflowLocator { + + /** + * jump workflow define page + */ + // click project name public static final By CLICK_PROJECT_NAME = By.xpath("//span/a"); // click workflow define public static final By CLICK_WORKFLOW_DEFINE = By.xpath("//li/span"); + /** + * create workflow + */ // click create workflow button public static final By CLICK_CREATE_WORKFLOW_BUTTON = By.xpath("//button/span"); @@ -96,15 +104,15 @@ public class CreateWorkflowLocator { public static final By INPUT_CUSTOM_PARAMETERS = By.xpath("//div[2]/div/div/div/div/div/input"); //input custom parameters value - public static final By INPUT_CUSTOM_PARAMETERS_VALUE = By.xpath("//div[2]/input"); + public static final By INPUT_CUSTOM_PARAMETERS_VALUE = By.xpath("//div[2]/div/div/div/div[1]/div[2]/input"); - //click add custom parameters - public static final By CLICK_ADD_CUSTOM_PARAMETERS = By.xpath("//span[2]/a/em"); + //click to add custom parameters + public static final By CLICK_ADD_CUSTOM_PARAMETERS = By.xpath("//div[2]/div/div/div/div/span[2]/a/em"); - //input add custom parameters - public static final By INPUT_ADD_CUSTOM_PARAMETERS = By.xpath("//div[2]/div/div/div/div[2]/div/input"); + //input to add custom parameters + public static final By INPUT_ADD_CUSTOM_PARAMETERS = By.xpath("//div[2]/div/div/div/div[2]/div[1]/input"); - //input add custom parameters value + //input to add custom parameters value public static final By INPUT_ADD_CUSTOM_PARAMETERS_VALUE = By.xpath("//div[2]/div[2]/input"); //delete custom parameters @@ -112,4 +120,53 @@ public class CreateWorkflowLocator { //click submit button public static final By CLICK_SUBMIT_BUTTON = By.xpath("//button[2]/span"); + + + /** + * save workflow + */ + //click save workflow button + public static final By CLICK_SAVE_WORKFLOW_BUTTON = By.xpath("//button/span"); + + //input workflow name + public static final By INPUT_WORKFLOW_NAME = By.xpath("//input"); + + //input workflow description + public static final By INPUT_WORKFLOW_DESCRIPTION = By.xpath("//textarea"); + + //select tenant + public static final By CLICK_TENANT = By.xpath("//div[4]/div/div/div/input"); + public static final By SELECT_TENANT = By.xpath("//div[2]/div/div/div/ul/li/span"); + + + //click workflow timeout alarm + public static final By CLICK_WORKFLOW_TIMEOUT_ALARM = By.xpath("//span[2]/span/span"); + + //input workflow timeout + public static final By INPUT_WORKFLOW_TIMEOUT = By.xpath("//span/div/input"); + + //click workflow global parameters + public static final By CLICK_WORKFLOW_GLOBAL_PARAMETERS = By.xpath("//span/a/em"); + + //input workflow global parameters + public static final By INPUT_WORKFLOW_GLOBAL_PARAMETERS = By.xpath("//div[8]/div/div/div/div/input"); + + //input workflow global parameters value + public static final By INPUT_WORKFLOW_GLOBAL_PARAMETERS_VALUES = By.xpath("//div[2]/input"); + + //click to add workflow global parameters + public static final By CLICK_ADD_WORKFLOW_GLOBAL_PARAMETERS = By.xpath("//span[2]/a/em"); + + //input to add workflow global parameters + public static final By INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS = By.xpath("//div[8]/div/div/div[2]/div/input"); + + //input to add workflow global parameters value + public static final By INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS_VALUES = By.xpath("//div[2]/div[2]/input"); + + //delete workflow global parameters value + public static final By CLICK_DELETE_WORKFLOW_GLOBAL_PARAMETERS = By.xpath("//div[2]/span/a/em"); + + //click add button + public static final By CLICK_ADD_BUTTON = By.xpath("//button[2]/span"); + } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/TenantManageLocator.java b/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/TenantManageLocator.java index 7d9c8a57ce..d95265cf0b 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/TenantManageLocator.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/TenantManageLocator.java @@ -19,7 +19,9 @@ package org.apache.dolphinscheduler.locator.security; import org.openqa.selenium.By; public class TenantManageLocator{ - public static final By CREATE_TENANT_BUTTON = By.xpath("//button[@class='ans-btn ans-btn-ghost ans-btn-small']"); + public static final By TENANT_MANAGE = By.xpath("//div[2]/div/a/div/a/span"); + + public static final By CREATE_TENANT_BUTTON = By.xpath("//button/span"); public static final By TENANT_INPUT_CODE = By.xpath("//div[2]/div/div/div[2]/div/input"); @@ -30,4 +32,8 @@ public class TenantManageLocator{ public static final By DESCRIPTION = By.xpath("//textarea"); public static final By SUBMIT_BUTTON = By.xpath("//div[3]/button[2]/span"); + + public static final By DELETE_TENANT_BUTTON = By.xpath("//div[3]/div[1]/div/table/tr[2]/td[8]/span/button"); + + public static final By CONFIRM_DELETE_TENANT_BUTTON = By.xpath("//div[2]/div/button[2]/span"); } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/UserManageLocator.java b/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/UserManageLocator.java index 0d84692cb2..b5ecccc67d 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/UserManageLocator.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/UserManageLocator.java @@ -42,4 +42,7 @@ public class UserManageLocator { public static final By SUBMIT = By.xpath("//div[3]/button[2]/span"); + public static final By DELETE_USER_BUTTON = By.xpath("//span[2]/button/i"); + + public static final By CONFIRM_DELETE_USER_BUTTON = By.xpath("//div[2]/div/button[2]/span"); } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/LoginPage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/LoginPage.java index cd6b318651..14970bead5 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/LoginPage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/LoginPage.java @@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.constant.TestConstant; import org.apache.dolphinscheduler.data.LoginData; import org.apache.dolphinscheduler.locator.LoginLocator; import org.apache.dolphinscheduler.util.RedisUtil; +import org.openqa.selenium.Cookie; import org.openqa.selenium.WebDriver; @@ -38,18 +39,25 @@ public class LoginPage extends PageCommon { /** * jump page */ - public void jumpPage() { - System.out.println("jump login page"); + public void jumpPageEnlish() { + System.out.println("jump to English login page"); super.jumpPage(LoginData.URL); + Cookie cookie = new Cookie("language", "en_US", "/", null); + driver.manage().addCookie(cookie); } + public void jumpPageChinese() { + super.jumpPage(LoginData.URL); + Cookie cookie = new Cookie("language", "zh_CN", "/", null); + + driver.manage().addCookie(cookie); + } /** * login * * @return Whether to enter the specified page after searching */ public boolean login() throws InterruptedException { - System.out.println("LoginPage"); // login data sendInput(LoginLocator.LOGIN_INPUT_USER, LoginData.USER); sendInput(LoginLocator.LOGIN_INPUT_PASSWORD, LoginData.PASSWORD); diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateProjectPage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateProjectPage.java index 8dd1010a82..d8e1ef655d 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateProjectPage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateProjectPage.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.page.project; import org.apache.dolphinscheduler.common.PageCommon; import org.apache.dolphinscheduler.constant.TestConstant; -import org.apache.dolphinscheduler.data.project.CreatProjectData; +import org.apache.dolphinscheduler.data.project.CreateProjectData; import org.apache.dolphinscheduler.locator.project.CreateProjectLocator; import org.openqa.selenium.WebDriver; @@ -26,6 +26,7 @@ public class CreateProjectPage extends PageCommon { public CreateProjectPage(WebDriver driver) { super(driver); } + /** * jump page */ @@ -36,7 +37,7 @@ public class CreateProjectPage extends PageCommon { } /** - * creatTenant + * createTenant * * @return Whether to enter the specified page after creat tenant */ @@ -46,13 +47,13 @@ public class CreateProjectPage extends PageCommon { Thread.sleep(TestConstant.ONE_THOUSANG); // input create project data - sendInput(CreateProjectLocator.PROJECT_NAME, CreatProjectData.PROJECT_NAME); - sendInput(CreateProjectLocator.PROJECT_DESCRIPTION, CreatProjectData.DESCRIPTION); + sendInput(CreateProjectLocator.PROJECT_NAME, CreateProjectData.PROJECT_NAME); + sendInput(CreateProjectLocator.PROJECT_DESCRIPTION, CreateProjectData.DESCRIPTION); // click submit button clickButton(CreateProjectLocator.SUBMIT_BUTTON); // Whether to enter the specified page after submit - return ifTitleContains(CreatProjectData.PROJECT_TITLE); + return ifTitleContains(CreateProjectData.PROJECT_TITLE); } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateWorkflowPage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateWorkflowPage.java index aeec83cd19..d115124d8f 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateWorkflowPage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateWorkflowPage.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.page.project; import org.apache.dolphinscheduler.common.PageCommon; import org.apache.dolphinscheduler.constant.TestConstant; -import org.apache.dolphinscheduler.data.project.CreatWorkflowData; +import org.apache.dolphinscheduler.data.project.CreateWorkflowData; import org.apache.dolphinscheduler.locator.project.CreateWorkflowLocator; import org.openqa.selenium.WebDriver; @@ -26,29 +26,33 @@ public class CreateWorkflowPage extends PageCommon { public CreateWorkflowPage(WebDriver driver) { super(driver); } + /** - * jump page + * jump create workflow page */ - public boolean createWorkflow() throws InterruptedException { + System.out.println("Click on the project name to jump to the project homepage"); // click project name clickElement(CreateWorkflowLocator.CLICK_PROJECT_NAME); Thread.sleep(TestConstant.ONE_THOUSANG); + System.out.println("Click on workflow define"); // click workflow define clickElement(CreateWorkflowLocator.CLICK_WORKFLOW_DEFINE); Thread.sleep(TestConstant.ONE_THOUSANG); + System.out.println("Click create workflow button"); // click create workflow button clickElement(CreateWorkflowLocator.CLICK_CREATE_WORKFLOW_BUTTON); Thread.sleep(TestConstant.ONE_THOUSANG); + System.out.println("drag shell task"); //drag shell_task dragAndDrop(CreateWorkflowLocator.MOUSE_DOWN_AT_SHELL,CreateWorkflowLocator.MOUSE_MOVE_SHELL_AT_DAG); //input shell task _name - sendInput(CreateWorkflowLocator.INPUT_SHELL_TASK_NAME , CreatWorkflowData.SHELL_TASK_NAME); + sendInput(CreateWorkflowLocator.INPUT_SHELL_TASK_NAME , CreateWorkflowData.SHELL_TASK_NAME); //click stop run type clickElement(CreateWorkflowLocator.CLICK_STOP_RUN_TYPE); @@ -57,7 +61,7 @@ public class CreateWorkflowPage extends PageCommon { clickElement(CreateWorkflowLocator.CLICK_NORMAL_RUN_TYPE); //input shell task description - sendInput(CreateWorkflowLocator.INPUT_SHELL_TASK_DESCRIPTION , CreatWorkflowData.SHELL_TASK_DESCRIPTION); + sendInput(CreateWorkflowLocator.INPUT_SHELL_TASK_DESCRIPTION , CreateWorkflowData.SHELL_TASK_DESCRIPTION); //select task priority clickElement(CreateWorkflowLocator.CLICK_TASK_PRIORITY); @@ -73,19 +77,15 @@ public class CreateWorkflowPage extends PageCommon { //select failed retry interval clickElement(CreateWorkflowLocator.SELECT_FAIL_RETRIES_INTERVAL); - //click timeout alarm clickElement(CreateWorkflowLocator.CLICK_TIMEOUT_ALARM); - //select timeout fail clickElement(CreateWorkflowLocator.SELECT_TIMEOUT_FAIL); - //cancel timeout alarm clickElement(CreateWorkflowLocator.CANCEL_TIMEOUT_ALARM); - //select timeout alarm clickElement(CreateWorkflowLocator.SELECT_TIMEOUT_ALARM); @@ -94,10 +94,10 @@ public class CreateWorkflowPage extends PageCommon { clearInput(CreateWorkflowLocator.SELECT_TIMEOUT); //input timeout - sendInput(CreateWorkflowLocator.SELECT_TIMEOUT,CreatWorkflowData.INPUT_TIMEOUT); + sendInput(CreateWorkflowLocator.SELECT_TIMEOUT, CreateWorkflowData.INPUT_TIMEOUT); //click codeMirror and input script - inputCodeMirror(CreateWorkflowLocator.CLICK_CODE_MIRROR, CreateWorkflowLocator.INPUT_SCRIPT,CreatWorkflowData.SHELL_SCRIPT); + inputCodeMirror(CreateWorkflowLocator.CLICK_CODE_MIRROR, CreateWorkflowLocator.INPUT_SCRIPT, CreateWorkflowData.SHELL_SCRIPT); scrollToElementBottom(); Thread.sleep(TestConstant.ONE_THOUSANG); @@ -105,10 +105,10 @@ public class CreateWorkflowPage extends PageCommon { clickElement(CreateWorkflowLocator.CLICK_CUSTOM_PARAMETERS); //input custom parameters - sendInput(CreateWorkflowLocator.INPUT_CUSTOM_PARAMETERS, CreatWorkflowData.INPUT_CUSTOM_PARAMETERS); + sendInput(CreateWorkflowLocator.INPUT_CUSTOM_PARAMETERS, CreateWorkflowData.INPUT_CUSTOM_PARAMETERS); //input custom parameters value - sendInput(CreateWorkflowLocator.INPUT_CUSTOM_PARAMETERS_VALUE, CreatWorkflowData.INPUT_CUSTOM_PARAMETERS_VALUE); + sendInput(CreateWorkflowLocator.INPUT_CUSTOM_PARAMETERS_VALUE, CreateWorkflowData.INPUT_CUSTOM_PARAMETERS_VALUE); //click add custom parameters clickElement(CreateWorkflowLocator.CLICK_ADD_CUSTOM_PARAMETERS); @@ -117,10 +117,10 @@ public class CreateWorkflowPage extends PageCommon { Thread.sleep(TestConstant.ONE_THOUSANG); //input add custom parameters - sendInput(CreateWorkflowLocator.INPUT_ADD_CUSTOM_PARAMETERS,CreatWorkflowData.INPUT_ADD_CUSTOM_PARAMETERS); + sendInput(CreateWorkflowLocator.INPUT_ADD_CUSTOM_PARAMETERS, CreateWorkflowData.INPUT_ADD_CUSTOM_PARAMETERS); //input add custom parameters value - sendInput(CreateWorkflowLocator.INPUT_ADD_CUSTOM_PARAMETERS_VALUE,CreatWorkflowData.INPUT_ADD_CUSTOM_PARAMETERS_VALUE); + sendInput(CreateWorkflowLocator.INPUT_ADD_CUSTOM_PARAMETERS_VALUE, CreateWorkflowData.INPUT_ADD_CUSTOM_PARAMETERS_VALUE); //click delete custom parameters clickElement(CreateWorkflowLocator.CLICK_DELETE_CUSTOM_PARAMETERS); @@ -129,9 +129,64 @@ public class CreateWorkflowPage extends PageCommon { //click submit button clickElement(CreateWorkflowLocator.CLICK_SUBMIT_BUTTON); Thread.sleep(TestConstant.ONE_THOUSANG); - + System.out.println("Task node set up successfully"); + System.out.println("move to Dag Element "); moveToDragElement(CreateWorkflowLocator.MOUSE_MOVE_SHELL_AT_DAG,-300,-100); - return ifTitleContains(CreatWorkflowData.WORKFLOW_TITLE); + return ifTitleContains(CreateWorkflowData.WORKFLOW_TITLE); + } + + /** + * save workflow + */ + public boolean saveWorkflow() throws InterruptedException { + System.out.println("start to save workflow "); + + //click save workflow button + clickElement(CreateWorkflowLocator.CLICK_SAVE_WORKFLOW_BUTTON); + + //input workflow name + sendInput(CreateWorkflowLocator.INPUT_WORKFLOW_NAME, CreateWorkflowData.INPUT_WORKFLOW_NAME); + + //input workflow description + sendInput(CreateWorkflowLocator.INPUT_WORKFLOW_DESCRIPTION, CreateWorkflowData.INPUT_WORKFLOW_DESCRIPTION); + + //select tenant + clickElement(CreateWorkflowLocator.CLICK_TENANT); + clickElement(CreateWorkflowLocator.SELECT_TENANT); + + //click workflow timeout alarm + clickElement(CreateWorkflowLocator.CLICK_WORKFLOW_TIMEOUT_ALARM); + clearInput(CreateWorkflowLocator.INPUT_WORKFLOW_TIMEOUT); + + //input workflow timeout + sendInput(CreateWorkflowLocator.INPUT_WORKFLOW_TIMEOUT, CreateWorkflowData.INPUT_WORKFLOW_TIMEOUT); + + //click workflow global parameters + clickElement(CreateWorkflowLocator.CLICK_WORKFLOW_GLOBAL_PARAMETERS); + + //input workflow global parameters + sendInput(CreateWorkflowLocator.INPUT_WORKFLOW_GLOBAL_PARAMETERS, CreateWorkflowData.INPUT_WORKFLOW_GLOBAL_PARAMETERS); + + //input workflow global parameters value + sendInput(CreateWorkflowLocator.INPUT_WORKFLOW_GLOBAL_PARAMETERS_VALUES, CreateWorkflowData.INPUT_WORKFLOW_GLOBAL_PARAMETERS_VALUES); + + //click to add workflow global parameters + clickElement(CreateWorkflowLocator.CLICK_ADD_WORKFLOW_GLOBAL_PARAMETERS); + + //input to add workflow global parameters + sendInput(CreateWorkflowLocator.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS, CreateWorkflowData.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS); + + //input to add workflow global parameters value + sendInput(CreateWorkflowLocator.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS_VALUES, CreateWorkflowData.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS_VALUES); + + //delete workflow global parameters value + clickElement(CreateWorkflowLocator.CLICK_DELETE_WORKFLOW_GLOBAL_PARAMETERS); + Thread.sleep(TestConstant.ONE_THOUSANG); + + //click add button + clickElement(CreateWorkflowLocator.CLICK_ADD_BUTTON); + System.out.println("submit workflow"); + return ifTitleContains(CreateWorkflowData.WORKFLOW_TITLE); } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/security/TenantManagePage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/security/TenantManagePage.java index 4c88f6575e..618e1bc213 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/security/TenantManagePage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/security/TenantManagePage.java @@ -34,15 +34,6 @@ public class TenantManagePage extends PageCommon { super(driver); } - - /** - * jump page - */ - public void jumpPage() { - System.out.println("jump tenant page"); - super.jumpPage(TenantManageData.TENANAT_URL); - } - /** * createTenant * @@ -50,6 +41,7 @@ public class TenantManagePage extends PageCommon { */ public boolean createTenant() throws InterruptedException { Thread.sleep(TestConstant.ONE_THOUSANG); + clickButton(TenantManageLocator.TENANT_MANAGE); //create tenant clickButton(TenantManageLocator.CREATE_TENANT_BUTTON); @@ -66,4 +58,20 @@ public class TenantManagePage extends PageCommon { // Whether to enter the specified page after submit return ifTitleContains(TenantManageData.TENANAT_MANAGE); } + + public boolean deleteTenant() throws InterruptedException { + Thread.sleep(TestConstant.ONE_THOUSANG); + clickButton(TenantManageLocator.TENANT_MANAGE); + Thread.sleep(TestConstant.ONE_THOUSANG); + + // click delete button + clickButton(TenantManageLocator.DELETE_TENANT_BUTTON); + Thread.sleep(TestConstant.ONE_THOUSANG); + + //click confirm delete button + clickButton(TenantManageLocator.CONFIRM_DELETE_TENANT_BUTTON); + + // Whether to enter the specified page after submit + return ifTitleContains(TenantManageData.TENANAT_MANAGE); + } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/security/UserManagePage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/security/UserManagePage.java index 3e2be2d126..70311f129e 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/security/UserManagePage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/security/UserManagePage.java @@ -26,16 +26,9 @@ public class UserManagePage extends PageCommon { public UserManagePage(WebDriver driver) { super(driver); } - /** - * jump page - */ - public void jumpPage() { - System.out.println("jump tenant page"); - super.jumpPage(UserManageData.USER_URL); - } /** - * creatTenant + * createTenant * * @return Whether to enter the specified page after creat tenant */ @@ -64,4 +57,20 @@ public class UserManagePage extends PageCommon { // Whether to enter the specified page after submit return ifTitleContains(UserManageData.USER_MANAGE); } + + public boolean deleteUser() throws InterruptedException { + Thread.sleep(TestConstant.ONE_THOUSANG); + // click user manage + clickElement(UserManageLocator.CLICK_USER_MANAGE); + Thread.sleep(TestConstant.ONE_THOUSANG); + + // click delete user button + clickButton(UserManageLocator.DELETE_USER_BUTTON ); + + // click confirm delete button + clickButton(UserManageLocator.CONFIRM_DELETE_USER_BUTTON); + + // Whether to enter the specified page after submit + return ifTitleContains(UserManageData.USER_MANAGE); + } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/LoginTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/LoginTest.java index bd3c31d38b..47ce11287d 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/LoginTest.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/LoginTest.java @@ -1,37 +1,26 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package org.apache.dolphinscheduler.testcase; -import org.apache.dolphinscheduler.base.BaseTest; import org.apache.dolphinscheduler.page.LoginPage; +import org.testng.Assert; import org.testng.annotations.Test; +import static org.apache.dolphinscheduler.base.BaseTest.driver; -public class LoginTest extends BaseTest { +@Test(groups={"functionTests","login"}) +public class LoginTest { private LoginPage loginPage; + @Test(description = "LoginTest", priority = 1) public void testLogin() throws InterruptedException { - // init login page loginPage = new LoginPage(driver); + System.out.println("==================================="); + System.out.println("jump to Chinese login page"); + loginPage.jumpPageChinese(); - // enter login page - loginPage.jumpPage(); + System.out.println("start login"); + assert loginPage.login(); + System.out.println("end login"); + System.out.println("==================================="); - //assert login page - assert loginPage.login(); } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/deleteData/DeleteTenantTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/deleteData/DeleteTenantTest.java new file mode 100644 index 0000000000..ddbc9d603b --- /dev/null +++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/deleteData/DeleteTenantTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.testcase.deleteData; + +import org.apache.dolphinscheduler.base.BaseTest; +import org.apache.dolphinscheduler.page.security.TenantManagePage; +import org.testng.annotations.Test; + +public class DeleteTenantTest extends BaseTest { + private TenantManagePage tenantManagePage; + + @Test(groups={"functionTests"},dependsOnGroups = { "login","createTenant"},description = "DeleteTenantTest") + public void testDeleteTenant() throws InterruptedException { + tenantManagePage = new TenantManagePage(driver); + //assert tenant manage page + System.out.println("start delete tenant"); + assert tenantManagePage.deleteTenant(); + System.out.println("end delete tenant"); + System.out.println("==================================="); + } + + +} diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/deleteData/DeleteUserTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/deleteData/DeleteUserTest.java new file mode 100644 index 0000000000..770fae077d --- /dev/null +++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/deleteData/DeleteUserTest.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.testcase.deleteData; + +import org.apache.dolphinscheduler.base.BaseTest; +import org.apache.dolphinscheduler.page.security.UserManagePage; +import org.testng.annotations.Test; + +public class DeleteUserTest extends BaseTest { + private UserManagePage userManagePage; + + @Test(groups={"functionTests"},dependsOnGroups = { "login","user" },description = "DeleteUserTest") + public void testDeleteUser() throws InterruptedException { + userManagePage = new UserManagePage(driver); + //assert user manage page + System.out.println("start delete user"); + assert userManagePage.deleteUser(); + System.out.println("end delete user"); + System.out.println("==================================="); + } +} + diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/project/CreateProjectTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/project/CreateProjectTest.java index 8abd09d37c..546792f0d8 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/project/CreateProjectTest.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/project/CreateProjectTest.java @@ -23,13 +23,16 @@ import org.testng.annotations.Test; public class CreateProjectTest extends BaseTest { private CreateProjectPage createProjectPage; - @Test(description = "TenantTest", priority = 1) - public void testUserManage() throws InterruptedException { + @Test(groups={"functionTests"},dependsOnGroups = { "login" },description = "CreateProjectTest",priority=4) + public void testCreateProject() throws InterruptedException { createProjectPage = new CreateProjectPage(driver); // enter user manage page createProjectPage.jumpProjectManagePage(); //assert user manage page + System.out.println("start create project"); assert createProjectPage.createProject(); + System.out.println("end create project"); + System.out.println("==================================="); } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/project/CreateWorkflowTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/project/CreateWorkflowTest.java index 6ac13f8124..ea87f48529 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/project/CreateWorkflowTest.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/project/CreateWorkflowTest.java @@ -26,12 +26,16 @@ public class CreateWorkflowTest extends BaseTest { private CreateProjectPage createProjectPage; - @Test(description = "CreateWorkflowTest", priority = 1) - public void CreateWorkflowTest() throws InterruptedException { + @Test(groups={"functionTests"},dependsOnGroups = { "login" },description = "CreateWorkflowTest",priority=5) + public void testCreateWorkflow() throws InterruptedException { createProjectPage = new CreateProjectPage(driver); createProjectPage.jumpProjectManagePage(); + createWorkflowPage = new CreateWorkflowPage(driver); - //assert create workflow + System.out.println("start create workflow"); assert createWorkflowPage.createWorkflow(); + assert createWorkflowPage.saveWorkflow(); + System.out.println("end create workflow"); + System.out.println("==================================="); } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/security/TenantManageTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/security/TenantManageTest.java index 7124b4e094..8b21d3bb61 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/security/TenantManageTest.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/security/TenantManageTest.java @@ -20,15 +20,20 @@ import org.apache.dolphinscheduler.base.BaseTest; import org.apache.dolphinscheduler.page.security.TenantManagePage; import org.testng.annotations.Test; + public class TenantManageTest extends BaseTest { private TenantManagePage tenantManagePage; - @Test(description = "TenantTest", priority = 1) + @Test(groups={"functionTests","createTenant"},dependsOnGroups = { "login" },description = "TenantManageTest") public void testTenantManage() throws InterruptedException { tenantManagePage = new TenantManagePage(driver); - // enter tenant manage page - tenantManagePage.jumpPage(); //assert tenant manage page + System.out.println("start create tenant"); assert tenantManagePage.createTenant(); + System.out.println("end create tenant"); + System.out.println("==================================="); + } + + } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/security/UserManageTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/security/UserManageTest.java index 834ebdbbf0..9381b68c75 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/security/UserManageTest.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/security/UserManageTest.java @@ -23,12 +23,14 @@ import org.testng.annotations.Test; public class UserManageTest extends BaseTest { private UserManagePage userManagePage; - @Test(description = "TenantTest", priority = 1) + @Test(groups={"functionTests","user"},dependsOnGroups = { "login" },description = "UserManageTest") public void testUserManage() throws InterruptedException { userManagePage = new UserManagePage(driver); - // enter user manage page - userManagePage.jumpPage(); //assert user manage page + System.out.println("start create user"); assert userManagePage.createUser(); + System.out.println("end create user"); + System.out.println("==================================="); + } } diff --git a/e2e/suite.xml b/e2e/suite.xml deleted file mode 100644 index d9d7ae6845..0000000000 --- a/e2e/suite.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/e2e/testng.xml b/e2e/testng.xml index 757ffab248..56c9649135 100644 --- a/e2e/testng.xml +++ b/e2e/testng.xml @@ -1,19 +1,23 @@ - - - - - - + + + + + + + + - + + + + + + + - - - - - + From 7789e05c20ed16422eb58971efbbc3dddf89f746 Mon Sep 17 00:00:00 2001 From: xingchun-chen <55787491+xingchun-chen@users.noreply.github.com> Date: Wed, 18 Mar 2020 10:03:03 +0800 Subject: [PATCH 59/59] add LoginTest license (#2217) Co-authored-by: chenxingchun <438044805@qq.com> --- .../dolphinscheduler/testcase/LoginTest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/LoginTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/LoginTest.java index 47ce11287d..024ffcdc8a 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/LoginTest.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/LoginTest.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.dolphinscheduler.testcase; import org.apache.dolphinscheduler.page.LoginPage;