From 5185d283f9bea4d0a345b256a080b8fa5ee913f6 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Tue, 14 Apr 2020 14:28:03 +0800 Subject: [PATCH 01/23] Change the name of the shell node and modify the parameter transmission method of spark, mr, python, and flink nodes (#2416) * Fix the problem of data echo in script edit box * Optimize resource tree * Change the name of the shell node and modify the parameter transmission method of spark, mr, python, and flink nodes Co-authored-by: dailidong --- .../dag/_source/formModel/tasks/flink.vue | 107 +++++++++++++++- .../pages/dag/_source/formModel/tasks/mr.vue | 118 +++++++++++++++-- .../dag/_source/formModel/tasks/python.vue | 119 ++++++++++++++++-- .../dag/_source/formModel/tasks/shell.vue | 4 +- .../dag/_source/formModel/tasks/spark.vue | 107 +++++++++++++++- .../src/js/module/i18n/locale/en_US.js | 2 +- .../src/js/module/i18n/locale/zh_CN.js | 2 +- 7 files changed, 425 insertions(+), 34 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue index 50866afc42..30b644dd69 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue @@ -211,7 +211,9 @@ return { label: node.name } - } + }, + allNoResources: [], + noRes: [], } }, props: { @@ -300,6 +302,12 @@ return false } + // noRes + if (this.noRes.length>0) { + this.$message.warning(`${i18n.$t('Please delete all non-existent resources')}`) + return false + } + // localParams Subcomponent verification if (!this.$refs.refLocalParams._verifProp()) { return false @@ -339,6 +347,67 @@ } delete item.children }, + searchTree(element, id) { + // 根据id查找节点 + if (element.id == id) { + return element; + } else if (element.children != null) { + var i; + var result = null; + for (i = 0; result == null && i < element.children.length; i++) { + result = this.searchTree(element.children[i], id); + } + return result; + } + return null; + }, + dataProcess(backResource) { + let isResourceId = [] + let resourceIdArr = [] + if(this.resourceList.length>0) { + this.resourceList.forEach(v=>{ + this.mainJarList.forEach(v1=>{ + if(this.searchTree(v1,v)) { + isResourceId.push(this.searchTree(v1,v)) + } + }) + }) + resourceIdArr = isResourceId.map(item=>{ + return item.id + }) + Array.prototype.diff = function(a) { + return this.filter(function(i) {return a.indexOf(i) < 0;}); + }; + let diffSet = this.resourceList.diff(resourceIdArr); + let optionsCmp = [] + if(diffSet.length>0) { + diffSet.forEach(item=>{ + backResource.forEach(item1=>{ + if(item==item1.id || item==item1.res) { + optionsCmp.push(item1) + } + }) + }) + } + let noResources = [{ + id: -1, + name: $t('No resources'), + fullName: '/'+$t('No resources'), + children: [] + }] + if(optionsCmp.length>0) { + this.allNoResources = optionsCmp + optionsCmp = optionsCmp.map(item=>{ + return {id: item.id,name: item.name,fullName: item.res} + }) + optionsCmp.forEach(item=>{ + item.isNew = true + }) + noResources[0].children = optionsCmp + this.mainJarList = this.mainJarList.concat(noResources) + } + } + }, }, watch: { // Listening type @@ -354,15 +423,37 @@ }, computed: { cacheParams () { + let isResourceId = [] + let resourceIdArr = [] + if(this.resourceList.length>0) { + this.resourceList.forEach(v=>{ + this.mainJarList.forEach(v1=>{ + if(this.searchTree(v1,v)) { + isResourceId.push(this.searchTree(v1,v)) + } + }) + }) + resourceIdArr = isResourceId.map(item=>{ + return {id: item.id,name: item.name,res: item.fullName} + }) + } + let result = [] + resourceIdArr.forEach(item=>{ + this.allNoResources.forEach(item1=>{ + if(item.id==item1.id) { + // resultBool = true + result.push(item1) + } + }) + }) + this.noRes = result return { mainClass: this.mainClass, mainJar: { id: this.mainJar }, deployMode: this.deployMode, - resourceList: _.map(this.resourceList, v => { - return {id: v} - }), + resourceList: resourceIdArr, localParams: this.localParams, slot: this.slot, taskManager: this.taskManager, @@ -404,20 +495,24 @@ this.programType = o.params.programType || 'SCALA' // backfill resourceList + let backResource = o.params.resourceList || [] let resourceList = o.params.resourceList || [] if (resourceList.length) { _.map(resourceList, v => { - if(v.res) { + if(!v.id) { this.store.dispatch('dag/getResourceId',{ type: 'FILE', fullName: '/'+v.res }).then(res => { this.resourceList.push(res.id) + this.dataProcess(backResource) }).catch(e => { - this.$message.error(e.msg || '') + this.resourceList.push(v.res) + this.dataProcess(backResource) }) } else { this.resourceList.push(v.id) + this.dataProcess(backResource) } }) this.cacheResourceList = resourceList diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue index 121147d7e5..1df1f93ba9 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue @@ -44,7 +44,7 @@
{{$t('Main jar package')}}
- +
{{ node.raw.fullName }}
@@ -109,6 +109,7 @@ name: 'mr', data () { return { + valueConsistsOf: 'LEAF_PRIORITY', // Main function class mainClass: '', // Master jar package @@ -134,7 +135,9 @@ return { label: node.name } - } + }, + allNoResources: [], + noRes: [] } }, props: { @@ -176,9 +179,76 @@ diGuiTree(item) { // Recursive convenience tree structure item.forEach(item => { item.children === '' || item.children === undefined || item.children === null || item.children.length === 0?　　　　　　　　 - delete item.children : this.diGuiTree(item.children); + this.operationTree(item) : this.diGuiTree(item.children); }) }, + operationTree(item) { + if(item.dirctory) { + item.isDisabled =true + } + delete item.children + }, + searchTree(element, id) { + // 根据id查找节点 + if (element.id == id) { + return element; + } else if (element.children != null) { + var i; + var result = null; + for (i = 0; result == null && i < element.children.length; i++) { + result = this.searchTree(element.children[i], id); + } + return result; + } + return null; + }, + dataProcess(backResource) { + let isResourceId = [] + let resourceIdArr = [] + if(this.resourceList.length>0) { + this.resourceList.forEach(v=>{ + this.mainJarList.forEach(v1=>{ + if(this.searchTree(v1,v)) { + isResourceId.push(this.searchTree(v1,v)) + } + }) + }) + resourceIdArr = isResourceId.map(item=>{ + return item.id + }) + Array.prototype.diff = function(a) { + return this.filter(function(i) {return a.indexOf(i) < 0;}); + }; + let diffSet = this.resourceList.diff(resourceIdArr); + let optionsCmp = [] + if(diffSet.length>0) { + diffSet.forEach(item=>{ + backResource.forEach(item1=>{ + if(item==item1.id || item==item1.res) { + optionsCmp.push(item1) + } + }) + }) + } + let noResources = [{ + id: -1, + name: $t('No resources'), + fullName: '/'+$t('No resources'), + children: [] + }] + if(optionsCmp.length>0) { + this.allNoResources = optionsCmp + optionsCmp = optionsCmp.map(item=>{ + return {id: item.id,name: item.name,fullName: item.res} + }) + optionsCmp.forEach(item=>{ + item.isNew = true + }) + noResources[0].children = optionsCmp + this.mainJarList = this.mainJarList.concat(noResources) + } + } + }, /** * verification */ @@ -193,6 +263,12 @@ return false } + // noRes + if (this.noRes.length>0) { + this.$message.warning(`${i18n.$t('Please delete all non-existent resources')}`) + return false + } + // localParams Subcomponent verification if (!this.$refs.refLocalParams._verifProp()) { return false @@ -231,14 +307,36 @@ }, computed: { cacheParams () { + let isResourceId = [] + let resourceIdArr = [] + if(this.resourceList.length>0) { + this.resourceList.forEach(v=>{ + this.mainJarList.forEach(v1=>{ + if(this.searchTree(v1,v)) { + isResourceId.push(this.searchTree(v1,v)) + } + }) + }) + resourceIdArr = isResourceId.map(item=>{ + return {id: item.id,name: item.name,res: item.fullName} + }) + } + let result = [] + resourceIdArr.forEach(item=>{ + this.allNoResources.forEach(item1=>{ + if(item.id==item1.id) { + // resultBool = true + result.push(item1) + } + }) + }) + this.noRes = result return { mainClass: this.mainClass, mainJar: { id: this.mainJar }, - resourceList: _.map(this.resourceList, v => { - return {id: v} - }), + resourceList: resourceIdArr, localParams: this.localParams, mainArgs: this.mainArgs, others: this.others, @@ -273,23 +371,27 @@ let resourceList = o.params.resourceList || [] if (resourceList.length) { _.map(resourceList, v => { - if(v.res) { + if(!v.id) { this.store.dispatch('dag/getResourceId',{ type: 'FILE', fullName: '/'+v.res }).then(res => { this.resourceList.push(res.id) + this.dataProcess(backResource) }).catch(e => { - this.$message.error(e.msg || '') + this.resourceList.push(v.res) + this.dataProcess(backResource) }) } else { this.resourceList.push(v.id) + this.dataProcess(backResource) } }) this.cacheResourceList = resourceList } // backfill localParams + let backResource = o.params.resourceList || [] let localParams = o.params.localParams || [] if (localParams.length) { this.localParams = localParams diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue index 67669b4654..3e2abcbb00 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue @@ -80,6 +80,13 @@ // Cache ResourceList cacheResourceList: [], resourceOptions: [], + normalizer(node) { + return { + label: node.name + } + }, + allNoResources: [], + noRes: [] } }, mixins: [disabledState], @@ -96,9 +103,9 @@ /** * return resourceList */ - _onResourcesData (a) { - this.resourceList = a - }, + // _onResourcesData (a) { + // this.resourceList = a + // }, /** * cache resourceList */ @@ -120,6 +127,12 @@ return false } + // noRes + if (this.noRes.length>0) { + this.$message.warning(`${i18n.$t('Please delete all non-existent resources')}`) + return false + } + // storage this.$emit('on-params', { resourceList: _.map(this.resourceList, v => { @@ -166,6 +179,67 @@ item.isDisabled =true } delete item.children + }, + searchTree(element, id) { + // 根据id查找节点 + if (element.id == id) { + return element; + } else if (element.children != null) { + var i; + var result = null; + for (i = 0; result == null && i < element.children.length; i++) { + result = this.searchTree(element.children[i], id); + } + return result; + } + return null; + }, + dataProcess(backResource) { + let isResourceId = [] + let resourceIdArr = [] + if(this.resourceList.length>0) { + this.resourceList.forEach(v=>{ + this.resourceOptions.forEach(v1=>{ + if(this.searchTree(v1,v)) { + isResourceId.push(this.searchTree(v1,v)) + } + }) + }) + resourceIdArr = isResourceId.map(item=>{ + return item.id + }) + Array.prototype.diff = function(a) { + return this.filter(function(i) {return a.indexOf(i) < 0;}); + }; + let diffSet = this.resourceList.diff(resourceIdArr); + let optionsCmp = [] + if(diffSet.length>0) { + diffSet.forEach(item=>{ + backResource.forEach(item1=>{ + if(item==item1.id || item==item1.res) { + optionsCmp.push(item1) + } + }) + }) + } + let noResources = [{ + id: -1, + name: $t('No resources'), + fullName: '/'+$t('No resources'), + children: [] + }] + if(optionsCmp.length>0) { + this.allNoResources = optionsCmp + optionsCmp = optionsCmp.map(item=>{ + return {id: item.id,name: item.name,fullName: item.res} + }) + optionsCmp.forEach(item=>{ + item.isNew = true + }) + noResources[0].children = optionsCmp + this.resourceOptions = this.resourceOptions.concat(noResources) + } + } } }, watch: { @@ -176,10 +250,32 @@ }, computed: { cacheParams () { + let isResourceId = [] + let resourceIdArr = [] + if(this.resourceList.length>0) { + this.resourceList.forEach(v=>{ + this.resourceOptions.forEach(v1=>{ + if(this.searchTree(v1,v)) { + isResourceId.push(this.searchTree(v1,v)) + } + }) + }) + resourceIdArr = isResourceId.map(item=>{ + return {id: item.id,name: item.name,res: item.fullName} + }) + } + let result = [] + resourceIdArr.forEach(item=>{ + this.allNoResources.forEach(item1=>{ + if(item.id==item1.id) { + // resultBool = true + result.push(item1) + } + }) + }) + this.noRes = result return { - resourceList: _.map(this.resourceList, v => { - return {id: v} - }), + resourceList: resourceIdArr, localParams: this.localParams } } @@ -187,7 +283,7 @@ created () { let item = this.store.state.dag.resourcesListS this.diGuiTree(item) - this.options = item + this.resourceOptions = item let o = this.backfillItem // Non-null objects represent backfill @@ -195,17 +291,20 @@ this.rawScript = o.params.rawScript || '' // backfill resourceList + let backResource = o.params.resourceList || [] let resourceList = o.params.resourceList || [] if (resourceList.length) { _.map(resourceList, v => { - if(v.res) { + if(!v.id) { this.store.dispatch('dag/getResourceId',{ type: 'FILE', fullName: '/'+v.res }).then(res => { this.resourceList.push(res.id) + this.dataProcess(backResource) }).catch(e => { - this.$message.error(e.msg || '') + this.resourceList.push(v.res) + this.dataProcess(backResource) }) } else { this.resourceList.push(v.id) @@ -230,6 +329,6 @@ editor.toTextArea() // Uninstall editor.off($('.code-python-mirror'), 'keypress', this.keypress) }, - components: { mLocalParams, mListBox, mResources } + components: { mLocalParams, mListBox, mResources,Treeselect } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue index df315a14f1..b627602e04 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue @@ -266,8 +266,8 @@ } let noResources = [{ id: -1, - name: $t('No resources'), - fullName: '/'+$t('No resources'), + name: $t('Unauthorized or deleted resources'), + fullName: '/'+$t('Unauthorized or deleted resources'), children: [] }] if(optionsCmp.length>0) { diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue index 7a00528149..2304ab1d20 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue @@ -254,7 +254,9 @@ return { label: node.name } - } + }, + allNoResources: [], + noRes: [] } }, props: { @@ -305,6 +307,67 @@ } delete item.children }, + searchTree(element, id) { + // 根据id查找节点 + if (element.id == id) { + return element; + } else if (element.children != null) { + var i; + var result = null; + for (i = 0; result == null && i < element.children.length; i++) { + result = this.searchTree(element.children[i], id); + } + return result; + } + return null; + }, + dataProcess(backResource) { + let isResourceId = [] + let resourceIdArr = [] + if(this.resourceList.length>0) { + this.resourceList.forEach(v=>{ + this.mainJarList.forEach(v1=>{ + if(this.searchTree(v1,v)) { + isResourceId.push(this.searchTree(v1,v)) + } + }) + }) + resourceIdArr = isResourceId.map(item=>{ + return item.id + }) + Array.prototype.diff = function(a) { + return this.filter(function(i) {return a.indexOf(i) < 0;}); + }; + let diffSet = this.resourceList.diff(resourceIdArr); + let optionsCmp = [] + if(diffSet.length>0) { + diffSet.forEach(item=>{ + backResource.forEach(item1=>{ + if(item==item1.id || item==item1.res) { + optionsCmp.push(item1) + } + }) + }) + } + let noResources = [{ + id: -1, + name: $t('No resources'), + fullName: '/'+$t('No resources'), + children: [] + }] + if(optionsCmp.length>0) { + this.allNoResources = optionsCmp + optionsCmp = optionsCmp.map(item=>{ + return {id: item.id,name: item.name,fullName: item.res} + }) + optionsCmp.forEach(item=>{ + item.isNew = true + }) + noResources[0].children = optionsCmp + this.mainJarList = this.mainJarList.concat(noResources) + } + } + }, /** * verification */ @@ -324,6 +387,12 @@ return false } + // noRes + if (this.noRes.length>0) { + this.$message.warning(`${i18n.$t('Please delete all non-existent resources')}`) + return false + } + if (!Number.isInteger(parseInt(this.numExecutors))) { this.$message.warning(`${i18n.$t('The number of Executors should be a positive integer')}`) return false @@ -400,15 +469,37 @@ }, computed: { cacheParams () { + let isResourceId = [] + let resourceIdArr = [] + if(this.resourceList.length>0) { + this.resourceList.forEach(v=>{ + this.mainJarList.forEach(v1=>{ + if(this.searchTree(v1,v)) { + isResourceId.push(this.searchTree(v1,v)) + } + }) + }) + resourceIdArr = isResourceId.map(item=>{ + return {id: item.id,name: item.name,res: item.fullName} + }) + } + let result = [] + resourceIdArr.forEach(item=>{ + this.allNoResources.forEach(item1=>{ + if(item.id==item1.id) { + // resultBool = true + result.push(item1) + } + }) + }) + this.noRes = result return { mainClass: this.mainClass, mainJar: { id: this.mainJar }, deployMode: this.deployMode, - resourceList: _.map(this.resourceList, v => { - return {id: v} - }), + resourceList: resourceIdArr, localParams: this.localParams, driverCores: this.driverCores, driverMemory: this.driverMemory, @@ -453,20 +544,24 @@ this.sparkVersion = o.params.sparkVersion || 'SPARK2' // backfill resourceList + let backResource = o.params.resourceList || [] let resourceList = o.params.resourceList || [] if (resourceList.length) { _.map(resourceList, v => { - if(v.res) { + if(!v.id) { this.store.dispatch('dag/getResourceId',{ type: 'FILE', fullName: '/'+v.res }).then(res => { this.resourceList.push(res.id) + this.dataProcess(backResource) }).catch(e => { - this.$message.error(e.msg || '') + this.resourceList.push(v.res) + this.dataProcess(backResource) }) } else { this.resourceList.push(v.id) + this.dataProcess(backResource) } }) this.cacheResourceList = resourceList diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js index 53dce962d8..db19e1fc0e 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js @@ -588,6 +588,6 @@ export default { 'Branch flow': 'Branch flow', 'Cannot select the same node for successful branch flow and failed branch flow': 'Cannot select the same node for successful branch flow and failed branch flow', 'Successful branch flow and failed branch flow are required': 'Successful branch flow and failed branch flow are required', - 'No resources': 'No resources', + 'Unauthorized or deleted resources': 'Unauthorized or deleted resources', 'Please delete all non-existent resources': 'Please delete all non-existent resources', } diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js index 5e0880ee38..80b45c7a11 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -588,6 +588,6 @@ export default { 'Branch flow': '分支流转', 'Cannot select the same node for successful branch flow and failed branch flow': '成功分支流转和失败分支流转不能选择同一个节点', 'Successful branch flow and failed branch flow are required': '成功分支流转和失败分支流转必填', - 'No resources': '未授权或已删除资源', + 'Unauthorized or deleted resources': '未授权或已删除资源', 'Please delete all non-existent resources': '请删除所有未授权或已删除资源', } From c881c9517e4348acc53c315ea6767d5c6d86f48d Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Wed, 15 Apr 2020 10:22:02 +0800 Subject: [PATCH 02/23] It's to remove `static` of method dataSource,If not remove `static` the transaction will not work. (#2422) * It's to remove `static` of method dataSource,If not remove `static` the transaction will not work. * update testQueryDetailsById because it didn't run success --- .../datasource/SpringConnectionFactory.java | 2 +- .../dao/mapper/UserMapperTest.java | 29 +++++++++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java index 1d6dc5f51a..9e27d949aa 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java @@ -64,7 +64,7 @@ public class SpringConnectionFactory { * @return druid dataSource */ @Bean(destroyMethod="") - public static DruidDataSource dataSource() { + public DruidDataSource dataSource() { DruidDataSource druidDataSource = new DruidDataSource(); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UserMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UserMapperTest.java index 651ca93f27..7b1849ef4d 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UserMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UserMapperTest.java @@ -16,11 +16,11 @@ */ package org.apache.dolphinscheduler.dao.mapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.utils.DateUtils; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.dao.entity.*; import org.junit.Assert; import org.junit.Test; @@ -179,6 +179,23 @@ public class UserMapperTest { return tenant; } + /** + * insert one Tenant + * @return Tenant + */ + private Tenant insertOneTenant(Queue queue){ + Tenant tenant = new Tenant(); + tenant.setTenantCode("dolphin"); + tenant.setTenantName("dolphin test"); + tenant.setDescription("dolphin user use"); + tenant.setQueueId(queue.getId()); + tenant.setQueue(queue.getQueue()); + tenant.setCreateTime(new Date()); + tenant.setUpdateTime(new Date()); + tenantMapper.insert(tenant); + return tenant; + } + /** * insert one Queue * @return Queue @@ -291,11 +308,13 @@ public class UserMapperTest { */ @Test public void testQueryDetailsById() { - //insertOne - User user = insertOne(); + //insertOneQueue and insertOneTenant + Queue queue = insertOneQueue(); + Tenant tenant = insertOneTenant(queue); + User user = insertOne(queue,tenant); //queryDetailsById User queryUser = userMapper.queryDetailsById(user.getId()); - Assert.assertEquals(queryUser.getUserName(), queryUser.getUserName()); + Assert.assertEquals(user.getUserName(), queryUser.getUserName()); } /** From 0bad7c2f539ef4aea8956610146cd70a7b8267ee Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Wed, 15 Apr 2020 13:50:06 +0800 Subject: [PATCH 03/23] add worker_group field and remove worker_group_id field of the table t_ds_command and t_ds_error_command (#2428) --- .../mysql/dolphinscheduler_ddl.sql | 79 +++++++++++++++++ .../postgresql/dolphinscheduler_ddl.sql | 84 +++++++++++++++++++ 2 files changed, 163 insertions(+) diff --git a/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql b/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql index 6a8665f199..33d801d4ec 100644 --- a/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql @@ -277,4 +277,83 @@ delimiter ; CALL dc_dolphin_T_t_ds_schedules_D_worker_group_id; DROP PROCEDURE dc_dolphin_T_t_ds_schedules_D_worker_group_id; +-- ac_dolphin_T_t_ds_command_A_worker_group +drop PROCEDURE if EXISTS ac_dolphin_T_t_ds_command_A_worker_group; +delimiter d// +CREATE PROCEDURE ac_dolphin_T_t_ds_command_A_worker_group() + BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_command' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='worker_group') + THEN + ALTER TABLE t_ds_command ADD `worker_group` varchar(255) DEFAULT '' COMMENT 'worker group'; + END IF; + END; + +d// + +delimiter ; +CALL ac_dolphin_T_t_ds_command_A_worker_group; +DROP PROCEDURE ac_dolphin_T_t_ds_command_A_worker_group; + +-- dc_dolphin_T_t_ds_command_D_worker_group_id +drop PROCEDURE if EXISTS dc_dolphin_T_t_ds_command_D_worker_group_id; +delimiter d// +CREATE PROCEDURE dc_dolphin_T_t_ds_command_D_worker_group_id() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_command' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='worker_group_id') + THEN + ALTER TABLE t_ds_command DROP COLUMN worker_group_id; + END IF; + END; + +d// + +delimiter ; +CALL dc_dolphin_T_t_ds_command_D_worker_group_id; +DROP PROCEDURE dc_dolphin_T_t_ds_command_D_worker_group_id; + +-- ac_dolphin_T_t_ds_error_command_A_worker_group +drop PROCEDURE if EXISTS ac_dolphin_T_t_ds_error_command_A_worker_group; +delimiter d// +CREATE PROCEDURE ac_dolphin_T_t_ds_error_command_A_worker_group() + BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_error_command' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='worker_group') + THEN + ALTER TABLE t_ds_error_command ADD `worker_group` varchar(255) DEFAULT '' COMMENT 'worker group'; + END IF; + END; + +d// + +delimiter ; +CALL ac_dolphin_T_t_ds_error_command_A_worker_group; +DROP PROCEDURE ac_dolphin_T_t_ds_error_command_A_worker_group; + +-- dc_dolphin_T_t_ds_error_command_D_worker_group_id +drop PROCEDURE if EXISTS dc_dolphin_T_t_ds_error_command_D_worker_group_id; +delimiter d// +CREATE PROCEDURE dc_dolphin_T_t_ds_error_command_D_worker_group_id() + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_error_command' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='worker_group_id') + THEN + ALTER TABLE t_ds_error_command DROP COLUMN worker_group_id; + END IF; + END; + +d// + +delimiter ; +CALL dc_dolphin_T_t_ds_error_command_D_worker_group_id; +DROP PROCEDURE dc_dolphin_T_t_ds_error_command_D_worker_group_id; diff --git a/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql b/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql index dd332533a6..a4fdc2d6bf 100644 --- a/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql @@ -279,4 +279,88 @@ delimiter ; select dc_dolphin_T_t_ds_schedules_D_worker_group_id(); DROP FUNCTION dc_dolphin_T_t_ds_schedules_D_worker_group_id(); +-- ac_dolphin_T_t_ds_command_A_worker_group +delimiter ; +DROP FUNCTION IF EXISTS ac_dolphin_T_t_ds_command_A_worker_group(); +delimiter d// +CREATE FUNCTION ac_dolphin_T_t_ds_command_A_worker_group() RETURNS void AS $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_CATALOG=current_database() + AND TABLE_SCHEMA=current_schema() + AND TABLE_NAME='t_ds_command' + AND COLUMN_NAME ='worker_group') + THEN + ALTER TABLE t_ds_command ADD COLUMN worker_group varchar(255) DEFAULT null; + END IF; +END; +$$ LANGUAGE plpgsql; +d// +delimiter ; +select ac_dolphin_T_t_ds_command_A_worker_group(); +DROP FUNCTION ac_dolphin_T_t_ds_command_A_worker_group(); + +-- dc_dolphin_T_t_ds_command_D_worker_group_id +delimiter ; +DROP FUNCTION IF EXISTS dc_dolphin_T_t_ds_command_D_worker_group_id(); +delimiter d// +CREATE FUNCTION dc_dolphin_T_t_ds_command_D_worker_group_id() RETURNS void AS $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_CATALOG=current_database() + AND TABLE_SCHEMA=current_schema() + AND TABLE_NAME='t_ds_command' + AND COLUMN_NAME ='worker_group_id') + THEN + ALTER TABLE t_ds_command DROP COLUMN worker_group_id; + END IF; +END; +$$ LANGUAGE plpgsql; +d// +delimiter ; +select dc_dolphin_T_t_ds_command_D_worker_group_id(); +DROP FUNCTION dc_dolphin_T_t_ds_command_D_worker_group_id(); + +-- ac_dolphin_T_t_ds_error_command_A_worker_group +delimiter ; +DROP FUNCTION IF EXISTS ac_dolphin_T_t_ds_error_command_A_worker_group(); +delimiter d// +CREATE FUNCTION ac_dolphin_T_t_ds_error_command_A_worker_group() RETURNS void AS $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_CATALOG=current_database() + AND TABLE_SCHEMA=current_schema() + AND TABLE_NAME='t_ds_error_command' + AND COLUMN_NAME ='worker_group') + THEN + ALTER TABLE t_ds_error_command ADD COLUMN worker_group varchar(255) DEFAULT null; + END IF; +END; +$$ LANGUAGE plpgsql; +d// +delimiter ; +select ac_dolphin_T_t_ds_error_command_A_worker_group(); +DROP FUNCTION ac_dolphin_T_t_ds_error_command_A_worker_group(); + +-- dc_dolphin_T_t_ds_error_command_D_worker_group_id +delimiter ; +DROP FUNCTION IF EXISTS dc_dolphin_T_t_ds_error_command_D_worker_group_id(); +delimiter d// +CREATE FUNCTION dc_dolphin_T_t_ds_error_command_D_worker_group_id() RETURNS void AS $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_CATALOG=current_database() + AND TABLE_SCHEMA=current_schema() + AND TABLE_NAME='t_ds_error_command' + AND COLUMN_NAME ='worker_group_id') + THEN + ALTER TABLE t_ds_error_command DROP COLUMN worker_group_id; + END IF; +END; +$$ LANGUAGE plpgsql; +d// +delimiter ; +select dc_dolphin_T_t_ds_error_command_D_worker_group_id(); +DROP FUNCTION dc_dolphin_T_t_ds_error_command_D_worker_group_id(); + From 96835ebda2d4f9c5aa13b39bdd1e90f5d702b2f6 Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Wed, 15 Apr 2020 16:19:42 +0800 Subject: [PATCH 04/23] =?UTF-8?q?1=EF=BC=8Cno=20worker=20condition=20,=20m?= =?UTF-8?q?aster=20will=20while=20ture=20wait=20for=20worker=20startup=202?= =?UTF-8?q?=EF=BC=8Cworker=20response=20task=20status=20sync=20wait=20for?= =?UTF-8?q?=20result=20(#2420)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * dispatch task fail will set task status failed * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result Co-authored-by: qiaozhanwei --- .../consumer/TaskPriorityQueueConsumer.java | 27 +++-- .../master/processor/TaskAckProcessor.java | 28 +++++- .../processor/TaskResponseProcessor.java | 25 +++++ .../worker/processor/TaskCallbackService.java | 15 ++- .../processor/TaskCallbackServiceTest.java | 98 +++++++++++++++++-- 5 files changed, 176 insertions(+), 17 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java index 50c851c483..b2cf53a575 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java @@ -53,6 +53,8 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.apache.dolphinscheduler.common.Constants.*; + /** * TaskUpdateQueue consumer */ @@ -68,7 +70,7 @@ public class TaskPriorityQueueConsumer extends Thread{ * taskUpdateQueue */ @Autowired - private TaskPriorityQueue taskUpdateQueue; + private TaskPriorityQueue taskPriorityQueue; /** * processService @@ -93,7 +95,7 @@ public class TaskPriorityQueueConsumer extends Thread{ while (Stopper.isRunning()){ try { // if not task , blocking here - String taskPriorityInfo = taskUpdateQueue.take(); + String taskPriorityInfo = taskPriorityQueue.take(); TaskPriority taskPriority = TaskPriority.of(taskPriorityInfo); @@ -114,13 +116,22 @@ public class TaskPriorityQueueConsumer extends Thread{ private Boolean dispatch(int taskInstanceId){ TaskExecutionContext context = getTaskExecutionContext(taskInstanceId); ExecutionContext executionContext = new ExecutionContext(context.toCommand(), ExecutorType.WORKER, context.getWorkerGroup()); - try { - return dispatcher.dispatch(executionContext); - } catch (ExecuteException e) { - logger.error("execute exception", e); - return false; - } + Boolean result = false; + while (Stopper.isRunning()){ + try { + result = dispatcher.dispatch(executionContext); + } catch (ExecuteException e) { + logger.error("dispatch error",e); + try { + Thread.sleep(SLEEP_TIME_MILLIS); + } catch (InterruptedException e1) {} + } + if (result){ + break; + } + } + return result; } /** diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java index 1eb40db152..7af9cdc2cc 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java @@ -19,7 +19,9 @@ package org.apache.dolphinscheduler.server.master.processor; import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.utils.Preconditions; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; @@ -31,9 +33,12 @@ import org.apache.dolphinscheduler.server.master.cache.impl.TaskInstanceCacheMan import org.apache.dolphinscheduler.server.master.processor.queue.TaskResponseEvent; import org.apache.dolphinscheduler.server.master.processor.queue.TaskResponseService; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.dolphinscheduler.common.Constants.*; + /** * task ack processor */ @@ -51,9 +56,16 @@ public class TaskAckProcessor implements NettyRequestProcessor { */ private final TaskInstanceCacheManager taskInstanceCacheManager; + + /** + * processService + */ + private ProcessService processService; + public TaskAckProcessor(){ this.taskResponseService = SpringApplicationContext.getBean(TaskResponseService.class); this.taskInstanceCacheManager = SpringApplicationContext.getBean(TaskInstanceCacheManagerImpl.class); + this.processService = SpringApplicationContext.getBean(ProcessService.class); } /** @@ -71,8 +83,10 @@ public class TaskAckProcessor implements NettyRequestProcessor { String workerAddress = ChannelUtils.toAddress(channel).getAddress(); + ExecutionStatus ackStatus = ExecutionStatus.of(taskAckCommand.getStatus()); + // TaskResponseEvent - TaskResponseEvent taskResponseEvent = TaskResponseEvent.newAck(ExecutionStatus.of(taskAckCommand.getStatus()), + TaskResponseEvent taskResponseEvent = TaskResponseEvent.newAck(ackStatus, taskAckCommand.getStartTime(), workerAddress, taskAckCommand.getExecutePath(), @@ -81,6 +95,18 @@ public class TaskAckProcessor implements NettyRequestProcessor { taskResponseService.addResponse(taskResponseEvent); + while (Stopper.isRunning()){ + TaskInstance taskInstance = processService.findTaskInstanceById(taskAckCommand.getTaskInstanceId()); + + if (taskInstance != null && ackStatus.typeIsRunning()){ + break; + } + + try { + Thread.sleep(SLEEP_TIME_MILLIS); + } catch (InterruptedException e) {} + } + } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java index 36b382313b..ecb8646ad0 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java @@ -19,7 +19,9 @@ package org.apache.dolphinscheduler.server.master.processor; import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.utils.Preconditions; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand; @@ -30,9 +32,12 @@ import org.apache.dolphinscheduler.server.master.cache.impl.TaskInstanceCacheMan import org.apache.dolphinscheduler.server.master.processor.queue.TaskResponseEvent; import org.apache.dolphinscheduler.server.master.processor.queue.TaskResponseService; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.dolphinscheduler.common.Constants.*; + /** * task response processor */ @@ -50,9 +55,15 @@ public class TaskResponseProcessor implements NettyRequestProcessor { */ private final TaskInstanceCacheManager taskInstanceCacheManager; + /** + * processService + */ + private ProcessService processService; + public TaskResponseProcessor(){ this.taskResponseService = SpringApplicationContext.getBean(TaskResponseService.class); this.taskInstanceCacheManager = SpringApplicationContext.getBean(TaskInstanceCacheManagerImpl.class); + this.processService = SpringApplicationContext.getBean(ProcessService.class); } /** @@ -71,6 +82,8 @@ public class TaskResponseProcessor implements NettyRequestProcessor { taskInstanceCacheManager.cacheTaskInstance(responseCommand); + ExecutionStatus responseStatus = ExecutionStatus.of(responseCommand.getStatus()); + // TaskResponseEvent TaskResponseEvent taskResponseEvent = TaskResponseEvent.newResult(ExecutionStatus.of(responseCommand.getStatus()), responseCommand.getEndTime(), @@ -79,6 +92,18 @@ public class TaskResponseProcessor implements NettyRequestProcessor { responseCommand.getTaskInstanceId()); taskResponseService.addResponse(taskResponseEvent); + + while (Stopper.isRunning()){ + TaskInstance taskInstance = processService.findTaskInstanceById(taskResponseEvent.getTaskInstanceId()); + + if (taskInstance != null && responseStatus.typeIsFinished()){ + break; + } + + try { + Thread.sleep(SLEEP_TIME_MILLIS); + } catch (InterruptedException e) {} + } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java index f966591df4..7cd25cba65 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java @@ -18,9 +18,11 @@ package org.apache.dolphinscheduler.server.worker.processor; +import com.alibaba.fastjson.JSONObject; import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelFutureListener; +import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.remote.NettyRemotingClient; import org.apache.dolphinscheduler.remote.command.Command; @@ -93,8 +95,17 @@ public class TaskCallbackService { } logger.warn("original master : {} is not reachable, random select master", nettyRemoteChannel.getHost()); Set masterNodes = zookeeperRegistryCenter.getMasterNodesDirectly(); - if(CollectionUtils.isEmpty(masterNodes)){ - throw new IllegalStateException("no available master node exception"); + while (Stopper.isRunning()) { + if (CollectionUtils.isEmpty(masterNodes)) { + logger.error("no available master node"); + try { + Thread.sleep(1000); + }catch (Exception e){ + + } + }else { + break; + } } for(String masterNode : masterNodes){ newChannel = nettyRemotingClient.getChannel(Host.of(masterNode)); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTest.java index 5f44e1cee2..a0fee7c36e 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTest.java @@ -17,21 +17,26 @@ package org.apache.dolphinscheduler.server.worker.processor; import io.netty.channel.Channel; +import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.remote.NettyRemotingClient; import org.apache.dolphinscheduler.remote.NettyRemotingServer; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; +import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand; import org.apache.dolphinscheduler.remote.config.NettyClientConfig; import org.apache.dolphinscheduler.remote.config.NettyServerConfig; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.processor.TaskAckProcessor; +import org.apache.dolphinscheduler.server.master.processor.TaskResponseProcessor; +import org.apache.dolphinscheduler.server.master.processor.queue.TaskResponseService; import org.apache.dolphinscheduler.server.master.registry.MasterRegistry; import org.apache.dolphinscheduler.server.registry.ZookeeperNodeManager; import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistry; import org.apache.dolphinscheduler.server.zk.SpringZKServer; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; import org.apache.dolphinscheduler.service.zk.ZookeeperConfig; import org.junit.Test; @@ -47,9 +52,10 @@ import java.util.Date; * test task call back service */ @RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes={TaskCallbackServiceTestConfig.class, SpringZKServer.class, MasterRegistry.class, WorkerRegistry.class, +@ContextConfiguration(classes={TaskCallbackServiceTestConfig.class, SpringZKServer.class, SpringApplicationContext.class, MasterRegistry.class, WorkerRegistry.class, ZookeeperRegistryCenter.class, MasterConfig.class, WorkerConfig.class, - ZookeeperCachedOperator.class, ZookeeperConfig.class, ZookeeperNodeManager.class, TaskCallbackService.class}) + ZookeeperCachedOperator.class, ZookeeperConfig.class, ZookeeperNodeManager.class, TaskCallbackService.class, + TaskResponseService.class, TaskAckProcessor.class,TaskResponseProcessor.class}) public class TaskCallbackServiceTest { @Autowired @@ -58,12 +64,22 @@ public class TaskCallbackServiceTest { @Autowired private MasterRegistry masterRegistry; + @Autowired + private TaskAckProcessor taskAckProcessor; + + @Autowired + private TaskResponseProcessor taskResponseProcessor; + + /** + * send ack test + * @throws Exception + */ @Test - public void testSendAck(){ + public void testSendAck() throws Exception{ final NettyServerConfig serverConfig = new NettyServerConfig(); serverConfig.setListenPort(30000); NettyRemotingServer nettyRemotingServer = new NettyRemotingServer(serverConfig); - nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_ACK, Mockito.mock(TaskAckProcessor.class)); + nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_ACK, taskAckProcessor); nettyRemotingServer.start(); final NettyClientConfig clientConfig = new NettyClientConfig(); @@ -75,22 +91,64 @@ public class TaskCallbackServiceTest { ackCommand.setStartTime(new Date()); taskCallbackService.sendAck(1, ackCommand.convert2Command()); + Thread.sleep(5000); + + Stopper.stop(); + + Thread.sleep(5000); + nettyRemotingServer.close(); nettyRemotingClient.close(); } + /** + * send result test + * @throws Exception + */ + @Test + public void testSendResult() throws Exception{ + final NettyServerConfig serverConfig = new NettyServerConfig(); + serverConfig.setListenPort(30000); + NettyRemotingServer nettyRemotingServer = new NettyRemotingServer(serverConfig); + nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_RESPONSE, taskResponseProcessor); + nettyRemotingServer.start(); + + final NettyClientConfig clientConfig = new NettyClientConfig(); + NettyRemotingClient nettyRemotingClient = new NettyRemotingClient(clientConfig); + Channel channel = nettyRemotingClient.getChannel(Host.of("localhost:30000")); + taskCallbackService.addRemoteChannel(1, new NettyRemoteChannel(channel, 1)); + TaskExecuteResponseCommand responseCommand = new TaskExecuteResponseCommand(); + responseCommand.setTaskInstanceId(1); + responseCommand.setEndTime(new Date()); + + taskCallbackService.sendResult(1, responseCommand.convert2Command()); + + Thread.sleep(5000); + + Stopper.stop(); + + Thread.sleep(5000); + + nettyRemotingServer.close(); + nettyRemotingClient.close(); + } + + + @Test(expected = IllegalArgumentException.class) public void testSendAckWithIllegalArgumentException(){ TaskExecuteAckCommand ackCommand = Mockito.mock(TaskExecuteAckCommand.class); taskCallbackService.sendAck(1, ackCommand.convert2Command()); + Stopper.stop(); } @Test(expected = IllegalStateException.class) public void testSendAckWithIllegalStateException1(){ + masterRegistry.registry(); final NettyServerConfig serverConfig = new NettyServerConfig(); serverConfig.setListenPort(30000); NettyRemotingServer nettyRemotingServer = new NettyRemotingServer(serverConfig); - nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_ACK, Mockito.mock(TaskAckProcessor.class)); + nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_ACK, taskAckProcessor); nettyRemotingServer.start(); final NettyClientConfig clientConfig = new NettyClientConfig(); @@ -103,7 +161,21 @@ public class TaskCallbackServiceTest { ackCommand.setStartTime(new Date()); nettyRemotingServer.close(); + taskCallbackService.sendAck(1, ackCommand.convert2Command()); + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + Stopper.stop(); + + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + e.printStackTrace(); + } } @Test(expected = IllegalStateException.class) @@ -112,7 +184,7 @@ public class TaskCallbackServiceTest { final NettyServerConfig serverConfig = new NettyServerConfig(); serverConfig.setListenPort(30000); NettyRemotingServer nettyRemotingServer = new NettyRemotingServer(serverConfig); - nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_ACK, Mockito.mock(TaskAckProcessor.class)); + nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_ACK, taskAckProcessor); nettyRemotingServer.start(); final NettyClientConfig clientConfig = new NettyClientConfig(); @@ -125,6 +197,20 @@ public class TaskCallbackServiceTest { ackCommand.setStartTime(new Date()); nettyRemotingServer.close(); + taskCallbackService.sendAck(1, ackCommand.convert2Command()); + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + Stopper.stop(); + + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + e.printStackTrace(); + } } } From c4acbdcfd0159df3c3336ff9b491b9fc7387116f Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Wed, 15 Apr 2020 17:00:31 +0800 Subject: [PATCH 05/23] Repair naming (#2426) * Fix the problem of data echo in script edit box * Optimize resource tree * Change the name of the shell node and modify the parameter transmission method of spark, mr, python, and flink nodes * Repair naming * Modify list style Co-authored-by: dailidong Co-authored-by: qiaozhanwei --- .../conf/home/pages/dag/_source/formModel/tasks/flink.vue | 4 ++-- .../js/conf/home/pages/dag/_source/formModel/tasks/mr.vue | 4 ++-- .../conf/home/pages/dag/_source/formModel/tasks/python.vue | 5 +++-- .../conf/home/pages/dag/_source/formModel/tasks/spark.vue | 4 ++-- .../projects/pages/instance/pages/list/_source/list.vue | 4 ++-- .../home/pages/projects/pages/taskInstance/_source/list.vue | 6 +++--- 6 files changed, 14 insertions(+), 13 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue index 30b644dd69..195e3c64f9 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue @@ -391,8 +391,8 @@ } let noResources = [{ id: -1, - name: $t('No resources'), - fullName: '/'+$t('No resources'), + name: $t('Unauthorized or deleted resources'), + fullName: '/'+$t('Unauthorized or deleted resources'), children: [] }] if(optionsCmp.length>0) { diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue index 1df1f93ba9..112e47dc4f 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue @@ -232,8 +232,8 @@ } let noResources = [{ id: -1, - name: $t('No resources'), - fullName: '/'+$t('No resources'), + name: $t('Unauthorized or deleted resources'), + fullName: '/'+$t('Unauthorized or deleted resources'), children: [] }] if(optionsCmp.length>0) { diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue index 3e2abcbb00..dd7ea942dd 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue @@ -224,8 +224,8 @@ } let noResources = [{ id: -1, - name: $t('No resources'), - fullName: '/'+$t('No resources'), + name: $t('Unauthorized or deleted resources'), + fullName: '/'+$t('Unauthorized or deleted resources'), children: [] }] if(optionsCmp.length>0) { @@ -308,6 +308,7 @@ }) } else { this.resourceList.push(v.id) + this.dataProcess(backResource) } }) this.cacheResourceList = resourceList diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue index 2304ab1d20..628d58ab0c 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue @@ -351,8 +351,8 @@ } let noResources = [{ id: -1, - name: $t('No resources'), - fullName: '/'+$t('No resources'), + name: $t('Unauthorized or deleted resources'), + fullName: '/'+$t('Unauthorized or deleted resources'), children: [] }] if(optionsCmp.length>0) { diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue index 513b8ec6dd..641c7aaa63 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue @@ -49,10 +49,10 @@ {{$t('Run Times')}} - + {{$t('host')}} - + {{$t('fault-tolerant sign')}} diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue index f7be553568..f014874175 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/_source/list.vue @@ -43,13 +43,13 @@ {{$t('Start Time')}} - + {{$t('End Time')}} - + {{$t('host')}} - + {{$t('Duration')}}(s) From f21837076e0647103fc901e604013d0e21139460 Mon Sep 17 00:00:00 2001 From: zixi0825 <649790970@qq.com> Date: Wed, 15 Apr 2020 17:39:12 +0800 Subject: [PATCH 06/23] Solve the failure to execute non-query sql (#2413) Co-authored-by: sunchaohe Co-authored-by: qiaozhanwei Co-authored-by: lgcareer <18610854716@163.com> --- .../dolphinscheduler/server/worker/task/sql/SqlTask.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index 61d93259df..22fa91dc1d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -214,11 +214,8 @@ public class SqlTask extends AbstractTask { try { // if upload resource is HDFS and kerberos startup CommonUtils.loadKerberosConf(); - - // create connection connection = createConnection(); - // create temp function if (CollectionUtils.isNotEmpty(createFuncs)) { createTempFunction(connection,createFuncs); @@ -226,13 +223,12 @@ public class SqlTask extends AbstractTask { // pre sql preSql(connection,preStatementsBinds); - - stmt = prepareStatementAndBind(connection, mainSqlBinds); - resultSet = stmt.executeQuery(); + // decide whether to executeQuery or executeUpdate based on sqlType if (sqlParameters.getSqlType() == SqlType.QUERY.ordinal()) { // query statements need to be convert to JsonArray and inserted into Alert to send + resultSet = stmt.executeQuery(); resultProcess(resultSet); } else if (sqlParameters.getSqlType() == SqlType.NON_QUERY.ordinal()) { From 347f05f08c3203c9e0890b244134196c79a27b9c Mon Sep 17 00:00:00 2001 From: xingchun-chen <55787491+xingchun-chen@users.noreply.github.com> Date: Wed, 15 Apr 2020 18:30:31 +0800 Subject: [PATCH 07/23] Update worker_group_id to worker_group (#2433) * add LoginTest license * Delete useless packages * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * Update worker_group_id to worker_group in init.sql * Update worker_group_id to worker_group in init.sql * Update worker_group_id to worker_group Co-authored-by: chenxingchun <438044805@qq.com> --- .../docker-entrypoint-initdb/init.sql | 6 ++-- .../dao/entity/ErrorCommand.java | 35 +++++++++---------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/docker/postgres/docker-entrypoint-initdb/init.sql b/docker/postgres/docker-entrypoint-initdb/init.sql index b48ddde042..b26520e29c 100755 --- a/docker/postgres/docker-entrypoint-initdb/init.sql +++ b/docker/postgres/docker-entrypoint-initdb/init.sql @@ -234,7 +234,7 @@ CREATE TABLE t_ds_command ( dependence varchar(255) DEFAULT NULL , update_time timestamp DEFAULT NULL , process_instance_priority int DEFAULT NULL , - worker_group_id int DEFAULT '-1' , + worker_group varchar(64), PRIMARY KEY (id) ) ; @@ -275,7 +275,7 @@ CREATE TABLE t_ds_error_command ( update_time timestamp DEFAULT NULL , dependence text , process_instance_priority int DEFAULT NULL , - worker_group_id int DEFAULT '-1' , + worker_group varchar(64), message text , PRIMARY KEY (id) ); @@ -748,7 +748,7 @@ CREATE SEQUENCE t_ds_worker_server_id_sequence; ALTER TABLE t_ds_worker_server ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_worker_server_id_sequence'); --- Records of t_ds_user，user : admin , password : dolphinscheduler123 +-- Records of t_ds_user?user : admin , password : dolphinscheduler123 INSERT INTO t_ds_user(user_name,user_password,user_type,email,phone,tenant_id,create_time,update_time) VALUES ('admin', '7ad2410b2f4c074479a8937a28a22b8f', '0', 'xxx@qq.com', 'xx', '0', '2018-03-27 15:48:50', '2018-10-24 17:40:22'); -- Records of t_ds_alertgroup，dolphinscheduler warning group diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java index 7f3eb38760..127c5b7322 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java @@ -101,9 +101,9 @@ public class ErrorCommand { private String message; /** - * worker group id + * worker group */ - private int workerGroupId; + private String workerGroup; public ErrorCommand(){} @@ -257,17 +257,25 @@ public class ErrorCommand { this.updateTime = updateTime; } - public int getWorkerGroupId() { - return workerGroupId; + public String getWorkerGroup() { + return workerGroup; } - public void setWorkerGroupId(int workerGroupId) { - this.workerGroupId = workerGroupId; + public void setWorkerGroup(String workerGroup) { + this.workerGroup = workerGroup; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; } @Override public String toString() { - return "Command{" + + return "ErrorCommand{" + "id=" + id + ", commandType=" + commandType + ", processDefinitionId=" + processDefinitionId + @@ -281,17 +289,8 @@ public class ErrorCommand { ", startTime=" + startTime + ", processInstancePriority=" + processInstancePriority + ", updateTime=" + updateTime + - ", message=" + message + + ", message='" + message + '\'' + + ", workerGroup='" + workerGroup + '\'' + '}'; } - - public String getMessage() { - return message; - } - - public void setMessage(String message) { - this.message = message; - } - - } From 8b4eb20b5cd27121cfe1c4bd57bb2369fef884d8 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Wed, 15 Apr 2020 22:28:49 +0800 Subject: [PATCH 08/23] fix NPE when grant resource or save process definition (#2434) Co-authored-by: dailidong --- .../api/service/ProcessDefinitionService.java | 6 ++++-- .../api/service/UsersService.java | 21 ++++++++++++------- .../mysql/dolphinscheduler_ddl.sql | 10 ++++----- .../postgresql/dolphinscheduler_ddl.sql | 10 ++++----- 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index eed9c78e74..368492388d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -173,8 +173,10 @@ public class ProcessDefinitionService extends BaseDAGService { for(TaskNode taskNode : tasks){ String taskParameter = taskNode.getParams(); AbstractParameters params = TaskParametersUtils.getParameters(taskNode.getType(),taskParameter); - Set tempSet = params.getResourceFilesList().stream().map(t->t.getId()).collect(Collectors.toSet()); - resourceIds.addAll(tempSet); + if (CollectionUtils.isNotEmpty(params.getResourceFilesList())) { + Set tempSet = params.getResourceFilesList().stream().map(t->t.getId()).collect(Collectors.toSet()); + resourceIds.addAll(tempSet); + } } StringBuilder sb = new StringBuilder(); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java index 4671188d28..220b4fc4d0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java @@ -423,6 +423,7 @@ public class UsersService extends BaseService { * @param projectIds project id array * @return grant result code */ + @Transactional(rollbackFor = Exception.class) public Map grantProject(User loginUser, int userId, String projectIds) { Map result = new HashMap<>(5); result.put(Constants.STATUS, false); @@ -472,6 +473,7 @@ public class UsersService extends BaseService { * @param resourceIds resource id array * @return grant result code */ + @Transactional(rollbackFor = Exception.class) public Map grantResources(User loginUser, int userId, String resourceIds) { Map result = new HashMap<>(5); //only admin can operate @@ -484,17 +486,20 @@ public class UsersService extends BaseService { return result; } - String[] resourceFullIdArr = resourceIds.split(","); - // need authorize resource id set Set needAuthorizeResIds = new HashSet(); - for (String resourceFullId : resourceFullIdArr) { - String[] resourceIdArr = resourceFullId.split("-"); - for (int i=0;i<=resourceIdArr.length-1;i++) { - int resourceIdValue = Integer.parseInt(resourceIdArr[i]); - needAuthorizeResIds.add(resourceIdValue); + if (StringUtils.isNotBlank(resourceIds)) { + String[] resourceFullIdArr = resourceIds.split(","); + // need authorize resource id set + for (String resourceFullId : resourceFullIdArr) { + String[] resourceIdArr = resourceFullId.split("-"); + for (int i=0;i<=resourceIdArr.length-1;i++) { + int resourceIdValue = Integer.parseInt(resourceIdArr[i]); + needAuthorizeResIds.add(resourceIdValue); + } } } + //get the authorized resource id list by user id List oldAuthorizedRes = resourceMapper.queryAuthorizedResourceList(userId); //if resource type is UDF,need check whether it is bound by UDF functon @@ -565,6 +570,7 @@ public class UsersService extends BaseService { * @param udfIds udf id array * @return grant result code */ + @Transactional(rollbackFor = Exception.class) public Map grantUDFFunction(User loginUser, int userId, String udfIds) { Map result = new HashMap<>(5); @@ -611,6 +617,7 @@ public class UsersService extends BaseService { * @param datasourceIds data source id array * @return grant result code */ + @Transactional(rollbackFor = Exception.class) public Map grantDataSource(User loginUser, int userId, String datasourceIds) { Map result = new HashMap<>(5); result.put(Constants.STATUS, false); diff --git a/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql b/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql index 33d801d4ec..7432396825 100644 --- a/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql @@ -167,7 +167,7 @@ CREATE PROCEDURE ac_dolphin_T_t_ds_process_instance_A_worker_group() AND TABLE_SCHEMA=(SELECT DATABASE()) AND COLUMN_NAME ='worker_group') THEN - ALTER TABLE t_ds_process_instance ADD `worker_group` varchar(255) DEFAULT '' COMMENT 'worker group'; + ALTER TABLE t_ds_process_instance ADD `worker_group` varchar(64) DEFAULT '' COMMENT 'worker group'; END IF; END; @@ -207,7 +207,7 @@ CREATE PROCEDURE ac_dolphin_T_t_ds_task_instance_A_worker_group() AND TABLE_SCHEMA=(SELECT DATABASE()) AND COLUMN_NAME ='worker_group') THEN - ALTER TABLE t_ds_task_instance ADD `worker_group` varchar(255) DEFAULT '' COMMENT 'worker group'; + ALTER TABLE t_ds_task_instance ADD `worker_group` varchar(64) DEFAULT '' COMMENT 'worker group'; END IF; END; @@ -247,7 +247,7 @@ CREATE PROCEDURE ac_dolphin_T_t_ds_schedules_A_worker_group() AND TABLE_SCHEMA=(SELECT DATABASE()) AND COLUMN_NAME ='worker_group') THEN - ALTER TABLE t_ds_schedules ADD `worker_group` varchar(255) DEFAULT '' COMMENT 'worker group'; + ALTER TABLE t_ds_schedules ADD `worker_group` varchar(64) DEFAULT '' COMMENT 'worker group'; END IF; END; @@ -287,7 +287,7 @@ CREATE PROCEDURE ac_dolphin_T_t_ds_command_A_worker_group() AND TABLE_SCHEMA=(SELECT DATABASE()) AND COLUMN_NAME ='worker_group') THEN - ALTER TABLE t_ds_command ADD `worker_group` varchar(255) DEFAULT '' COMMENT 'worker group'; + ALTER TABLE t_ds_command ADD `worker_group` varchar(64) DEFAULT '' COMMENT 'worker group'; END IF; END; @@ -327,7 +327,7 @@ CREATE PROCEDURE ac_dolphin_T_t_ds_error_command_A_worker_group() AND TABLE_SCHEMA=(SELECT DATABASE()) AND COLUMN_NAME ='worker_group') THEN - ALTER TABLE t_ds_error_command ADD `worker_group` varchar(255) DEFAULT '' COMMENT 'worker group'; + ALTER TABLE t_ds_error_command ADD `worker_group` varchar(64) DEFAULT '' COMMENT 'worker group'; END IF; END; diff --git a/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql b/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql index a4fdc2d6bf..5312d895c8 100644 --- a/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql @@ -164,7 +164,7 @@ BEGIN AND TABLE_NAME='t_ds_process_instance' AND COLUMN_NAME ='worker_group') THEN - ALTER TABLE t_ds_process_instance ADD COLUMN worker_group varchar(255) DEFAULT null; + ALTER TABLE t_ds_process_instance ADD COLUMN worker_group varchar(64) DEFAULT null; END IF; END; $$ LANGUAGE plpgsql; @@ -207,7 +207,7 @@ BEGIN AND TABLE_NAME='t_ds_task_instance' AND COLUMN_NAME ='worker_group') THEN - ALTER TABLE t_ds_task_instance ADD COLUMN worker_group varchar(255) DEFAULT null; + ALTER TABLE t_ds_task_instance ADD COLUMN worker_group varchar(64) DEFAULT null; END IF; END; $$ LANGUAGE plpgsql; @@ -249,7 +249,7 @@ BEGIN AND TABLE_NAME='t_ds_schedules' AND COLUMN_NAME ='worker_group') THEN - ALTER TABLE t_ds_schedules ADD COLUMN worker_group varchar(255) DEFAULT null; + ALTER TABLE t_ds_schedules ADD COLUMN worker_group varchar(64) DEFAULT null; END IF; END; $$ LANGUAGE plpgsql; @@ -291,7 +291,7 @@ BEGIN AND TABLE_NAME='t_ds_command' AND COLUMN_NAME ='worker_group') THEN - ALTER TABLE t_ds_command ADD COLUMN worker_group varchar(255) DEFAULT null; + ALTER TABLE t_ds_command ADD COLUMN worker_group varchar(64) DEFAULT null; END IF; END; $$ LANGUAGE plpgsql; @@ -333,7 +333,7 @@ BEGIN AND TABLE_NAME='t_ds_error_command' AND COLUMN_NAME ='worker_group') THEN - ALTER TABLE t_ds_error_command ADD COLUMN worker_group varchar(255) DEFAULT null; + ALTER TABLE t_ds_error_command ADD COLUMN worker_group varchar(64) DEFAULT null; END IF; END; $$ LANGUAGE plpgsql; From 7c70ef8218a593d1fc9f6cdd845b753047a82457 Mon Sep 17 00:00:00 2001 From: Rubik-W Date: Thu, 16 Apr 2020 15:07:13 +0800 Subject: [PATCH 09/23] fix: Gantt chart bug --- .../pages/instance/pages/gantt/_source/gantt.js | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/gantt/_source/gantt.js b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/gantt/_source/gantt.js index a5fdc96023..e808c94627 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/gantt/_source/gantt.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/gantt/_source/gantt.js @@ -24,7 +24,7 @@ let Gantt = function () { this.tasks = [] this.width = null this.height = null - this.tasksName = [] + this.taskNames = [] this.tickFormat = `%H:%M:%S` this.margin = { top: 10, @@ -39,8 +39,12 @@ let Gantt = function () { Gantt.prototype.init = function ({ el, tasks }) { this.el = el this.tasks = tasks - this.tasksName = _.map(_.cloneDeep(tasks), v => v.taskName) - this.height = parseInt(this.tasksName.length * 30) + this.taskNames = _.map(_.cloneDeep(tasks), v => v.taskName) + this.taskNames = this.taskNames.reduce(function (prev, cur) { + prev.indexOf(cur) === -1 && prev.push(cur); + return prev; + },[]) + this.height = parseInt(this.taskNames.length * 30) this.width = $(this.el).width() - this.margin.right - this.margin.left - 5 this.x = d3.time.scale() @@ -49,7 +53,7 @@ Gantt.prototype.init = function ({ el, tasks }) { .clamp(true) this.y = d3.scale.ordinal() - .domain(this.tasksName) + .domain(this.taskNames) .rangeRoundBands([ 0, this.height - this.margin.top - this.margin.bottom ], 0.1) this.xAxis = d3.svg.axis() @@ -98,7 +102,7 @@ Gantt.prototype.initializeXAxis = function () { .clamp(true) this.y = d3.scale.ordinal() - .domain(this.tasksName) + .domain(this.taskNames) .rangeRoundBands([ 0, this.height - this.margin.top - this.margin.bottom ], 0.1) this.xAxis = d3.svg.axis() From d6798c6db7fe50189cd83e4b2a619587c7c5a47a Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Thu, 16 Apr 2020 18:16:19 +0800 Subject: [PATCH 10/23] =?UTF-8?q?1=EF=BC=8Ctask=20status=20statistics=20an?= =?UTF-8?q?d=20process=20status=20statistics=20bug=20fix=20(#2357)=20=202?= =?UTF-8?q?=EF=BC=8Cworker=20group=20bug=20fix=20(#2430)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * dispatch task fail will set task status failed * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix Co-authored-by: qiaozhanwei --- .../api/service/DataAnalysisService.java | 14 +++++--------- .../api/service/DataAnalysisServiceTest.java | 7 ------- .../common/thread/ThreadUtils.java | 10 ++++++++++ .../dolphinscheduler/dao/entity/Command.java | 2 +- .../dao/mapper/CommandMapperTest.java | 13 ++++--------- .../master/consumer/TaskPriorityQueueConsumer.java | 5 ++--- .../server/master/processor/TaskAckProcessor.java | 6 ++---- .../master/processor/TaskResponseProcessor.java | 6 ++---- .../worker/processor/TaskCallbackService.java | 10 ++++------ sql/dolphinscheduler-postgre.sql | 6 +++--- sql/dolphinscheduler_mysql.sql | 4 ++-- 11 files changed, 35 insertions(+), 48 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java index 0c93e00a80..39bec56357 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java @@ -106,14 +106,12 @@ public class DataAnalysisService extends BaseService{ List taskInstanceStateCounts = taskInstanceMapper.countTaskInstanceStateByUser(start, end, projectIds); - if (taskInstanceStateCounts != null && !taskInstanceStateCounts.isEmpty()) { + if (taskInstanceStateCounts != null) { TaskCountDto taskCountResult = new TaskCountDto(taskInstanceStateCounts); result.put(Constants.DATA_LIST, taskCountResult); putMsg(result, Status.SUCCESS); - } else { - putMsg(result, Status.TASK_INSTANCE_STATE_COUNT_ERROR); } - return result; + return result; } private void putErrorRequestParamsMsg(Map result) { @@ -153,14 +151,12 @@ public class DataAnalysisService extends BaseService{ processInstanceMapper.countInstanceStateByUser(start, end, projectIdArray); - if (processInstanceStateCounts != null && !processInstanceStateCounts.isEmpty()) { + if (processInstanceStateCounts != null) { TaskCountDto taskCountResult = new TaskCountDto(processInstanceStateCounts); result.put(Constants.DATA_LIST, taskCountResult); putMsg(result, Status.SUCCESS); - } else { - putMsg(result, Status.COUNT_PROCESS_INSTANCE_STATE_ERROR); } - return result; + return result; } @@ -234,7 +230,7 @@ public class DataAnalysisService extends BaseService{ // count error command state List errorCommandStateCounts = errorCommandMapper.countCommandState( - start, end, projectIdArray); + start, end, projectIdArray); // Map> dataMap = new HashMap<>(); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java index 35cc6ae9a6..14612fcef8 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java @@ -114,9 +114,6 @@ public class DataAnalysisServiceTest { Map result = dataAnalysisService.countTaskStateByProject(user, 2, startDate, endDate); Assert.assertTrue(result.isEmpty()); - // task instance state count error - result = dataAnalysisService.countTaskStateByProject(user, 1, startDate, endDate); - Assert.assertEquals(Status.TASK_INSTANCE_STATE_COUNT_ERROR,result.get(Constants.STATUS)); //SUCCESS Mockito.when(taskInstanceMapper.countTaskInstanceStateByUser(DateUtils.getScheduleDate(startDate), @@ -137,10 +134,6 @@ public class DataAnalysisServiceTest { Map result = dataAnalysisService.countProcessInstanceStateByProject(user,2,startDate,endDate); Assert.assertTrue(result.isEmpty()); - //COUNT_PROCESS_INSTANCE_STATE_ERROR - result = dataAnalysisService.countProcessInstanceStateByProject(user,1,startDate,endDate); - Assert.assertEquals(Status.COUNT_PROCESS_INSTANCE_STATE_ERROR,result.get(Constants.STATUS)); - //SUCCESS Mockito.when(processInstanceMapper.countInstanceStateByUser(DateUtils.getScheduleDate(startDate), DateUtils.getScheduleDate(endDate), new Integer[]{1})).thenReturn(getTaskInstanceStateCounts()); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java index e348835363..a9a124547a 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java @@ -223,4 +223,14 @@ public class ThreadUtils { } return id + " (" + name + ")"; } + + /** + * sleep + * @param millis millis + */ + public static void sleep(final long millis) { + try { + Thread.sleep(millis); + } catch (final InterruptedException ignore) {} + } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java index 5a6974803c..7d52dc93f3 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java @@ -111,7 +111,7 @@ public class Command { /** * worker group */ - @TableField(exist = false) + @TableField("worker_group") private String workerGroup; public Command() { diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java index 468824bf6e..297ea66c94 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java @@ -76,7 +76,8 @@ public class CommandMapperTest { //query Command actualCommand = commandMapper.selectById(expectedCommand.getId()); - assertEquals(expectedCommand, actualCommand); + assertNotNull(actualCommand); + assertEquals(expectedCommand.getProcessDefinitionId(), actualCommand.getProcessDefinitionId()); } /** @@ -94,7 +95,8 @@ public class CommandMapperTest { Command actualCommand = commandMapper.selectById(expectedCommand.getId()); - assertEquals(expectedCommand,actualCommand); + assertNotNull(actualCommand); + assertEquals(expectedCommand.getUpdateTime(),actualCommand.getUpdateTime()); } @@ -127,13 +129,6 @@ public class CommandMapperTest { List actualCommands = commandMapper.selectList(null); assertThat(actualCommands.size(), greaterThanOrEqualTo(count)); - - for (Command actualCommand : actualCommands){ - Command expectedCommand = commandMap.get(actualCommand.getId()); - if (expectedCommand != null){ - assertEquals(expectedCommand,actualCommand); - } - } } /** diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java index b2cf53a575..cdd9ff2219 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java @@ -31,6 +31,7 @@ import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceMysqlParameter; import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetMysqlParameter; import org.apache.dolphinscheduler.common.thread.Stopper; +import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.server.builder.TaskExecutionContextBuilder; @@ -122,9 +123,7 @@ public class TaskPriorityQueueConsumer extends Thread{ result = dispatcher.dispatch(executionContext); } catch (ExecuteException e) { logger.error("dispatch error",e); - try { - Thread.sleep(SLEEP_TIME_MILLIS); - } catch (InterruptedException e1) {} + ThreadUtils.sleep(SLEEP_TIME_MILLIS); } if (result){ diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java index 7af9cdc2cc..3460248dfb 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessor.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.server.master.processor; import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.thread.Stopper; +import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.Preconditions; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.remote.command.Command; @@ -101,10 +102,7 @@ public class TaskAckProcessor implements NettyRequestProcessor { if (taskInstance != null && ackStatus.typeIsRunning()){ break; } - - try { - Thread.sleep(SLEEP_TIME_MILLIS); - } catch (InterruptedException e) {} + ThreadUtils.sleep(SLEEP_TIME_MILLIS); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java index ecb8646ad0..721b146d86 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessor.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.server.master.processor; import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.thread.Stopper; +import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.Preconditions; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.remote.command.Command; @@ -99,10 +100,7 @@ public class TaskResponseProcessor implements NettyRequestProcessor { if (taskInstance != null && responseStatus.typeIsFinished()){ break; } - - try { - Thread.sleep(SLEEP_TIME_MILLIS); - } catch (InterruptedException e) {} + ThreadUtils.sleep(SLEEP_TIME_MILLIS); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java index 7cd25cba65..ecae9edc1c 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java @@ -18,11 +18,11 @@ package org.apache.dolphinscheduler.server.worker.processor; -import com.alibaba.fastjson.JSONObject; import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelFutureListener; import org.apache.dolphinscheduler.common.thread.Stopper; +import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.remote.NettyRemotingClient; import org.apache.dolphinscheduler.remote.command.Command; @@ -37,6 +37,8 @@ import org.springframework.stereotype.Service; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import static org.apache.dolphinscheduler.common.Constants.SLEEP_TIME_MILLIS; + /** * taks callback service */ @@ -98,11 +100,7 @@ public class TaskCallbackService { while (Stopper.isRunning()) { if (CollectionUtils.isEmpty(masterNodes)) { logger.error("no available master node"); - try { - Thread.sleep(1000); - }catch (Exception e){ - - } + ThreadUtils.sleep(SLEEP_TIME_MILLIS); }else { break; } diff --git a/sql/dolphinscheduler-postgre.sql b/sql/dolphinscheduler-postgre.sql index b48ddde042..b26520e29c 100644 --- a/sql/dolphinscheduler-postgre.sql +++ b/sql/dolphinscheduler-postgre.sql @@ -234,7 +234,7 @@ CREATE TABLE t_ds_command ( dependence varchar(255) DEFAULT NULL , update_time timestamp DEFAULT NULL , process_instance_priority int DEFAULT NULL , - worker_group_id int DEFAULT '-1' , + worker_group varchar(64), PRIMARY KEY (id) ) ; @@ -275,7 +275,7 @@ CREATE TABLE t_ds_error_command ( update_time timestamp DEFAULT NULL , dependence text , process_instance_priority int DEFAULT NULL , - worker_group_id int DEFAULT '-1' , + worker_group varchar(64), message text , PRIMARY KEY (id) ); @@ -748,7 +748,7 @@ CREATE SEQUENCE t_ds_worker_server_id_sequence; ALTER TABLE t_ds_worker_server ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_worker_server_id_sequence'); --- Records of t_ds_user，user : admin , password : dolphinscheduler123 +-- Records of t_ds_user?user : admin , password : dolphinscheduler123 INSERT INTO t_ds_user(user_name,user_password,user_type,email,phone,tenant_id,create_time,update_time) VALUES ('admin', '7ad2410b2f4c074479a8937a28a22b8f', '0', 'xxx@qq.com', 'xx', '0', '2018-03-27 15:48:50', '2018-10-24 17:40:22'); -- Records of t_ds_alertgroup，dolphinscheduler warning group diff --git a/sql/dolphinscheduler_mysql.sql b/sql/dolphinscheduler_mysql.sql index 559e684133..c0c81dcb21 100644 --- a/sql/dolphinscheduler_mysql.sql +++ b/sql/dolphinscheduler_mysql.sql @@ -333,7 +333,7 @@ CREATE TABLE `t_ds_command` ( `dependence` varchar(255) DEFAULT NULL COMMENT 'dependence', `update_time` datetime DEFAULT NULL COMMENT 'update time', `process_instance_priority` int(11) DEFAULT NULL COMMENT 'process instance priority: 0 Highest,1 High,2 Medium,3 Low,4 Lowest', - `worker_group_id` int(11) DEFAULT '-1' COMMENT 'worker group id', + `worker_group` varchar(64) COMMENT 'worker group', PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; @@ -380,7 +380,7 @@ CREATE TABLE `t_ds_error_command` ( `update_time` datetime DEFAULT NULL COMMENT 'update time', `dependence` text COMMENT 'dependence', `process_instance_priority` int(11) DEFAULT NULL COMMENT 'process instance priority, 0 Highest,1 High,2 Medium,3 Low,4 Lowest', - `worker_group_id` int(11) DEFAULT '-1' COMMENT 'worker group id', + `worker_group` varchar(64) COMMENT 'worker group', `message` text COMMENT 'message', PRIMARY KEY (`id`) USING BTREE ) ENGINE=InnoDB DEFAULT CHARSET=utf8 ROW_FORMAT=DYNAMIC; From f929c6f3b7a43b78db977206c3425d88ae3c357b Mon Sep 17 00:00:00 2001 From: songgg <1172417734@qq.com> Date: Thu, 16 Apr 2020 22:50:26 +0800 Subject: [PATCH 11/23] =?UTF-8?q?[optimization]=EF=BC=9Aprevent=20repeated?= =?UTF-8?q?=20database=20updates=20(#2396)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * sqlTask failed to run * prevent repeated database updates * prevent repeated database updates * prevent repeated database updates Co-authored-by: songqh Co-authored-by: dailidong --- .../api/service/ExecutorService.java | 25 +++++++++---------- .../master/runner/MasterExecThread.java | 2 +- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index 7ce7497e98..d290886572 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -225,20 +225,14 @@ public class ExecutorService extends BaseService{ if (processInstance.getState() == ExecutionStatus.READY_STOP) { putMsg(result, Status.PROCESS_INSTANCE_ALREADY_CHANGED, processInstance.getName(), processInstance.getState()); } else { - processInstance.setCommandType(CommandType.STOP); - processInstance.addHistoryCmd(CommandType.STOP); - processService.updateProcessInstance(processInstance); - result = updateProcessInstanceState(processInstanceId, ExecutionStatus.READY_STOP); + result = updateProcessInstancePrepare(processInstance, CommandType.STOP, ExecutionStatus.READY_STOP); } break; case PAUSE: if (processInstance.getState() == ExecutionStatus.READY_PAUSE) { putMsg(result, Status.PROCESS_INSTANCE_ALREADY_CHANGED, processInstance.getName(), processInstance.getState()); } else { - processInstance.setCommandType(CommandType.PAUSE); - processInstance.addHistoryCmd(CommandType.PAUSE); - processService.updateProcessInstance(processInstance); - result = updateProcessInstanceState(processInstanceId, ExecutionStatus.READY_PAUSE); + result = updateProcessInstancePrepare(processInstance, CommandType.PAUSE, ExecutionStatus.READY_PAUSE); } break; default: @@ -308,22 +302,27 @@ public class ExecutorService extends BaseService{ } /** - * update process instance state + * prepare to update process instance command type and status * - * @param processInstanceId process instance id + * @param processInstance process instance + * @param commandType command type * @param executionStatus execute status * @return update result */ - private Map updateProcessInstanceState(Integer processInstanceId, ExecutionStatus executionStatus) { + private Map updateProcessInstancePrepare(ProcessInstance processInstance, CommandType commandType, ExecutionStatus executionStatus) { Map result = new HashMap<>(5); - int update = processService.updateProcessInstanceState(processInstanceId, executionStatus); + processInstance.setCommandType(commandType); + processInstance.addHistoryCmd(commandType); + processInstance.setState(executionStatus); + int update = processService.updateProcessInstance(processInstance); + + // determine whether the process is normal if (update > 0) { putMsg(result, Status.SUCCESS); } else { putMsg(result, Status.EXECUTE_PROCESS_INSTANCE_ERROR); } - return result; } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index b1ac73cb54..e1e8c090fd 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -914,7 +914,7 @@ public class MasterExecThread implements Runnable { processInstance.getId(), processInstance.getName(), processInstance.getState(), state, processInstance.getCommandType()); - processInstance.setState(state); + ProcessInstance instance = processService.findProcessInstanceById(processInstance.getId()); instance.setState(state); instance.setProcessDefinition(processInstance.getProcessDefinition()); From f5a8b5667444b18a7de5ae8304a654e4d4d9a39b Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Fri, 17 Apr 2020 10:53:11 +0800 Subject: [PATCH 12/23] Online editing is prohibited (#2444) * Fix the problem of data echo in script edit box * Optimize resource tree * Change the name of the shell node and modify the parameter transmission method of spark, mr, python, and flink nodes * Repair naming * Modify list style * Online editing is prohibited Co-authored-by: dailidong Co-authored-by: qiaozhanwei --- .../js/conf/home/pages/dag/_source/formModel/tasks/flink.vue | 2 +- .../src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue | 2 +- .../js/conf/home/pages/dag/_source/formModel/tasks/python.vue | 2 +- .../js/conf/home/pages/dag/_source/formModel/tasks/shell.vue | 2 +- .../js/conf/home/pages/dag/_source/formModel/tasks/spark.vue | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue index 195e3c64f9..9d4189406c 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue @@ -141,7 +141,7 @@
{{$t('Resources')}}
- +
{{ node.raw.fullName }}
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue index 112e47dc4f..a5c23d45c7 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue @@ -78,7 +78,7 @@
{{$t('Resources')}}
- +
{{ node.raw.fullName }}
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue index dd7ea942dd..b9ed72c1ea 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue @@ -28,7 +28,7 @@
{{$t('Resources')}}
- +
{{ node.raw.fullName }}
 ${log.base}/dolphinscheduler-master.log @@ -52,6 +75,7 @@ + diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java new file mode 100644 index 0000000000..299d4ba800 --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.server.master; + + +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.server.master.config.MasterConfig; +import org.apache.dolphinscheduler.server.master.runner.ConditionsTaskExecThread; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationContext; + +import java.util.ArrayList; +import java.util.List; + +@RunWith(MockitoJUnitRunner.Silent.class) +public class ConditionsTaskTest { + + + private static final Logger logger = LoggerFactory.getLogger(DependentTaskTest.class); + + private ProcessService processService; + private ApplicationContext applicationContext; + + + private MasterConfig config; + + @Before + public void before() { + config = new MasterConfig(); + config.setMasterTaskCommitRetryTimes(3); + config.setMasterTaskCommitInterval(1000); + processService = Mockito.mock(ProcessService.class); + applicationContext = Mockito.mock(ApplicationContext.class); + SpringApplicationContext springApplicationContext = new SpringApplicationContext(); + springApplicationContext.setApplicationContext(applicationContext); + Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService); + Mockito.when(applicationContext.getBean(MasterConfig.class)).thenReturn(config); + + Mockito.when(processService + .findTaskInstanceById(252612)) + .thenReturn(getTaskInstance()); + + Mockito.when(processService.saveTaskInstance(getTaskInstance())) + .thenReturn(true); + + Mockito.when(processService.findProcessInstanceById(10112)) + .thenReturn(getProcessInstance()); + + Mockito.when(processService + .findValidTaskListByProcessId(10112)) + .thenReturn(getTaskInstances()); + } + + @Test + public void testCondition(){ + TaskInstance taskInstance = getTaskInstance(); + String dependString = "{\"dependTaskList\":[{\"dependItemList\":[{\"depTasks\":\"1\",\"status\":\"SUCCESS\"}],\"relation\":\"AND\"}],\"relation\":\"AND\"}"; + String conditionResult = "{\"successNode\":[\"2\"],\"failedNode\":[\"3\"]}"; + + taskInstance.setDependency(dependString); + Mockito.when(processService.submitTask(taskInstance)) + .thenReturn(taskInstance); + ConditionsTaskExecThread conditions = + new ConditionsTaskExecThread(taskInstance); + + try { + conditions.call(); + } catch (Exception e) { + e.printStackTrace(); + } + + Assert.assertEquals(ExecutionStatus.SUCCESS, conditions.getTaskInstance().getState()); + } + + + private TaskInstance getTaskInstance(){ + TaskInstance taskInstance = new TaskInstance(); + taskInstance.setId(252612); + taskInstance.setName("C"); + taskInstance.setTaskType("CONDITIONS"); + taskInstance.setProcessInstanceId(10112); + taskInstance.setProcessDefinitionId(100001); + return taskInstance; + } + + + + private List getTaskInstances(){ + List list = new ArrayList<>(); + TaskInstance taskInstance = new TaskInstance(); + taskInstance.setId(199999); + taskInstance.setName("1"); + taskInstance.setState(ExecutionStatus.SUCCESS); + list.add(taskInstance); + return list; + } + + private ProcessInstance getProcessInstance(){ + ProcessInstance processInstance = new ProcessInstance(); + processInstance.setId(10112); + processInstance.setProcessDefinitionId(100001); + processInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + + return processInstance; + } + +} diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java similarity index 66% rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTaskTest.java rename to dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java index 3477f4ac67..a65b0508d8 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java @@ -14,17 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.server.worker.task.dependent; +package org.apache.dolphinscheduler.server.master; -import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.DateInterval; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.utils.dependent.DependentDateUtils; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; -import org.apache.dolphinscheduler.server.worker.task.TaskProps; +import org.apache.dolphinscheduler.server.master.config.MasterConfig; +import org.apache.dolphinscheduler.server.master.runner.DependentTaskExecThread; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; import org.junit.Assert; @@ -50,12 +50,23 @@ public class DependentTaskTest { private ApplicationContext applicationContext; + private MasterConfig config; + @Before public void before() throws Exception{ + + config = new MasterConfig(); + config.setMasterTaskCommitRetryTimes(3); + config.setMasterTaskCommitInterval(1000); processService = Mockito.mock(ProcessService.class); + DateInterval dateInterval =DependentDateUtils.getTodayInterval(new Date()).get(0); Mockito.when(processService - .findLastRunningProcess(4,DependentDateUtils.getTodayInterval(new Date()).get(0))) + .findLastRunningProcess(4, dateInterval.getStartTime(), + dateInterval.getEndTime())) .thenReturn(findLastProcessInterval()); + + + Mockito.when(processService .getTaskNodeListByDefinitionId(4)) .thenReturn(getTaskNodes()); @@ -66,32 +77,62 @@ public class DependentTaskTest { Mockito.when(processService .findTaskInstanceById(252612)) .thenReturn(getTaskInstance()); + + + Mockito.when(processService.findProcessInstanceById(10111)) + .thenReturn(getProcessInstance()); + Mockito.when(processService.findProcessDefineById(0)) + .thenReturn(getProcessDefinition()); + Mockito.when(processService.saveTaskInstance(getTaskInstance())) + .thenReturn(true); + applicationContext = Mockito.mock(ApplicationContext.class); SpringApplicationContext springApplicationContext = new SpringApplicationContext(); springApplicationContext.setApplicationContext(applicationContext); Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService); + Mockito.when(applicationContext.getBean(MasterConfig.class)).thenReturn(config); } @Test public void test() throws Exception{ - TaskProps taskProps = new TaskProps(); + TaskInstance taskInstance = getTaskInstance(); String dependString = "{\"dependTaskList\":[{\"dependItemList\":[{\"dateValue\":\"today\",\"depTasks\":\"ALL\",\"projectId\":1,\"definitionList\":[{\"label\":\"C\",\"value\":4},{\"label\":\"B\",\"value\":3},{\"label\":\"A\",\"value\":2}],\"cycle\":\"day\",\"definitionId\":4}],\"relation\":\"AND\"}],\"relation\":\"AND\"}"; - taskProps.setDependence(dependString); - taskProps.setTaskStartTime(new Date()); - DependentTask dependentTask = new DependentTask(new TaskExecutionContext(), logger); - dependentTask.init(); - dependentTask.handle(); - Assert.assertEquals(dependentTask.getExitStatusCode(), Constants.EXIT_CODE_SUCCESS ); + taskInstance.setDependency(dependString); + Mockito.when(processService.submitTask(taskInstance)) + .thenReturn(taskInstance); + DependentTaskExecThread dependentTask = + new DependentTaskExecThread(taskInstance); + + dependentTask.call(); + + Assert.assertEquals(ExecutionStatus.SUCCESS, dependentTask.getTaskInstance().getState()); } private ProcessInstance findLastProcessInterval(){ ProcessInstance processInstance = new ProcessInstance(); processInstance.setId(11); + processInstance.setProcessDefinitionId(4); processInstance.setState(ExecutionStatus.SUCCESS); return processInstance; } + private ProcessDefinition getProcessDefinition(){ + ProcessDefinition processDefinition = new ProcessDefinition(); + processDefinition.setId(0); + return processDefinition; + } + + private ProcessInstance getProcessInstance(){ + ProcessInstance processInstance = new ProcessInstance(); + processInstance.setId(10111); + processInstance.setProcessDefinitionId(0); + processInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + + return processInstance; + } + + private List getTaskNodes(){ List list = new ArrayList<>(); TaskNode taskNode = new TaskNode(); @@ -113,9 +154,10 @@ public class DependentTaskTest { private TaskInstance getTaskInstance(){ TaskInstance taskInstance = new TaskInstance(); + taskInstance.setTaskType("DEPENDENT"); taskInstance.setId(252612); taskInstance.setName("C"); - taskInstance.setState(ExecutionStatus.SUCCESS); + taskInstance.setProcessInstanceId(10111); return taskInstance; } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index d2a13aebab..c69ea34c5c 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -782,14 +782,13 @@ public class ProcessService { * submit task to db * submit sub process to command * @param taskInstance taskInstance - * @param processInstance processInstance * @return task instance */ @Transactional(rollbackFor = Exception.class) - public TaskInstance submitTask(TaskInstance taskInstance, ProcessInstance processInstance){ - logger.info("start submit task : {}, instance id:{}, state: {}, ", - taskInstance.getName(), processInstance.getId(), processInstance.getState() ); - processInstance = this.findProcessInstanceDetailById(processInstance.getId()); + public TaskInstance submitTask(TaskInstance taskInstance){ + ProcessInstance processInstance = this.findProcessInstanceDetailById(taskInstance.getProcessInstanceId()); + logger.info("start submit task : {}, instance id:{}, state: {}", + taskInstance.getName(), taskInstance.getProcessInstanceId(), processInstance.getState()); //submit to db TaskInstance task = submitTaskInstanceToDB(taskInstance, processInstance); if(task == null){ @@ -1637,13 +1636,14 @@ public class ProcessService { /** * find last running process instance * @param definitionId process definition id - * @param dateInterval dateInterval + * @param startTime start time + * @param endTime end time * @return process instance */ - public ProcessInstance findLastRunningProcess(int definitionId, DateInterval dateInterval) { + public ProcessInstance findLastRunningProcess(int definitionId, Date startTime, Date endTime) { return processInstanceMapper.queryLastRunningProcess(definitionId, - dateInterval.getStartTime(), - dateInterval.getEndTime(), + startTime, + endTime, stateArray); } @@ -1799,5 +1799,22 @@ public class ProcessService { return resourceMapper.listResourceByIds(resIds); } + /** + * format task app id in task instance + * @param taskInstance + * @return + */ + public String formatTaskAppId(TaskInstance taskInstance){ + ProcessDefinition definition = this.findProcessDefineById(taskInstance.getProcessDefinitionId()); + ProcessInstance processInstanceById = this.findProcessInstanceById(taskInstance.getProcessInstanceId()); + + if(definition == null || processInstanceById == null){ + return ""; + } + return String.format("%s_%s_%s", + definition.getId(), + processInstanceById.getId(), + taskInstance.getId()); + } } diff --git a/pom.xml b/pom.xml index f90700e318..9e9a43dac3 100644 --- a/pom.xml +++ b/pom.xml @@ -782,6 +782,8 @@ **/server/master/register/MasterRegistryTest.java **/server/master/AlertManagerTest.java **/server/master/MasterCommandTest.java + **/server/master/DependentTaskTest.java + **/server/master/ConditionsTaskTest.java **/server/master/MasterExecThreadTest.java **/server/master/ParamsTest.java **/server/register/ZookeeperNodeManagerTest.java From ef686811e5d9fd86dcd1f9d77ea9add7c59c6c17 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Mon, 20 Apr 2020 13:49:58 +0800 Subject: [PATCH 16/23] Change the background color of the disabled state of the radio button when the workflow goes online #2451 (#2453) * Fix the problem of data echo in script edit box * Optimize resource tree * Change the name of the shell node and modify the parameter transmission method of spark, mr, python, and flink nodes * Repair naming * Modify list style * Online editing is prohibited * Change the background color of the disabled state of the radio button when the workflow goes online Co-authored-by: dailidong Co-authored-by: qiaozhanwei --- .../js/conf/home/pages/dag/_source/formModel/formModel.vue | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index 6651f2014f..a2b3ad9bae 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -694,4 +694,9 @@ From 0f54149ae3b8d42154bca97438e0eb6083759a18 Mon Sep 17 00:00:00 2001 From: dailidong Date: Mon, 20 Apr 2020 14:38:46 +0800 Subject: [PATCH 17/23] exclude HadoopUtils Test (#2465) * Update pom.xml * Update pom.xml --- pom.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9e9a43dac3..7b28a88af4 100644 --- a/pom.xml +++ b/pom.xml @@ -747,7 +747,6 @@ **/common/utils/DependentUtilsTest.java **/common/utils/EncryptionUtilsTest.java **/common/utils/FileUtilsTest.java - **/common/utils/HadoopUtilsTest.java **/common/utils/IpUtilsTest.java **/common/utils/JSONUtilsTest.java **/common/utils/LoggerUtilsTest.java @@ -827,6 +826,9 @@ **/dao/utils/DagHelperTest.java **/dao/AlertDaoTest.java + + **/common/utils/HadoopUtils.java + From dc36c77ba7de2d610fb22cfd6c5a67471045c897 Mon Sep 17 00:00:00 2001 From: dailidong Date: Mon, 20 Apr 2020 16:10:21 +0800 Subject: [PATCH 18/23] Update pom.xml (#2467) --- pom.xml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 7b28a88af4..f8cbe7fca2 100644 --- a/pom.xml +++ b/pom.xml @@ -761,6 +761,7 @@ **/common/utils/StringUtilsTest.java **/common/utils/TaskParametersUtilsTest.java **/common/ConstantsTest.java + **/common/utils/HadoopUtils.java **/dao/mapper/AccessTokenMapperTest.java **/dao/mapper/AlertGroupMapperTest.java **/dao/mapper/CommandMapperTest.java @@ -826,9 +827,6 @@ **/dao/utils/DagHelperTest.java **/dao/AlertDaoTest.java - - **/common/utils/HadoopUtils.java - From eec92ed1fc1a35e22b8e38ac40681a1dd803399b Mon Sep 17 00:00:00 2001 From: dailidong Date: Tue, 21 Apr 2020 10:25:45 +0800 Subject: [PATCH 19/23] simplify and optimize config (#2469) * [Refactor worker] simplify and optimize config (#2386) * simplify config * simplify config * simplify and optimize config * Update HadoopUtils.java optimize HadoopUtils * Update HadoopUtilsTest.java * Update HadoopUtilsTest.java * Update HadoopUtilsTest.java * Update HttpUtils.java * Update pom.xml * Update HadoopUtilsTest.java * Update HadoopUtilsTest.java * Update HadoopUtilsTest.java * Update HadoopUtilsTest.java * Update HadoopUtilsTest.java * Update HadoopUtilsTest.java --- .../alert/template/AlertTemplateFactory.java | 17 +- .../alert/utils/Constants.java | 2 - .../alert/utils/MailUtils.java | 9 +- .../src/main/resources/alert.properties | 3 - .../template/AlertTemplateFactoryTest.java | 1 - .../common/utils/HadoopUtils.java | 60 +-- .../common/utils/HttpUtils.java | 2 +- .../src/main/resources/common.properties | 24 +- .../common/utils/HadoopUtilsTest.java | 162 +++++-- .../src/main/resources/datasource.properties | 3 - .../main/resources/config/install_config.conf | 131 +++++- install.sh | 398 ++---------------- pom.xml | 3 +- script/dolphinscheduler-daemon.sh | 6 +- script/scp-hosts.sh | 22 +- script/start-all.sh | 13 +- script/stop-all.sh | 14 +- 17 files changed, 359 insertions(+), 511 deletions(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactory.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactory.java index 58e3800339..965677e7e1 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactory.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactory.java @@ -17,9 +17,6 @@ package org.apache.dolphinscheduler.alert.template; import org.apache.dolphinscheduler.alert.template.impl.DefaultHTMLTemplate; -import org.apache.dolphinscheduler.alert.utils.Constants; -import org.apache.dolphinscheduler.alert.utils.PropertyUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,8 +27,6 @@ public class AlertTemplateFactory { private static final Logger logger = LoggerFactory.getLogger(AlertTemplateFactory.class); - private static final String alertTemplate = PropertyUtils.getString(Constants.ALERT_TEMPLATE); - private AlertTemplateFactory(){} /** @@ -39,16 +34,6 @@ public class AlertTemplateFactory { * @return a template, default is DefaultHTMLTemplate */ public static AlertTemplate getMessageTemplate() { - - if(StringUtils.isEmpty(alertTemplate)){ - return new DefaultHTMLTemplate(); - } - - switch (alertTemplate){ - case "html": - return new DefaultHTMLTemplate(); - default: - throw new IllegalArgumentException(String.format("not support alert template: %s",alertTemplate)); - } + return new DefaultHTMLTemplate(); } } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java index 94d95b3c26..28be8aa195 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java @@ -77,8 +77,6 @@ public class Constants { public static final int NUMBER_1000 = 1000; - public static final String ALERT_TEMPLATE = "alert.template"; - public static final String SPRING_DATASOURCE_DRIVER_CLASS_NAME = "spring.datasource.driver-class-name"; public static final String SPRING_DATASOURCE_URL = "spring.datasource.url"; diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java index e20b75a096..ef364cb1c2 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java @@ -260,9 +260,14 @@ public class MailUtils { part1.setContent(partContent, Constants.TEXT_HTML_CHARSET_UTF_8); // set attach file MimeBodyPart part2 = new MimeBodyPart(); - // make excel file - ExcelUtils.genExcelFile(content,title, xlsFilePath); File file = new File(xlsFilePath + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS); + if (!file.getParentFile().exists()) { + file.getParentFile().mkdirs(); + } + // make excel file + + ExcelUtils.genExcelFile(content,title,xlsFilePath); + part2.attachFile(file); part2.setFileName(MimeUtility.encodeText(title + Constants.EXCEL_SUFFIX_XLS,Constants.UTF_8,"B")); // add components to collection diff --git a/dolphinscheduler-alert/src/main/resources/alert.properties b/dolphinscheduler-alert/src/main/resources/alert.properties index c359b27ff2..3e83c01235 100644 --- a/dolphinscheduler-alert/src/main/resources/alert.properties +++ b/dolphinscheduler-alert/src/main/resources/alert.properties @@ -18,9 +18,6 @@ #alert type is EMAIL/SMS alert.type=EMAIL -# alter msg template, default is html template -#alert.template=html - # mail server configuration mail.protocol=SMTP mail.server.host=xxx.xxx.com diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactoryTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactoryTest.java index 6865b895e2..32201e6011 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactoryTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactoryTest.java @@ -47,7 +47,6 @@ public class AlertTemplateFactoryTest { public void testGetMessageTemplate(){ PowerMockito.mockStatic(PropertyUtils.class); - when(PropertyUtils.getString(Constants.ALERT_TEMPLATE)).thenReturn("html"); AlertTemplate defaultTemplate = AlertTemplateFactory.getMessageTemplate(); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 5d4f867295..02f00ce330 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -57,7 +57,8 @@ public class HadoopUtils implements Closeable { private static String hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER); public static final String resourceUploadPath = PropertyUtils.getString(RESOURCE_UPLOAD_PATH, "/dolphinscheduler"); - + public static final String rmHaIds = PropertyUtils.getString(Constants.YARN_RESOURCEMANAGER_HA_RM_IDS); + public static final String appAddress = PropertyUtils.getString(Constants.YARN_APPLICATION_STATUS_ADDRESS); private static final String HADOOP_UTILS_KEY = "HADOOP_UTILS_KEY"; @@ -110,14 +111,15 @@ public class HadoopUtils implements Closeable { try { configuration = new Configuration(); - String resUploadStartupType = PropertyUtils.getString(Constants.RESOURCE_STORAGE_TYPE); - ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); + String resourceStorageType = PropertyUtils.getString(Constants.RESOURCE_STORAGE_TYPE); + ResUploadType resUploadType = ResUploadType.valueOf(resourceStorageType); - if (resUploadType == ResUploadType.HDFS) { - if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE,false)) { + if (resUploadType == ResUploadType.HDFS){ + if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE,false)){ System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)); - configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION,"kerberos"); + hdfsUser = ""; UserGroupInformation.setConfiguration(configuration); UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)); @@ -166,21 +168,6 @@ public class HadoopUtils implements Closeable { } - String rmHaIds = PropertyUtils.getString(Constants.YARN_RESOURCEMANAGER_HA_RM_IDS); - String appAddress = PropertyUtils.getString(Constants.YARN_APPLICATION_STATUS_ADDRESS); - //not use resourcemanager - if (rmHaIds.contains(Constants.YARN_RESOURCEMANAGER_HA_XX)){ - yarnEnabled = false; - } else if (!StringUtils.isEmpty(rmHaIds)) { - //resourcemanager HA enabled - appAddress = getAppAddress(appAddress, rmHaIds); - yarnEnabled = true; - logger.info("appAddress : {}", appAddress); - } else { - //single resourcemanager enabled - yarnEnabled = true; - } - configuration.set(Constants.YARN_APPLICATION_STATUS_ADDRESS, appAddress); } catch (Exception e) { logger.error(e.getMessage(), e); } @@ -200,7 +187,29 @@ public class HadoopUtils implements Closeable { * @return url of application */ public String getApplicationUrl(String applicationId) { - return String.format(configuration.get(Constants.YARN_APPLICATION_STATUS_ADDRESS), applicationId); + /** + * if rmHaIds contains xx, it signs not use resourcemanager + * otherwise: + * if rmHaIds is empty, single resourcemanager enabled + * if rmHaIds not empty: resourcemanager HA enabled + */ + String appUrl = ""; + //not use resourcemanager + if (rmHaIds.contains(Constants.YARN_RESOURCEMANAGER_HA_XX)){ + + yarnEnabled = false; + logger.warn("should not step here"); + } else if (!StringUtils.isEmpty(rmHaIds)) { + //resourcemanager HA enabled + appUrl = getAppAddress(appAddress, rmHaIds); + yarnEnabled = true; + logger.info("application url : {}", appUrl); + } else { + //single resourcemanager enabled + yarnEnabled = true; + } + + return String.format(appUrl, applicationId); } /** @@ -484,13 +493,6 @@ public class HadoopUtils implements Closeable { return String.format("%s/udfs", getHdfsTenantDir(tenantCode)); } - /** - * get absolute path and name for file on hdfs - * - * @param tenantCode tenant code - * @param fileName file name - * @return get absolute path and name for file on hdfs - */ /** * get hdfs file name diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java index 7de198f28b..98d9cf16ec 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java @@ -40,7 +40,7 @@ public class HttpUtils { /** * get http request content * @param url url - * @return http response + * @return http get request response content */ public static String get(String url){ CloseableHttpClient httpclient = HttpClients.createDefault(); diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index c6484ff998..db3b241ca9 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -19,22 +19,22 @@ resource.storage.type=NONE # resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended -#resource.upload.path=/dolphinscheduler +resource.upload.path=/dolphinscheduler # user data local directory path, please make sure the directory exists and have read write permissions #data.basedir.path=/tmp/dolphinscheduler # whether kerberos starts -#hadoop.security.authentication.startup.state=false +hadoop.security.authentication.startup.state=false # java.security.krb5.conf path -#java.security.krb5.conf.path=/opt/krb5.conf +java.security.krb5.conf.path=/opt/krb5.conf -# loginUserFromKeytab user -#login.user.keytab.username=hdfs-mycluster@ESZ.COM +# login user from keytab username +login.user.keytab.username=hdfs-mycluster@ESZ.COM # loginUserFromKeytab path -#login.user.keytab.path=/opt/hdfs.headless.keytab +login.user.keytab.path=/opt/hdfs.headless.keytab #resource.view.suffixs #resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties @@ -46,21 +46,21 @@ hdfs.root.user=hdfs fs.defaultFS=hdfs://mycluster:8020 # if resource.storage.type=S3，s3 endpoint -#fs.s3a.endpoint=http://192.168.199.91:9010 +fs.s3a.endpoint=http://192.168.199.91:9010 # if resource.storage.type=S3，s3 access key -#fs.s3a.access.key=A3DXS30FO22544RE +fs.s3a.access.key=A3DXS30FO22544RE # if resource.storage.type=S3，s3 secret key -#fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK +fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK -# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty TODO +# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx -# If resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ark1 to actual resourcemanager hostname. +# if resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ark1 to actual resourcemanager hostname. yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s -# system env path. self configuration, please make sure the directory and file exists and have read write execute permissions, TODO +# system env path #dolphinscheduler.env.path=env/dolphinscheduler_env.sh kerberos.expire.time=7 \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java index b7bf2209d6..00b8f1c5c6 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java @@ -17,88 +17,186 @@ package org.apache.dolphinscheduler.common.utils; import org.apache.dolphinscheduler.common.enums.ResourceType; -import org.junit.Ignore; +import org.apache.hadoop.conf.Configuration; +import org.junit.Assert; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; -@Ignore +@RunWith(MockitoJUnitRunner.class) +//todo there is no hadoop environment public class HadoopUtilsTest { private static final Logger logger = LoggerFactory.getLogger(HadoopUtilsTest.class); + private HadoopUtils hadoopUtils = HadoopUtils.getInstance(); @Test public void getActiveRMTest() { - logger.info(HadoopUtils.getAppAddress("http://ark1:8088/ws/v1/cluster/apps/%s","192.168.xx.xx,192.168.xx.xx")); + try{ + hadoopUtils.getAppAddress("http://ark1:8088/ws/v1/cluster/apps/%s","192.168.xx.xx,192.168.xx.xx"); + } catch (Exception e) { + logger.error(e.getMessage(),e); + } } @Test - public void getApplicationStatusAddressTest(){ - logger.info(HadoopUtils.getInstance().getApplicationUrl("application_1548381297012_0030")); + public void rename() { + + boolean result = false; + try { + result = hadoopUtils.rename("/dolphinscheduler/hdfs1","/dolphinscheduler/hdfs2"); + } catch (Exception e) { + logger.error(e.getMessage(),e); + } + Assert.assertEquals(false, result); } + @Test - public void test() throws IOException { - HadoopUtils.getInstance().copyLocalToHdfs("/root/teamviewer_13.1.8286.x86_64.rpm", "/journey", true, true); + public void getConfiguration(){ + Configuration conf = hadoopUtils.getConfiguration(); + } @Test - public void readFileTest(){ + public void mkdir() { + boolean result = false; try { - byte[] bytes = HadoopUtils.getInstance().catFile("/dolphinscheduler/hdfs/resources/35435.sh"); - logger.info(new String(bytes)); + result = hadoopUtils.mkdir("/dolphinscheduler/hdfs"); } catch (Exception e) { - + logger.error(e.getMessage(), e); } + Assert.assertEquals(false, result); } - @Test - public void testCapacity(){ + @Test + public void delete() { + boolean result = false; + try { + result = hadoopUtils.delete("/dolphinscheduler/hdfs",true); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + Assert.assertEquals(false, result); } + @Test - public void testMove(){ - HadoopUtils instance = HadoopUtils.getInstance(); + public void exists() { + boolean result = false; try { - instance.copy("/opt/apptest/test.dat","/opt/apptest/test.dat.back",true,true); + result = hadoopUtils.exists("/dolphinscheduler/hdfs"); } catch (Exception e) { logger.error(e.getMessage(), e); } + Assert.assertEquals(false, result); + } + @Test + public void getHdfsDataBasePath() { + String result = hadoopUtils.getHdfsDataBasePath(); + Assert.assertEquals("/dolphinscheduler", result); + } + @Test + public void getHdfsResDir() { + String result = hadoopUtils.getHdfsResDir("11000"); + Assert.assertEquals("/dolphinscheduler/11000/resources", result); } @Test - public void getApplicationStatus() { - logger.info(HadoopUtils.getInstance().getApplicationStatus("application_1542010131334_0029").toString()); + public void getHdfsUserDir() { + String result = hadoopUtils.getHdfsUserDir("11000",1000); + Assert.assertEquals("/dolphinscheduler/11000/home/1000", result); } @Test - public void getApplicationUrl(){ - String application_1516778421218_0042 = HadoopUtils.getInstance().getApplicationUrl("application_1529051418016_0167"); - logger.info(application_1516778421218_0042); + public void getHdfsUdfDir() { + String result = hadoopUtils.getHdfsUdfDir("11000"); + Assert.assertEquals("/dolphinscheduler/11000/udfs", result); } @Test - public void catFileTest()throws Exception{ - List stringList = HadoopUtils.getInstance().catFile("/dolphinscheduler/hdfs/resources/WCSparkPython.py", 0, 1000); - logger.info(String.join(",",stringList)); + public void getHdfsFileName() { + String result = hadoopUtils.getHdfsFileName(ResourceType.FILE,"11000","aa.txt"); + Assert.assertEquals("/dolphinscheduler/11000/resources/aa.txt", result); } @Test - public void getHdfsFileNameTest(){ - logger.info(HadoopUtils.getHdfsFileName(ResourceType.FILE,"test","/test")); + public void isYarnEnabled() { + boolean result = hadoopUtils.isYarnEnabled(); + Assert.assertEquals(false, result); } @Test - public void getHdfsResourceFileNameTest(){ - logger.info(HadoopUtils.getHdfsResourceFileName("test","/test")); + public void test() { + try { + hadoopUtils.copyLocalToHdfs("/root/teamviewer_13.1.8286.x86_64.rpm", "/journey", true, true); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } } @Test - public void getHdfsUdfFileNameTest(){ - logger.info(HadoopUtils.getHdfsUdfFileName("test","/test.jar")); + public void readFileTest(){ + try { + byte[] bytes = hadoopUtils.catFile("/dolphinscheduler/hdfs/resources/35435.sh"); + logger.info(new String(bytes)); + } catch (Exception e) { + logger.error(e.getMessage(),e); + } + } + + + @Test + public void testMove(){ + try { + hadoopUtils.copy("/opt/apptest/test.dat","/opt/apptest/test.dat.back",true,true); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + + } + + @Test + public void getApplicationStatus() { + try { + logger.info(hadoopUtils.getApplicationStatus("application_1542010131334_0029").toString()); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + @Test + public void getApplicationUrl(){ + String application_1516778421218_0042 = hadoopUtils.getApplicationUrl("application_1529051418016_0167"); + logger.info(application_1516778421218_0042); + } + + @Test + public void catFileWithLimitTest() { + List stringList = new ArrayList<>(); + try { + stringList = hadoopUtils.catFile("/dolphinscheduler/hdfs/resources/WCSparkPython.py", 0, 1000); + logger.info(String.join(",",stringList)); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + @Test + public void catFileTest() { + byte[] content = new byte[0]; + try { + content = hadoopUtils.catFile("/dolphinscheduler/hdfs/resources/WCSparkPython.py"); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + logger.info(Arrays.toString(content)); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/main/resources/datasource.properties b/dolphinscheduler-dao/src/main/resources/datasource.properties index 63c694e9a4..2f28ca2b0b 100644 --- a/dolphinscheduler-dao/src/main/resources/datasource.properties +++ b/dolphinscheduler-dao/src/main/resources/datasource.properties @@ -25,9 +25,6 @@ spring.datasource.url=jdbc:postgresql://localhost:5432/dolphinscheduler spring.datasource.username=test spring.datasource.password=test -## base spring data source configuration todo need to remove -#spring.datasource.type=com.alibaba.druid.pool.DruidDataSource - # connection configuration #spring.datasource.initialSize=5 # min connection number diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/dolphinscheduler-server/src/main/resources/config/install_config.conf index 0378490abb..4671be7371 100644 --- a/dolphinscheduler-server/src/main/resources/config/install_config.conf +++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf @@ -15,11 +15,126 @@ # limitations under the License. # -installPath=/data1_1T/dolphinscheduler -deployUser=dolphinscheduler -ips=ark0,ark1,ark2,ark3,ark4 -sshPort=22 -masters=ark0,ark1 -workers=ark2,ark3,ark4 -alertServer=ark3 -apiServers=ark1 + +# NOTICE : If the following config has special characters in the variable `.*[]^${}\+?|()@#&`, Please escape, for example, `[` escape to `\[` +# postgresql or mysql +dbtype="mysql" + +# db config +# db address and port +dbhost="192.168.xx.xx:3306" + +# db username +username="xx" + +# db passwprd +# NOTICE: if there are special characters, please use the \ to escape, for example, `[` escape to `\[` +password="xx" + +# zk cluster +zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181" + +# Note: the target installation path for dolphinscheduler, please not config as the same as the current path (pwd) +installPath="/data1_1T/dolphinscheduler" + +# deployment user +# Note: the deployment user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled, the root directory needs to be created by itself +deployUser="dolphinscheduler" + + +# alert config +# mail server host +mailServerHost="smtp.exmail.qq.com" + +# mail server port +# note: Different protocols and encryption methods correspond to different ports, when SSL/TLS is enabled, make sure the port is correct. +mailServerPort="25" + +# sender +mailSender="xxxxxxxxxx" + +# user +mailUser="xxxxxxxxxx" + +# sender password +# note: The mail.passwd is email service authorization code, not the email login password. +mailPassword="xxxxxxxxxx" + +# TLS mail protocol support +starttlsEnable="false" + +sslTrust="xxxxxxxxxx" + +# SSL mail protocol support +# note: The SSL protocol is enabled by default. +# only one of TLS and SSL can be in the true state. +sslEnable="true" + + +# resource storage type：HDFS,S3,NONE +resourceStorageType="NONE" + +# if resourceStorageType is HDFS，defaultFS write namenode address，HA you need to put core-site.xml and hdfs-site.xml in the conf directory. +# if S3，write S3 address，HA，for example ：s3a://dolphinscheduler， +# Note，s3 be sure to create the root directory /dolphinscheduler +defaultFS="hdfs://mycluster:8020" + +# if resourceStorageType is S3, the following three configuration is required, otherwise please ignore +s3Endpoint="http://192.168.xx.xx:9010" +s3AccessKey="xxxxxxxxxx" +s3SecretKey="xxxxxxxxxx" + +# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty +yarnHaIps="192.168.xx.xx,192.168.xx.xx" + +# if resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ark1 to actual resourcemanager hostname. +singleYarnIp="ark1" + +# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。/dolphinscheduler is recommended +resourceUploadPath="/dolphinscheduler" + +# who have permissions to create directory under HDFS/S3 root path +# Note: if kerberos is enabled, please config hdfsRootUser= +hdfsRootUser="hdfs" + +# kerberos config +# whether kerberos starts, if kerberos starts, following four items need to config, otherwise please ignore +kerberosStartUp="false" +# kdc krb5 config file path +krb5ConfPath="$installPath/conf/krb5.conf" +# keytab username +keytabUserName="hdfs-mycluster@ESZ.COM" +# username keytab path +keytabPath="$installPath/conf/hdfs.headless.keytab" + + +# api server port +apiServerPort="12345" + + +# install hosts +# Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname +ips="ark0,ark1,ark2,ark3,ark4" + +# ssh port, default 22 +# Note: if ssh port is not default, modify here +sshPort="22" + +# run master machine +# Note: list of hosts hostname for deploying master +masters="ark0,ark1" + +# run worker machine +# note: list of machine hostnames for deploying workers +workers="ark2,ark3,ark4" + +# run alert machine +# note: list of machine hostnames for deploying alert server +alertServer="ark3" + +# run api machine +# note: list of machine hostnames for deploying api server +apiServers="ark1" + +# whether to start monitoring self-starting scripts +monitorServerState="false" diff --git a/install.sh b/install.sh index ed66ce5bd9..20b293f697 100644 --- a/install.sh +++ b/install.sh @@ -19,309 +19,25 @@ workDir=`dirname $0` workDir=`cd ${workDir};pwd` -#To be compatible with MacOS and Linux +source ${workDir}/conf/config/install_config.conf + +# 1.replace file +echo "1.replace file" + txt="" if [[ "$OSTYPE" == "darwin"* ]]; then # Mac OSX txt="''" -elif [[ "$OSTYPE" == "linux-gnu" ]]; then - # linux - txt="" -elif [[ "$OSTYPE" == "cygwin" ]]; then - # POSIX compatibility layer and Linux environment emulation for Windows - echo "DolphinScheduler not support Windows operating system" - exit 1 -elif [[ "$OSTYPE" == "msys" ]]; then - # Lightweight shell and GNU utilities compiled for Windows (part of MinGW) - echo "DolphinScheduler not support Windows operating system" - exit 1 -elif [[ "$OSTYPE" == "win32" ]]; then - echo "DolphinScheduler not support Windows operating system" - exit 1 -elif [[ "$OSTYPE" == "freebsd"* ]]; then - # ... - txt="" -else - # Unknown. - echo "Operating system unknown, please tell us(submit issue) for better service" - exit 1 -fi - -source ${workDir}/conf/config/install_config.conf - -# for example postgresql or mysql ... -dbtype="postgresql" - -# db config -# db address and port -dbhost="192.168.xx.xx:5432" - -# db name -dbname="dolphinscheduler" - -# db username -username="xx" - -# db passwprd -# Note: if there are special characters, please use the \ transfer character to transfer -passowrd="xx" - -# conf/config/install_config.conf config -# Note: the installation path is not the same as the current path (pwd) -installPath="/data1_1T/dolphinscheduler" - -# deployment user -# Note: the deployment user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled, the root directory needs to be created by itself -deployUser="dolphinscheduler" - -# zk cluster -zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181" - -# install hosts -# Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname -ips="ark0,ark1,ark2,ark3,ark4" - -# ssh port, default 22 -# Note: if ssh port is not default, modify here -sshPort=22 - -# run master machine -# Note: list of hosts hostname for deploying master -masters="ark0,ark1" - -# run worker machine -# note: list of machine hostnames for deploying workers -workers="ark2,ark3,ark4" - -# run alert machine -# note: list of machine hostnames for deploying alert server -alertServer="ark3" - -# run api machine -# note: list of machine hostnames for deploying api server -apiServers="ark1" - -# alert config -# mail protocol -mailProtocol="SMTP" - -# mail server host -mailServerHost="smtp.exmail.qq.com" - -# mail server port -# note: Different protocols and encryption methods correspond to different ports, when SSL/TLS is enabled, make sure the port is correct. -mailServerPort="25" - -# sender -mailSender="xxxxxxxxxx" - -# user -mailUser="xxxxxxxxxx" - -# sender password -# note: The mail.passwd is email service authorization code, not the email login password. -mailPassword="xxxxxxxxxx" - -# TLS mail protocol support -starttlsEnable="false" - -sslTrust="xxxxxxxxxx" - -# SSL mail protocol support -# note: The SSL protocol is enabled by default. -# only one of TLS and SSL can be in the true state. -sslEnable="true" - -# download excel path -xlsFilePath="/tmp/xls" - -# Enterprise WeChat Enterprise ID Configuration -enterpriseWechatCorpId="xxxxxxxxxx" - -# Enterprise WeChat application Secret configuration -enterpriseWechatSecret="xxxxxxxxxx" - -# Enterprise WeChat Application AgentId Configuration -enterpriseWechatAgentId="xxxxxxxxxx" - -# Enterprise WeChat user configuration, multiple users to , split -enterpriseWechatUsers="xxxxx,xxxxx" - - -# whether to start monitoring self-starting scripts -monitorServerState="false" - -# resource Center upload and select storage method：HDFS,S3,NONE -resUploadStartupType="NONE" - -# if resUploadStartupType is HDFS，defaultFS write namenode address，HA you need to put core-site.xml and hdfs-site.xml in the conf directory. -# if S3，write S3 address，HA，for example ：s3a://dolphinscheduler， -# Note，s3 be sure to create the root directory /dolphinscheduler -defaultFS="hdfs://mycluster:8020" - -# if S3 is configured, the following configuration is required. -s3Endpoint="http://192.168.xx.xx:9010" -s3AccessKey="xxxxxxxxxx" -s3SecretKey="xxxxxxxxxx" - -# resourcemanager HA configuration, if it is a single resourcemanager, here is yarnHaIps="" -yarnHaIps="192.168.xx.xx,192.168.xx.xx" - -# if it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine. -singleYarnIp="ark1" - -# hdfs root path, the owner of the root path must be the deployment user. -# versions prior to 1.1.0 do not automatically create the hdfs root directory, you need to create it yourself. -hdfsPath="/dolphinscheduler" - -# have users who create directory permissions under hdfs root path / -# Note: if kerberos is enabled, hdfsRootUser="" can be used directly. -hdfsRootUser="hdfs" - -# common config -# Program root path -programPath="/tmp/dolphinscheduler" - -# download path -downloadPath="/tmp/dolphinscheduler/download" - -# task execute path -execPath="/tmp/dolphinscheduler/exec" - -# SHELL environmental variable path -shellEnvPath="$installPath/conf/env/dolphinscheduler_env.sh" - -# suffix of the resource file -resSuffixs="txt,log,sh,conf,cfg,py,java,sql,hql,xml" - -# development status, if true, for the SHELL script, you can view the encapsulated SHELL script in the execPath directory. -# If it is false, execute the direct delete -devState="true" - -# kerberos config -# kerberos whether to start -kerberosStartUp="false" - -# kdc krb5 config file path -krb5ConfPath="$installPath/conf/krb5.conf" - -# keytab username -keytabUserName="hdfs-mycluster@ESZ.COM" - -# username keytab path -keytabPath="$installPath/conf/hdfs.headless.keytab" - -# zk config -# zk root directory -zkRoot="/dolphinscheduler" - -# zk session timeout -zkSessionTimeout="300" - -# zk connection timeout -zkConnectionTimeout="300" - -# zk retry interval -zkRetryMaxSleep="100" - -# zk retry maximum number of times -zkRetryMaxtime="5" - - -# master config -# master execution thread maximum number, maximum parallelism of process instance -masterExecThreads="100" - -# the maximum number of master task execution threads, the maximum degree of parallelism for each process instance -masterExecTaskNum="20" - -# master heartbeat interval -masterHeartbeatInterval="10" - -# master task submission retries -masterTaskCommitRetryTimes="5" - -# master task submission retry interval -masterTaskCommitInterval="1000" - -# master maximum cpu average load, used to determine whether the master has execution capability -masterMaxCpuLoadAvg="100" - -# master reserve memory to determine if the master has execution capability -masterReservedMemory="0.1" - -# worker config -# worker execution thread -workerExecThreads="100" - -# worker heartbeat interval -workerHeartbeatInterval="10" - -# worker number of fetch tasks -workerFetchTaskNum="3" - -# worker reserve memory to determine if the master has execution capability -workerReservedMemory="0.1" - -# api config -# api server port -apiServerPort="12345" - -# api session timeout -apiServerSessionTimeout="7200" - -# api server context path -apiServerContextPath="/dolphinscheduler/" - -# spring max file size -springMaxFileSize="1024MB" - -# spring max request size -springMaxRequestSize="1024MB" - -# api max http post size -apiMaxHttpPostSize="5000000" - - -# 1,replace file -echo "1,replace file" -if [ $dbtype == "mysql" ];then - sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application.properties - sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application.properties - sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application.properties - sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=com.mysql.jdbc.Driver#g" conf/application.properties - - sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${username}#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.dataSource.myDs.password.*#org.quartz.dataSource.myDs.password=${passowrd}#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.dataSource.myDs.driver.*#org.quartz.dataSource.myDs.driver=com.mysql.jdbc.Driver#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.jobStore.driverDelegateClass.*#org.quartz.jobStore.driverDelegateClass=org.quartz.impl.jdbcjobstore.StdJDBCDelegate#g" conf/quartz.properties fi +datasourceDriverClassname="com.mysql.jdbc.Driver" if [ $dbtype == "postgresql" ];then - sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application.properties - sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application.properties - sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application.properties - sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=org.postgresql.Driver#g" conf/application.properties - - sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${username}#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.dataSource.myDs.password.*#org.quartz.dataSource.myDs.password=${passowrd}#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.dataSource.myDs.driver.*#org.quartz.dataSource.myDs.driver=org.postgresql.Driver#g" conf/quartz.properties - sed -i ${txt} "s#org.quartz.jobStore.driverDelegateClass.*#org.quartz.jobStore.driverDelegateClass=org.quartz.impl.jdbcjobstore.PostgreSQLDelegate#g" conf/quartz.properties + datasourceDriverClassname="org.postgresql.Driver" fi - -sed -i ${txt} "s#master.exec.threads.*#master.exec.threads=${masterExecThreads}#g" conf/application.properties -sed -i ${txt} "s#master.exec.task.num.*#master.exec.task.num=${masterExecTaskNum}#g" conf/application.properties -sed -i ${txt} "s#master.heartbeat.interval.*#master.heartbeat.interval=${masterHeartbeatInterval}#g" conf/application.properties -sed -i ${txt} "s#master.task.commit.retryTimes.*#master.task.commit.retryTimes=${masterTaskCommitRetryTimes}#g" conf/application.properties -sed -i ${txt} "s#master.task.commit.interval.*#master.task.commit.interval=${masterTaskCommitInterval}#g" conf/application.properties -sed -i ${txt} "s#master.reserved.memory.*#master.reserved.memory=${masterReservedMemory}#g" conf/application.properties - -sed -i ${txt} "s#worker.exec.threads.*#worker.exec.threads=${workerExecThreads}#g" conf/application.properties -sed -i ${txt} "s#worker.heartbeat.interval.*#worker.heartbeat.interval=${workerHeartbeatInterval}#g" conf/application.properties -sed -i ${txt} "s#worker.fetch.task.num.*#worker.fetch.task.num=${workerFetchTaskNum}#g" conf/application.properties -sed -i ${txt} "s#worker.reserved.memory.*#worker.reserved.memory=${workerReservedMemory}#g" conf/application.properties +sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=${datasourceDriverClassname}#g" conf/datasource.properties +sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:${dbtype}://${dbhost}/dolphinscheduler?characterEncoding=UTF-8#g" conf/datasource.properties +sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/datasource.properties +sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${password}#g" conf/datasource.properties sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g" conf/common.properties sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g" conf/common.properties @@ -329,37 +45,18 @@ sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g" conf/co sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g" conf/common.properties sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common.properties sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common.properties - -sed -i ${txt} "s#data.basedir.path.*#data.basedir.path=${programPath}#g" conf/common.properties -sed -i ${txt} "s#data.download.basedir.path.*#data.download.basedir.path=${downloadPath}#g" conf/common.properties -sed -i ${txt} "s#process.exec.basepath.*#process.exec.basepath=${execPath}#g" conf/common.properties sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common.properties -sed -i ${txt} "s#data.store2hdfs.basepath.*#data.store2hdfs.basepath=${hdfsPath}#g" conf/common.properties -sed -i ${txt} "s#res.upload.startup.type.*#res.upload.startup.type=${resUploadStartupType}#g" conf/common.properties -sed -i ${txt} "s#dolphinscheduler.env.path.*#dolphinscheduler.env.path=${shellEnvPath}#g" conf/common.properties -sed -i ${txt} "s#resource.view.suffixs.*#resource.view.suffixs=${resSuffixs}#g" conf/common.properties -sed -i ${txt} "s#development.state.*#development.state=${devState}#g" conf/common.properties +sed -i ${txt} "s#resource.upload.path.*#resource.upload.path=${resourceUploadPath}#g" conf/common.properties +sed -i ${txt} "s#resource.storage.type.*#resource.storage.type=${resourceStorageType}#g" conf/common.properties sed -i ${txt} "s#hadoop.security.authentication.startup.state.*#hadoop.security.authentication.startup.state=${kerberosStartUp}#g" conf/common.properties sed -i ${txt} "s#java.security.krb5.conf.path.*#java.security.krb5.conf.path=${krb5ConfPath}#g" conf/common.properties sed -i ${txt} "s#login.user.keytab.username.*#login.user.keytab.username=${keytabUserName}#g" conf/common.properties sed -i ${txt} "s#login.user.keytab.path.*#login.user.keytab.path=${keytabPath}#g" conf/common.properties - sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/common.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/common.properties -sed -i ${txt} "s#zookeeper.session.timeout.*#zookeeper.session.timeout=${zkSessionTimeout}#g" conf/common.properties -sed -i ${txt} "s#zookeeper.connection.timeout.*#zookeeper.connection.timeout=${zkConnectionTimeout}#g" conf/common.properties -sed -i ${txt} "s#zookeeper.retry.max.sleep.*#zookeeper.retry.max.sleep=${zkRetryMaxSleep}#g" conf/common.properties -sed -i ${txt} "s#zookeeper.retry.maxtime.*#zookeeper.retry.maxtime=${zkRetryMaxtime}#g" conf/common.properties sed -i ${txt} "s#server.port.*#server.port=${apiServerPort}#g" conf/application-api.properties -sed -i ${txt} "s#server.servlet.session.timeout.*#server.servlet.session.timeout=${apiServerSessionTimeout}#g" conf/application-api.properties -sed -i ${txt} "s#server.servlet.context-path.*#server.servlet.context-path=${apiServerContextPath}#g" conf/application-api.properties -sed -i ${txt} "s#spring.servlet.multipart.max-file-size.*#spring.servlet.multipart.max-file-size=${springMaxFileSize}#g" conf/application-api.properties -sed -i ${txt} "s#spring.servlet.multipart.max-request-size.*#spring.servlet.multipart.max-request-size=${springMaxRequestSize}#g" conf/application-api.properties -sed -i ${txt} "s#server.jetty.max-http-post-size.*#server.jetty.max-http-post-size=${apiMaxHttpPostSize}#g" conf/application-api.properties -sed -i ${txt} "s#mail.protocol.*#mail.protocol=${mailProtocol}#g" conf/alert.properties sed -i ${txt} "s#mail.server.host.*#mail.server.host=${mailServerHost}#g" conf/alert.properties sed -i ${txt} "s#mail.server.port.*#mail.server.port=${mailServerPort}#g" conf/alert.properties sed -i ${txt} "s#mail.sender.*#mail.sender=${mailSender}#g" conf/alert.properties @@ -368,85 +65,38 @@ sed -i ${txt} "s#mail.passwd.*#mail.passwd=${mailPassword}#g" conf/alert.propert sed -i ${txt} "s#mail.smtp.starttls.enable.*#mail.smtp.starttls.enable=${starttlsEnable}#g" conf/alert.properties sed -i ${txt} "s#mail.smtp.ssl.trust.*#mail.smtp.ssl.trust=${sslTrust}#g" conf/alert.properties sed -i ${txt} "s#mail.smtp.ssl.enable.*#mail.smtp.ssl.enable=${sslEnable}#g" conf/alert.properties -sed -i ${txt} "s#xls.file.path.*#xls.file.path=${xlsFilePath}#g" conf/alert.properties -sed -i ${txt} "s#enterprise.wechat.corp.id.*#enterprise.wechat.corp.id=${enterpriseWechatCorpId}#g" conf/alert.properties -sed -i ${txt} "s#enterprise.wechat.secret.*#enterprise.wechat.secret=${enterpriseWechatSecret}#g" conf/alert.properties -sed -i ${txt} "s#enterprise.wechat.agent.id.*#enterprise.wechat.agent.id=${enterpriseWechatAgentId}#g" conf/alert.properties -sed -i ${txt} "s#enterprise.wechat.users.*#enterprise.wechat.users=${enterpriseWechatUsers}#g" conf/alert.properties - - - -sed -i ${txt} "s#installPath.*#installPath=${installPath}#g" conf/config/install_config.conf -sed -i ${txt} "s#deployUser.*#deployUser=${deployUser}#g" conf/config/install_config.conf -sed -i ${txt} "s#ips.*#ips=${ips}#g" conf/config/install_config.conf -sed -i ${txt} "s#sshPort.*#sshPort=${sshPort}#g" conf/config/install_config.conf - - -sed -i ${txt} "s#masters.*#masters=${masters}#g" conf/config/install_config.conf -sed -i ${txt} "s#workers.*#workers=${workers}#g" conf/config/install_config.conf -sed -i ${txt} "s#alertServer.*#alertServer=${alertServer}#g" conf/config/install_config.conf -sed -i ${txt} "s#apiServers.*#apiServers=${apiServers}#g" conf/config/install_config.conf -sed -i ${txt} "s#sshPort.*#sshPort=${sshPort}#g" conf/config/install_config.conf - -# 2,create directory -echo "2,create directory" +# 2.create directory +echo "2.create directory" if [ ! -d $installPath ];then sudo mkdir -p $installPath sudo chown -R $deployUser:$deployUser $installPath fi -hostsArr=(${ips//,/ }) -for host in ${hostsArr[@]} -do - -# create if programPath does not exist -if ! ssh -p $sshPort $host test -e $programPath; then - ssh -p $sshPort $host "sudo mkdir -p $programPath;sudo chown -R $deployUser:$deployUser $programPath" -fi - -# create if downloadPath does not exist -if ! ssh -p $sshPort $host test -e $downloadPath; then - ssh -p $sshPort $host "sudo mkdir -p $downloadPath;sudo chown -R $deployUser:$deployUser $downloadPath" -fi - -# create if execPath does not exist -if ! ssh -p $sshPort $host test -e $execPath; then - ssh -p $sshPort $host "sudo mkdir -p $execPath; sudo chown -R $deployUser:$deployUser $execPath" -fi - -# create if xlsFilePath does not exist -if ! ssh -p $sshPort $host test -e $xlsFilePath; then - ssh -p $sshPort $host "sudo mkdir -p $xlsFilePath; sudo chown -R $deployUser:$deployUser $xlsFilePath" -fi - -done - - -# 3,scp resources -echo "3,scp resources" +# 3.scp resources +echo "3.scp resources" sh ${workDir}/script/scp-hosts.sh if [ $? -eq 0 ] then echo 'scp copy completed' else echo 'scp copy failed to exit' - exit -1 + exit 1 fi -# 4,stop server -echo "4,stop server" +# 4.stop server +echo "4.stop server" sh ${workDir}/script/stop-all.sh -# 5,delete zk node -echo "5,delete zk node" +# 5.delete zk node +echo "5.delete zk node" sh ${workDir}/script/remove-zk-node.sh $zkRoot -# 6,startup -echo "6,startup" +# 6.startup +echo "6.startup" sh ${workDir}/script/start-all.sh \ No newline at end of file diff --git a/pom.xml b/pom.xml index f8cbe7fca2..053652fc92 100644 --- a/pom.xml +++ b/pom.xml @@ -760,8 +760,9 @@ **/common/utils/StringTest.java **/common/utils/StringUtilsTest.java **/common/utils/TaskParametersUtilsTest.java + **/common/utils/HadoopUtilsTest.java + **/common/utils/HttpUtilsTest.java **/common/ConstantsTest.java - **/common/utils/HadoopUtils.java **/dao/mapper/AccessTokenMapperTest.java **/dao/mapper/AlertGroupMapperTest.java **/dao/mapper/CommandMapperTest.java diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index ab3b5c4bd5..19669e5ede 100644 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -35,6 +35,8 @@ BIN_DIR=`dirname $0` BIN_DIR=`cd "$BIN_DIR"; pwd` DOLPHINSCHEDULER_HOME=$BIN_DIR/.. +source /etc/profile + export JAVA_HOME=$JAVA_HOME #export JAVA_HOME=/opt/soft/jdk export HOSTNAME=`hostname` @@ -90,8 +92,8 @@ case $startStop in exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS" - echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 > /dev/null &" - nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 > /dev/null & + echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &" + nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 & echo $! > $pid ;; diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh index 05878c3c51..f4949f311a 100644 --- a/script/scp-hosts.sh +++ b/script/scp-hosts.sh @@ -24,16 +24,18 @@ hostsArr=(${ips//,/ }) for host in ${hostsArr[@]} do - if ! ssh -p $sshPort $host test -e $installPath; then - ssh -p $sshPort $host "sudo mkdir -p $installPath; sudo chown -R $deployUser:$deployUser $installPath" - fi + if ! ssh -p $sshPort $host test -e $installPath; then + ssh -p $sshPort $host "sudo mkdir -p $installPath; sudo chown -R $deployUser:$deployUser $installPath" + fi + echo "scp dirs to $host/$installPath starting" ssh -p $sshPort $host "cd $installPath/; rm -rf bin/ conf/ lib/ script/ sql/ ui/" - scp -P $sshPort -r $workDir/../bin $host:$installPath - scp -P $sshPort -r $workDir/../conf $host:$installPath - scp -P $sshPort -r $workDir/../lib $host:$installPath - scp -P $sshPort -r $workDir/../script $host:$installPath - scp -P $sshPort -r $workDir/../sql $host:$installPath - scp -P $sshPort -r $workDir/../ui $host:$installPath - scp -P $sshPort $workDir/../install.sh $host:$installPath + + for dsDir in bin conf lib script sql ui install.sh + do + echo "start to scp $dsDir to $host/$installPath" + scp -P $sshPort -r $workDir/../$dsDir $host:$installPath + done + + echo "scp dirs to $host/$installPath complete" done diff --git a/script/start-all.sh b/script/start-all.sh index bb4b0a1fcb..11e4572059 100644 --- a/script/start-all.sh +++ b/script/start-all.sh @@ -23,7 +23,7 @@ source $workDir/../conf/config/install_config.conf mastersHost=(${masters//,/ }) for master in ${mastersHost[@]} do - echo $master + echo "$master master server is starting" ssh -p $sshPort $master "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start master-server;" done @@ -31,10 +31,10 @@ done workersHost=(${workers//,/ }) for worker in ${workersHost[@]} do - echo $worker + echo "$worker worker server is starting" - ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start worker-server;" - ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start logger-server;" + ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start worker-server;" + ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start logger-server;" done ssh -p $sshPort $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start alert-server;" @@ -42,8 +42,7 @@ ssh -p $sshPort $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon. apiServersHost=(${apiServers//,/ }) for apiServer in ${apiServersHost[@]} do - echo $apiServer - - ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start api-server;" + echo "$apiServer worker server is starting" + ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start api-server;" done diff --git a/script/stop-all.sh b/script/stop-all.sh index c0c6f4dd45..f761579cc8 100644 --- a/script/stop-all.sh +++ b/script/stop-all.sh @@ -24,7 +24,7 @@ source $workDir/../conf/config/install_config.conf mastersHost=(${masters//,/ }) for master in ${mastersHost[@]} do - echo $master + echo "$master master server is stopping" ssh -p $sshPort $master "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop master-server;" done @@ -32,10 +32,9 @@ done workersHost=(${workers//,/ }) for worker in ${workersHost[@]} do - echo $worker - - ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop worker-server;" - ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop logger-server;" + echo "$worker worker server is stopping" + ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop worker-server;" + ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop logger-server;" done ssh -p $sshPort $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop alert-server;" @@ -43,8 +42,7 @@ ssh -p $sshPort $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon. apiServersHost=(${apiServers//,/ }) for apiServer in ${apiServersHost[@]} do - echo $apiServer - - ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop api-server;" + echo "$apiServer worker server is stopping" + ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop api-server;" done From 3740b2410e62dcb57a878c8620a88e7c8db63072 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Tue, 21 Apr 2020 10:48:32 +0800 Subject: [PATCH 20/23] Modify worker parameters (#2479) * Fix the problem of data echo in script edit box * Optimize resource tree * Change the name of the shell node and modify the parameter transmission method of spark, mr, python, and flink nodes * Repair naming * Modify list style * Online editing is prohibited * Change the background color of the disabled state of the radio button when the workflow goes online * Modify worker parameters Co-authored-by: dailidong Co-authored-by: qiaozhanwei --- .../pages/dag/_source/formModel/formModel.vue | 26 ++++++++++--------- .../instance/pages/list/_source/list.vue | 4 +-- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index a2b3ad9bae..459f9a8605 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -611,25 +611,27 @@ this.failedBranch = o.conditionResult.failedNode[0] } // If the workergroup has been deleted, set the default workergroup - var hasMatch = false; - for (let i = 0; i < this.store.state.security.workerGroupsListAll.length; i++) { - var workerGroup = this.store.state.security.workerGroupsListAll[i].id - if (o.workerGroup == workerGroup) { - hasMatch = true; - break; - } + var hasMatch = false; + for (let i = 0; i < this.store.state.security.workerGroupsListAll.length; i++) { + var workerGroup = this.store.state.security.workerGroupsListAll[i].id + if (o.workerGroup == workerGroup) { + hasMatch = true; + break; } + } - if(!hasMatch){ - this.workerGroup = 'default' - }else{ - this.workerGroup = o.workerGroup - } + if(!hasMatch){ + this.workerGroup = 'default' + } else { + this.workerGroup = o.workerGroup + } this.params = o.params || {} this.dependence = o.dependence || {} this.cacheDependence = o.dependence || {} + } else { + this.workerGroup = this.store.state.security.workerGroupsListAll[0].id } this.isContentBox = true }, diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue index 641c7aaa63..9019e697d0 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue @@ -311,9 +311,9 @@ name: 'list', data () { return { - // 数据 + // data list: [], - // 按钮类型 + // btn type buttonType: '', strDelete: '', checkAll: false From f3dc1c94ba0ac3f6268da59b119feeaeeaa1863b Mon Sep 17 00:00:00 2001 From: zhangchunyang <18910529250@163.com> Date: Tue, 21 Apr 2020 11:08:04 +0800 Subject: [PATCH 21/23] resolve #2474 (#2477) * In the dev branch, in order to the RPM package which is printed under the dist module contain configuration files: master.properties, worker.properties * Restore files deleted by mistake Co-authored-by: zhangchunyang Co-authored-by: qiaozhanwei --- dolphinscheduler-dist/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/dolphinscheduler-dist/pom.xml b/dolphinscheduler-dist/pom.xml index 0ead2b9880..f4b8d2b7ab 100644 --- a/dolphinscheduler-dist/pom.xml +++ b/dolphinscheduler-dist/pom.xml @@ -257,6 +257,7 @@ config/*.* + **/*.xml From 28766df53fac118dd0740ad33f2fc6b52e014670 Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Tue, 21 Apr 2020 14:40:19 +0800 Subject: [PATCH 22/23] send mail error, #2466 bug fix (#2470) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * dispatch task fail will set task status failed * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * send mail error, #2466 bug fix * send mail error, #2466 bug fix * send mail error, #2466 bug fix * send mail error, #2466 bug fix Co-authored-by: qiaozhanwei --- .../alert/utils/EnterpriseWeChatUtils.java | 3 +- .../utils/EnterpriseWeChatUtilsTest.java | 269 +++++++++++++----- 2 files changed, 203 insertions(+), 69 deletions(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java index 900c120cd4..170c0dd37e 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java @@ -48,7 +48,7 @@ public class EnterpriseWeChatUtils { private static final String ENTERPRISE_WE_CHAT_SECRET = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_SECRET); private static final String ENTERPRISE_WE_CHAT_TOKEN_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TOKEN_URL); - private static final String ENTERPRISE_WE_CHAT_TOKEN_URL_REPLACE = ENTERPRISE_WE_CHAT_TOKEN_URL + private static final String ENTERPRISE_WE_CHAT_TOKEN_URL_REPLACE = ENTERPRISE_WE_CHAT_TOKEN_URL == null ? null : ENTERPRISE_WE_CHAT_TOKEN_URL .replaceAll("\\$corpId", ENTERPRISE_WE_CHAT_CORP_ID) .replaceAll("\\$secret", ENTERPRISE_WE_CHAT_SECRET); @@ -266,5 +266,4 @@ public class EnterpriseWeChatUtils { return result; } - } diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java index 15b92a622e..852d245a2e 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java @@ -17,13 +17,21 @@ package org.apache.dolphinscheduler.alert.utils; import com.alibaba.fastjson.JSON; +import org.apache.dolphinscheduler.common.enums.AlertType; +import org.apache.dolphinscheduler.common.enums.ShowType; +import org.apache.dolphinscheduler.dao.entity.Alert; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; +import java.util.*; /** * Please manually modify the configuration file before testing. @@ -36,84 +44,211 @@ import java.util.Collection; * enterprise.wechat.agent.id * enterprise.wechat.users */ -@Ignore +@PrepareForTest(PropertyUtils.class) +@RunWith(PowerMockRunner.class) public class EnterpriseWeChatUtilsTest { - private String agentId = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_AGENT_ID); // app id - private Collection listUserId = Arrays.asList(PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USERS).split(",")); + private static final String toParty = "wwc99134b6fc1edb6"; + private static final String enterpriseWechatSecret = "Uuv2KFrkdf7SeKOsTDCpsTkpawXBMNRhFy6VKX5FV"; + private static final String enterpriseWechatAgentId = "1000004"; + private static final String enterpriseWechatUsers="LiGang,journey"; + private static final String msg = "hello world"; + + + @Test + public void testIsEnable(){ + PowerMockito.mockStatic(PropertyUtils.class); + Mockito.when(PropertyUtils.getBoolean(Constants.ENTERPRISE_WECHAT_ENABLE)).thenReturn(true); + Boolean weChartEnable = EnterpriseWeChatUtils.isEnable(); + Assert.assertTrue(weChartEnable); + } + + + @Test + public void testMakeTeamSendMsg1(){ + String sendMsg = EnterpriseWeChatUtils.makeTeamSendMsg(toParty, enterpriseWechatSecret, msg); + Assert.assertTrue(sendMsg.contains(toParty)); + Assert.assertTrue(sendMsg.contains(enterpriseWechatSecret)); + Assert.assertTrue(sendMsg.contains(msg)); + + } + - // Please change - private String partyId = "2"; - private Collection listPartyId = Arrays.asList("2","4"); @Test - public void testSendSingleTeamWeChat() { - try { - String token = EnterpriseWeChatUtils.getToken(); - String msg = EnterpriseWeChatUtils.makeTeamSendMsg(partyId, agentId, "hello world"); - String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); - - String errmsg = JSON.parseObject(resp).getString("errmsg"); - Assert.assertEquals("ok",errmsg); - } catch (IOException e) { - e.printStackTrace(); - } + public void testMakeTeamSendMsg2(){ + List parties = new ArrayList<>(); + parties.add(toParty); + parties.add("test1"); + + String sendMsg = EnterpriseWeChatUtils.makeTeamSendMsg(parties, enterpriseWechatSecret, msg); + Assert.assertTrue(sendMsg.contains(toParty)); + Assert.assertTrue(sendMsg.contains(enterpriseWechatSecret)); + Assert.assertTrue(sendMsg.contains(msg)); } @Test - public void testSendMultiTeamWeChat() { - - try { - String token = EnterpriseWeChatUtils.getToken(); - String msg = EnterpriseWeChatUtils.makeTeamSendMsg(listPartyId, agentId, "hello world"); - String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); - - String errmsg = JSON.parseObject(resp).getString("errmsg"); - Assert.assertEquals("ok",errmsg); - } catch (IOException e) { - e.printStackTrace(); - } + public void tesMakeUserSendMsg1(){ + String sendMsg = EnterpriseWeChatUtils.makeUserSendMsg(enterpriseWechatUsers, enterpriseWechatAgentId, msg); + Assert.assertTrue(sendMsg.contains(enterpriseWechatUsers)); + Assert.assertTrue(sendMsg.contains(enterpriseWechatAgentId)); + Assert.assertTrue(sendMsg.contains(msg)); } @Test - public void testSendSingleUserWeChat() { - try { - String token = EnterpriseWeChatUtils.getToken(); - String msg = EnterpriseWeChatUtils.makeUserSendMsg(listUserId.stream().findFirst().get(), agentId, "your meeting room has been booked and will be synced to the 'mailbox' later \n" + - ">**matter details** \n" + - ">matter：meeting
" + - ">organizer：@miglioguan \n" + - ">participant：@miglioguan、@kunliu、@jamdeezhou、@kanexiong、@kisonwang \n" + - "> \n" + - ">meeting room：Guangzhou TIT 1st Floor 301 \n" + - ">date：May 18, 2018 \n" + - ">time：9:00-11:00 am \n" + - "> \n" + - ">please attend the meeting on time\n" + - "> \n" + - ">to modify the meeting information, please click: [Modify Meeting Information](https://work.weixin.qq.com)\""); - - String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); - - String errmsg = JSON.parseObject(resp).getString("errmsg"); - Assert.assertEquals("ok",errmsg); - } catch (IOException e) { - e.printStackTrace(); - } + public void tesMakeUserSendMsg2(){ + List users = new ArrayList<>(); + users.add("user1"); + users.add("user2"); + + String sendMsg = EnterpriseWeChatUtils.makeUserSendMsg(users, enterpriseWechatAgentId, msg); + Assert.assertTrue(sendMsg.contains(users.get(0))); + Assert.assertTrue(sendMsg.contains(users.get(1))); + Assert.assertTrue(sendMsg.contains(enterpriseWechatAgentId)); + Assert.assertTrue(sendMsg.contains(msg)); } @Test - public void testSendMultiUserWeChat() { - try { - String token = EnterpriseWeChatUtils.getToken(); - - String msg = EnterpriseWeChatUtils.makeUserSendMsg(listUserId, agentId, "hello world"); - String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); - - String errmsg = JSON.parseObject(resp).getString("errmsg"); - Assert.assertEquals("ok",errmsg); - } catch (IOException e) { - e.printStackTrace(); - } + public void testMarkdownByAlertForText(){ + Alert alertForText = createAlertForText(); + String result = EnterpriseWeChatUtils.markdownByAlert(alertForText); + Assert.assertNotNull(result); } + @Test + public void testMarkdownByAlertForTable(){ + Alert alertForText = createAlertForTable(); + String result = EnterpriseWeChatUtils.markdownByAlert(alertForText); + Assert.assertNotNull(result); + } + + private Alert createAlertForText(){ + String content ="[\"id:69\"," + + "\"name:UserBehavior-0--1193959466\"," + + "\"Job name: Start workflow\"," + + "\"State: SUCCESS\"," + + "\"Recovery:NO\"," + + "\"Run time: 1\"," + + "\"Start time: 2018-08-06 10:31:34.0\"," + + "\"End time: 2018-08-06 10:31:49.0\"," + + "\"Host: 192.168.xx.xx\"," + + "\"Notify group :4\"]"; + + Alert alert = new Alert(); + alert.setTitle("Mysql Exception"); + alert.setShowType(ShowType.TEXT); + alert.setContent(content); + alert.setAlertType(AlertType.EMAIL); + alert.setAlertGroupId(4); + + return alert; + } + + private String list2String(){ + + LinkedHashMap map1 = new LinkedHashMap<>(); + map1.put("mysql service name","mysql200"); + map1.put("mysql address","192.168.xx.xx"); + map1.put("port","3306"); + map1.put("no index of number","80"); + map1.put("database client connections","190"); + + LinkedHashMap map2 = new LinkedHashMap<>(); + map2.put("mysql service name","mysql210"); + map2.put("mysql address","192.168.xx.xx"); + map2.put("port", "3306"); + map2.put("no index of number", "10"); + map2.put("database client connections", "90"); + + List> maps = new ArrayList<>(); + maps.add(0, map1); + maps.add(1, map2); + String mapjson = JSONUtils.toJsonString(maps); + return mapjson; + } + + private Alert createAlertForTable(){ + Alert alert = new Alert(); + alert.setTitle("Mysql Exception"); + alert.setShowType(ShowType.TABLE); + String content= list2String(); + alert.setContent(content); + alert.setAlertType(AlertType.EMAIL); + alert.setAlertGroupId(1); + return alert; + } + + + + +// @Test +// public void testSendSingleTeamWeChat() { +// try { +// String token = EnterpriseWeChatUtils.getToken(); +// String msg = EnterpriseWeChatUtils.makeTeamSendMsg(partyId, agentId, "hello world"); +// String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); +// +// String errmsg = JSON.parseObject(resp).getString("errmsg"); +// Assert.assertEquals("ok",errmsg); +// } catch (IOException e) { +// e.printStackTrace(); +// } +// } +// +// @Test +// public void testSendMultiTeamWeChat() { +// +// try { +// String token = EnterpriseWeChatUtils.getToken(); +// String msg = EnterpriseWeChatUtils.makeTeamSendMsg(listPartyId, agentId, "hello world"); +// String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); +// +// String errmsg = JSON.parseObject(resp).getString("errmsg"); +// Assert.assertEquals("ok",errmsg); +// } catch (IOException e) { +// e.printStackTrace(); +// } +// } +// +// @Test +// public void testSendSingleUserWeChat() { +// try { +// String token = EnterpriseWeChatUtils.getToken(); +// String msg = EnterpriseWeChatUtils.makeUserSendMsg(listUserId.stream().findFirst().get(), agentId, "your meeting room has been booked and will be synced to the 'mailbox' later \n" + +// ">**matter details** \n" + +// ">matter：meeting
" + +// ">organizer：@miglioguan \n" + +// ">participant：@miglioguan、@kunliu、@jamdeezhou、@kanexiong、@kisonwang \n" + +// "> \n" + +// ">meeting room：Guangzhou TIT 1st Floor 301 \n" + +// ">date：May 18, 2018 \n" + +// ">time：9:00-11:00 am \n" + +// "> \n" + +// ">please attend the meeting on time\n" + +// "> \n" + +// ">to modify the meeting information, please click: [Modify Meeting Information](https://work.weixin.qq.com)\""); +// +// String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); +// +// String errmsg = JSON.parseObject(resp).getString("errmsg"); +// Assert.assertEquals("ok",errmsg); +// } catch (IOException e) { +// e.printStackTrace(); +// } +// } +// +// @Test +// public void testSendMultiUserWeChat() { +// try { +// String token = EnterpriseWeChatUtils.getToken(); +// +// String msg = EnterpriseWeChatUtils.makeUserSendMsg(listUserId, agentId, "hello world"); +// String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); +// +// String errmsg = JSON.parseObject(resp).getString("errmsg"); +// Assert.assertEquals("ok",errmsg); +// } catch (IOException e) { +// e.printStackTrace(); +// } +// } + } From 54fc32e20714b1e6269d5cfb0279d9315a813c4f Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Tue, 21 Apr 2020 18:52:07 +0800 Subject: [PATCH 23/23] #2486 bug fix (#2487) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * dispatch task fail will set task status failed * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * send mail error, #2466 bug fix * send mail error, #2466 bug fix * send mail error, #2466 bug fix * send mail error, #2466 bug fix * #2486 bug fix Co-authored-by: qiaozhanwei --- .../server/worker/processor/TaskCallbackService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java index ecae9edc1c..1e8bf9d0e7 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java @@ -96,8 +96,9 @@ public class TaskCallbackService { return getRemoteChannel(newChannel, nettyRemoteChannel.getOpaque(), taskInstanceId); } logger.warn("original master : {} is not reachable, random select master", nettyRemoteChannel.getHost()); - Set masterNodes = zookeeperRegistryCenter.getMasterNodesDirectly(); + Set masterNodes = null; while (Stopper.isRunning()) { + masterNodes = zookeeperRegistryCenter.getMasterNodesDirectly(); if (CollectionUtils.isEmpty(masterNodes)) { logger.error("no available master node"); ThreadUtils.sleep(SLEEP_TIME_MILLIS);