From c441cd499d0a6c2ac536f2f5644122349e8767f5 Mon Sep 17 00:00:00 2001 From: lilin Date: Wed, 18 Sep 2019 22:03:31 +0800 Subject: [PATCH 01/53] misspell words --- docs/en_US/EasyScheduler Proposal.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en_US/EasyScheduler Proposal.md b/docs/en_US/EasyScheduler Proposal.md index 965605f63b..552067fe47 100644 --- a/docs/en_US/EasyScheduler Proposal.md +++ b/docs/en_US/EasyScheduler Proposal.md @@ -6,7 +6,7 @@ EasyScheduler is a distributed ETL scheduling engine with powerful DAG visualiza ## Proposal -EasyScheduler provides many easy-to-use features to accelerate the engineer enficiency on data ETL workflow job. We propose a new concept of 'instance of process' and 'instance of task' to let developers to tuning their jobs on the running state of workflow instead of changing the task's template. Its main objectives are as follows: +EasyScheduler provides many easy-to-use features to accelerate the engineer efficiency on data ETL workflow job. We propose a new concept of 'instance of process' and 'instance of task' to let developers to tuning their jobs on the running state of workflow instead of changing the task's template. Its main objectives are as follows: - Define the complex tasks' dependencies & triggers in a DAG graph by dragging and dropping. - Support cluster HA. From d1e7a796ceb684b2bbd22e6c9be7bc32a7419bfa Mon Sep 17 00:00:00 2001 From: lilin Date: Tue, 17 Dec 2019 20:50:48 +0800 Subject: [PATCH 02/53] modify common queue TaskQueueZKImplTest.java unit test --- .../common/queue/TaskQueueZkImpl.java | 6 +- .../common/queue/TaskQueueImplTest.java | 112 --------- .../common/queue/TaskQueueZKImplTest.java | 228 ++++++++++++++++++ pom.xml | 1 + 4 files changed, 230 insertions(+), 117 deletions(-) delete mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueImplTest.java create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java index 76d88868f2..b0ecc62710 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java @@ -145,7 +145,6 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue { public List poll(String key, int tasksNum) { try{ CuratorFramework zk = getZkClient(); - String tasksQueuePath = getTasksPath(key) + Constants.SINGLE_SLASH; List list = zk.getChildren().forPath(getTasksPath(key)); if(list != null && list.size() > 0){ @@ -155,7 +154,6 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue { int size = list.size(); - Set taskTreeSet = new TreeSet<>(new Comparator() { @Override public int compare(String o1, String o2) { @@ -183,7 +181,7 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue { String taskDetail = list.get(i); String[] taskDetailArrs = taskDetail.split(Constants.UNDERLINE); - //forward compatibility 向前版本兼容 + //forward compatibility if(taskDetailArrs.length >= 4){ //format ${processInstancePriority}_${processInstanceId}_${taskInstancePriority}_${taskId} @@ -201,9 +199,7 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue { formatTask += Constants.UNDERLINE + taskDetailArrs[4]; } taskTreeSet.add(formatTask); - } - } List taskslist = getTasksListFromTreeSet(tasksNum, taskTreeSet); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueImplTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueImplTest.java deleted file mode 100644 index 14e90ebcdc..0000000000 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueImplTest.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.common.queue; - -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.IpUtils; -import org.apache.dolphinscheduler.common.utils.OSUtils; -import org.apache.dolphinscheduler.common.zk.ZKServer; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; -import java.util.Random; - -import static org.junit.Assert.assertEquals; - -/** - * task queue test - */ -public class TaskQueueImplTest { - - private static final Logger logger = LoggerFactory.getLogger(TaskQueueImplTest.class); - - ITaskQueue tasksQueue = null; - - @Before - public void before(){ - ZKServer.start(); - - tasksQueue = TaskQueueFactory.getTaskQueueInstance(); - - //clear all data - tasksQueue.delete(); - - } - - - @After - public void after(){ - //clear all data - tasksQueue.delete(); - ZKServer.stop(); - } - - - @Test - public void testAdd(){ - - - //add - tasksQueue.add(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,"1_0_1_1_-1"); - tasksQueue.add(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,"0_1_1_1_-1"); - tasksQueue.add(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,"0_0_0_1_" + IpUtils.ipToLong(OSUtils.getHost())); - tasksQueue.add(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,"1_2_1_1_" + IpUtils.ipToLong(OSUtils.getHost()) + 10); - - List tasks = tasksQueue.poll(Constants.DOLPHINSCHEDULER_TASKS_QUEUE, 1); - - if(tasks.size() <= 0){ - return; - } - - //pop - String node1 = tasks.get(0); - - assertEquals(node1,"0_0_0_1_" + IpUtils.ipToLong(OSUtils.getHost())); - - - } - - - - /** - * test one million data from zookeeper queue - */ - @Ignore - @Test - public void extremeTest(){ - int total = 30 * 10000; - - for(int i = 0; i < total; i++) { - for(int j = 0; j < total; j++) { - //${processInstancePriority}_${processInstanceId}_${taskInstancePriority}_${taskId} - //format ${processInstancePriority}_${processInstanceId}_${taskInstancePriority}_${taskId} - String formatTask = String.format("%s_%d_%s_%d", i, i + 1, j, j == 0 ? 0 : j + new Random().nextInt(100)); - tasksQueue.add(Constants.DOLPHINSCHEDULER_TASKS_QUEUE, formatTask); - } - } - - String node1 = tasksQueue.poll(Constants.DOLPHINSCHEDULER_TASKS_QUEUE, 1).get(0); - assertEquals(node1,"0"); - - } - -} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java new file mode 100644 index 0000000000..3db253daf0 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.queue; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.IpUtils; +import org.apache.dolphinscheduler.common.utils.OSUtils; + +import org.apache.dolphinscheduler.common.zk.ZKServer; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import java.util.List; +import java.util.Random; + +import static org.junit.Assert.*; + +/** + * task queue test + */ +public class TaskQueueZKImplTest { + + + private ITaskQueue tasksQueue = null; + + @Before + public void before(){ + + ZKServer.start(); + tasksQueue = TaskQueueFactory.getTaskQueueInstance(); + //clear all data + tasksQueue.delete(); + } + + @After + public void after(){ + //clear all data + tasksQueue.delete(); + ZKServer.stop(); + } + + + /** + * test take out all the elements + */ + @Test + public void getAllTasks(){ + + //add + init(); + // get all + List allTasks = tasksQueue.getAllTasks(Constants.DOLPHINSCHEDULER_TASKS_QUEUE); + assertEquals(allTasks.size(),2); + //delete all + tasksQueue.delete(); + allTasks = tasksQueue.getAllTasks(Constants.DOLPHINSCHEDULER_TASKS_QUEUE); + assertEquals(allTasks.size(),0); + } + + /** + * test check task exists in the task queue or not + */ + @Test + public void checkTaskExists(){ + + String task= "1_0_1_1_-1"; + //add + init(); + // check Exist true + boolean taskExists = tasksQueue.checkTaskExists(Constants.DOLPHINSCHEDULER_TASKS_QUEUE, task); + assertTrue(taskExists); + + //remove task + tasksQueue.removeNode(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,task); + // check Exist false + taskExists = tasksQueue.checkTaskExists(Constants.DOLPHINSCHEDULER_TASKS_QUEUE, task); + assertFalse(taskExists); + } + + /** + * test add element to the queue + */ + @Test + public void add(){ + + //add + tasksQueue.add(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,"1_0_1_1_-1"); + tasksQueue.add(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,"0_1_1_1_-1"); + tasksQueue.add(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,"0_0_0_1_" + IpUtils.ipToLong(OSUtils.getHost())); + tasksQueue.add(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,"1_2_1_1_" + IpUtils.ipToLong(OSUtils.getHost()) + 10); + + List tasks = tasksQueue.poll(Constants.DOLPHINSCHEDULER_TASKS_QUEUE, 1); + + if(tasks.size() <= 0){ + return; + } + + //pop + String node1 = tasks.get(0); + assertEquals(node1,"0_0_0_1_" + IpUtils.ipToLong(OSUtils.getHost())); + } + + /** + * test element pops out of the queue + */ + @Test + public void poll(){ + + //add + init(); + List taskList = tasksQueue.poll(Constants.DOLPHINSCHEDULER_TASKS_QUEUE, 2); + assertEquals(taskList.size(),2); + + assertEquals(taskList.get(0),"0_1_1_1_-1"); + assertEquals(taskList.get(1),"1_0_1_1_-1"); + } + + /** + * test remove element from queue + */ + @Test + public void removeNode(){ + String task = "1_0_1_1_-1"; + //add + init(); + tasksQueue.removeNode(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,task); + assertFalse(tasksQueue.checkTaskExists(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,task)); + } + + /** + * test add an element to the set + */ + @Test + public void sadd(){ + + String task = "1_0_1_1_-1"; + tasksQueue.sadd(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,task); + //check size + assertEquals(tasksQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_QUEUE).size(),1); + } + + + /** + * test delete the value corresponding to the key in the set + */ + @Test + public void srem(){ + + String task = "1_0_1_1_-1"; + tasksQueue.sadd(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,task); + //check size + assertEquals(tasksQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_QUEUE).size(),1); + //remove and get size + tasksQueue.srem(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,task); + assertEquals(tasksQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_QUEUE).size(),0); + } + + /** + * test gets all the elements of the set based on the key + */ + @Test + public void smembers(){ + + //first init + assertEquals(tasksQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_QUEUE).size(),0); + //add + String task = "1_0_1_1_-1"; + tasksQueue.sadd(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,task); + //check size + assertEquals(tasksQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_QUEUE).size(),1); + //add + task = "0_1_1_1_"; + tasksQueue.sadd(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,task); + //check size + assertEquals(tasksQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_QUEUE).size(),2); + } + + + /** + * init data + */ + private void init(){ + //add + tasksQueue.add(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,"1_0_1_1_-1"); + tasksQueue.add(Constants.DOLPHINSCHEDULER_TASKS_QUEUE,"0_1_1_1_-1"); + } + + + + /** + * test one million data from zookeeper queue + */ + @Ignore + @Test + public void extremeTest(){ + int total = 30 * 10000; + + for(int i = 0; i < total; i++) { + for(int j = 0; j < total; j++) { + //${processInstancePriority}_${processInstanceId}_${taskInstancePriority}_${taskId} + //format ${processInstancePriority}_${processInstanceId}_${taskInstancePriority}_${taskId} + String formatTask = String.format("%s_%d_%s_%d", i, i + 1, j, j == 0 ? 0 : j + new Random().nextInt(100)); + tasksQueue.add(Constants.DOLPHINSCHEDULER_TASKS_QUEUE, formatTask); + } + } + + String node1 = tasksQueue.poll(Constants.DOLPHINSCHEDULER_TASKS_QUEUE, 1).get(0); + assertEquals(node1,"0"); + + } + +} diff --git a/pom.xml b/pom.xml index 2524461575..2413c56f21 100644 --- a/pom.xml +++ b/pom.xml @@ -614,6 +614,7 @@ **/common/utils/*.java **/common/graph/*.java + **/common/queue/*.java **/api/utils/CheckUtilsTest.java **/api/utils/FileUtilsTest.java From f98a5ddbd0cb2ab579665a53a4667e4d98f1bc6e Mon Sep 17 00:00:00 2001 From: lilin Date: Wed, 18 Dec 2019 11:10:19 +0800 Subject: [PATCH 03/53] extends BaseTaskQueueTest get zkServer --- .../common/queue/TaskQueueZKImplTest.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java index 3db253daf0..b13f4f63c5 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java @@ -19,8 +19,6 @@ package org.apache.dolphinscheduler.common.queue; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.IpUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; - -import org.apache.dolphinscheduler.common.zk.ZKServer; import org.junit.After; import org.junit.Before; import org.junit.Ignore; @@ -34,16 +32,14 @@ import static org.junit.Assert.*; /** * task queue test */ -public class TaskQueueZKImplTest { +public class TaskQueueZKImplTest extends BaseTaskQueueTest { + - private ITaskQueue tasksQueue = null; @Before public void before(){ - ZKServer.start(); - tasksQueue = TaskQueueFactory.getTaskQueueInstance(); //clear all data tasksQueue.delete(); } @@ -52,10 +48,8 @@ public class TaskQueueZKImplTest { public void after(){ //clear all data tasksQueue.delete(); - ZKServer.stop(); } - /** * test take out all the elements */ From deb215eceff5fb2896519c0131e3f96ad3e63698 Mon Sep 17 00:00:00 2001 From: lilin Date: Wed, 18 Dec 2019 13:19:51 +0800 Subject: [PATCH 04/53] modify zk config --- dolphinscheduler-common/src/main/resources/zookeeper.properties | 2 +- .../java/org/apache/dolphinscheduler/common/zk/ZKServer.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-common/src/main/resources/zookeeper.properties b/dolphinscheduler-common/src/main/resources/zookeeper.properties index 529e8c2803..66b533c0c1 100644 --- a/dolphinscheduler-common/src/main/resources/zookeeper.properties +++ b/dolphinscheduler-common/src/main/resources/zookeeper.properties @@ -16,7 +16,7 @@ # #zookeeper cluster. multiple are separated by commas. eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181 -zookeeper.quorum=192.168.220.188:2181 +zookeeper.quorum=localhost:2181 #dolphinscheduler root directory zookeeper.dolphinscheduler.root=/dolphinscheduler diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/zk/ZKServer.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/zk/ZKServer.java index 9d33fe1a1d..fc39e62ed8 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/zk/ZKServer.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/zk/ZKServer.java @@ -39,7 +39,7 @@ public class ZKServer { private static volatile PublicZooKeeperServerMain zkServer = null; - public static final int DEFAULT_ZK_TEST_PORT = 22181; + public static final int DEFAULT_ZK_TEST_PORT = 2181; public static final String DEFAULT_ZK_STR = "localhost:" + DEFAULT_ZK_TEST_PORT; From 947d8044d7101ace1f95f08b511088737eb4ba1e Mon Sep 17 00:00:00 2001 From: "dk.technoboy" Date: Wed, 18 Dec 2019 21:00:54 +0800 Subject: [PATCH 05/53] fix #1515 --- .../dolphinscheduler/server/worker/runner/FetchTaskThread.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java index ae4ee792c5..68f27d76d8 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java @@ -147,9 +147,8 @@ public class FetchTaskThread implements Runnable{ //check memory and cpu usage and threads boolean runCheckFlag = OSUtils.checkResource(workerConfig.getWorkerMaxCpuloadAvg(), workerConfig.getWorkerReservedMemory()) && checkThreadCount(poolExecutor); - Thread.sleep(Constants.SLEEP_TIME_MILLIS); - if(!runCheckFlag) { + Thread.sleep(Constants.SLEEP_TIME_MILLIS); continue; } From 836bf483f8b59bb5b429d86812636e687695a4ec Mon Sep 17 00:00:00 2001 From: "dk.technoboy" Date: Thu, 19 Dec 2019 15:01:41 +0800 Subject: [PATCH 06/53] sleep when resource in not satisfy. fix #1522 --- .../master/runner/MasterSchedulerThread.java | 57 +++++++++---------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java index 69c2304cd7..6e95285ce3 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java @@ -98,39 +98,38 @@ public class MasterSchedulerThread implements Runnable { InterProcessMutex mutex = null; try { - if(OSUtils.checkResource(masterConfig.getMasterMaxCpuloadAvg(), masterConfig.getMasterReservedMemory())){ - if (zkMasterClient.getZkClient().getState() == CuratorFrameworkState.STARTED) { - - // create distributed lock with the root node path of the lock space as /dolphinscheduler/lock/masters - String znodeLock = zkMasterClient.getMasterLockPath(); - - mutex = new InterProcessMutex(zkMasterClient.getZkClient(), znodeLock); - mutex.acquire(); - - ThreadPoolExecutor poolExecutor = (ThreadPoolExecutor) masterExecService; - int activeCount = poolExecutor.getActiveCount(); - // make sure to scan and delete command table in one transaction - Command command = processDao.findOneCommand(); - if (command != null) { - logger.info(String.format("find one command: id: %d, type: %s", command.getId(),command.getCommandType().toString())); - - try{ - processInstance = processDao.handleCommand(logger, OSUtils.getHost(), this.masterExecThreadNum - activeCount, command); - if (processInstance != null) { - logger.info("start master exec thread , split DAG ..."); - masterExecService.execute(new MasterExecThread(processInstance,processDao)); - } - }catch (Exception e){ - logger.error("scan command error ", e); - processDao.moveToErrorCommand(command, e.toString()); + boolean runCheckFlag = OSUtils.checkResource(masterConfig.getMasterMaxCpuloadAvg(), masterConfig.getMasterReservedMemory()); + if(!runCheckFlag) { + Thread.sleep(Constants.SLEEP_TIME_MILLIS); + continue; + } + if (zkMasterClient.getZkClient().getState() == CuratorFrameworkState.STARTED) { + + // create distributed lock with the root node path of the lock space as /dolphinscheduler/lock/masters + String znodeLock = zkMasterClient.getMasterLockPath(); + + mutex = new InterProcessMutex(zkMasterClient.getZkClient(), znodeLock); + mutex.acquire(); + + ThreadPoolExecutor poolExecutor = (ThreadPoolExecutor) masterExecService; + int activeCount = poolExecutor.getActiveCount(); + // make sure to scan and delete command table in one transaction + Command command = processDao.findOneCommand(); + if (command != null) { + logger.info(String.format("find one command: id: %d, type: %s", command.getId(),command.getCommandType().toString())); + + try{ + processInstance = processDao.handleCommand(logger, OSUtils.getHost(), this.masterExecThreadNum - activeCount, command); + if (processInstance != null) { + logger.info("start master exec thread , split DAG ..."); + masterExecService.execute(new MasterExecThread(processInstance,processDao)); } + }catch (Exception e){ + logger.error("scan command error ", e); + processDao.moveToErrorCommand(command, e.toString()); } } } - - // accessing the command table every SLEEP_TIME_MILLIS milliseconds - Thread.sleep(Constants.SLEEP_TIME_MILLIS); - }catch (Exception e){ logger.error("master scheduler thread exception : " + e.getMessage(),e); }finally{ From 1868c56764a380a7dca80429c90b8faf58e2b51f Mon Sep 17 00:00:00 2001 From: "dk.technoboy" Date: Fri, 20 Dec 2019 15:30:45 +0800 Subject: [PATCH 07/53] add sleep 1s for no command --- .../server/master/runner/MasterSchedulerThread.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java index 6e95285ce3..8d7d5a0add 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java @@ -128,6 +128,9 @@ public class MasterSchedulerThread implements Runnable { logger.error("scan command error ", e); processDao.moveToErrorCommand(command, e.toString()); } + } else{ + //indicate that no command ,sleep for 1s + Thread.sleep(Constants.SLEEP_TIME_MILLIS); } } }catch (Exception e){ From 7bb0bebffbe8ae1bfb5d067048494f37cf42b243 Mon Sep 17 00:00:00 2001 From: "dk.technoboy" Date: Fri, 20 Dec 2019 19:11:23 +0800 Subject: [PATCH 08/53] fix MasterBaseTaskExecThread submit method bug --- .../runner/MasterBaseTaskExecThread.java | 22 +++++++++---------- .../master/runner/MasterTaskExecThread.java | 3 +++ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java index 9bb5c555fd..62789a9c6d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java @@ -114,30 +114,29 @@ public class MasterBaseTaskExecThread implements Callable { Integer commitRetryInterval = masterConfig.getMasterTaskCommitInterval(); int retryTimes = 1; - boolean taskDBFlag = false; - boolean taskQueueFlag = false; + boolean submitDB = false; + boolean submitQueue = false; TaskInstance task = null; - while (true){ + while (retryTimes <= commitRetryTimes){ try { - if(!taskDBFlag){ + if(!submitDB){ // submit task to db task = processDao.submitTask(taskInstance, processInstance); if(task != null && task.getId() != 0){ - taskDBFlag = true; + submitDB = true; } } - if(taskDBFlag && !taskQueueFlag){ + if(submitDB && !submitQueue){ // submit task to queue - taskQueueFlag = processDao.submitTaskToQueue(task); + submitQueue = processDao.submitTaskToQueue(task); } - if(taskDBFlag && taskQueueFlag){ + if(submitDB && submitQueue){ return task; } - if(!taskDBFlag){ + if(!submitDB){ logger.error("task commit to db failed , task has already retry {} times, please check the database", retryTimes); - }else if(!taskQueueFlag){ + }else if(!submitDB){ logger.error("task commit to queue failed , task has already retry {} times, please check the database", retryTimes); - } Thread.sleep(commitRetryInterval); } catch (Exception e) { @@ -145,6 +144,7 @@ public class MasterBaseTaskExecThread implements Callable { } retryTimes += 1; } + return task; } /** diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java index e91deca511..16bb16de38 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java @@ -74,6 +74,9 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { public Boolean submitWaitComplete() { Boolean result = false; this.taskInstance = submit(); + if(this.taskInstance == null){ + return result; + } if(!this.taskInstance.getState().typeIsFinished()) { result = waitTaskQuit(); } From d14cefdfa3f0e9d3e1fb9758bd8fa57272d7ac4a Mon Sep 17 00:00:00 2001 From: Technoboy- Date: Fri, 20 Dec 2019 22:50:53 +0800 Subject: [PATCH 09/53] updates --- .../server/master/runner/MasterBaseTaskExecThread.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java index 62789a9c6d..bc524e28f9 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java @@ -135,7 +135,7 @@ public class MasterBaseTaskExecThread implements Callable { } if(!submitDB){ logger.error("task commit to db failed , task has already retry {} times, please check the database", retryTimes); - }else if(!submitDB){ + }else if(!submitQueue){ logger.error("task commit to queue failed , task has already retry {} times, please check the database", retryTimes); } Thread.sleep(commitRetryInterval); From 7eeeb9f5206b0b03abfafa67ecd3f3663a658070 Mon Sep 17 00:00:00 2001 From: lilin Date: Sun, 22 Dec 2019 14:54:32 +0800 Subject: [PATCH 10/53] add MonitorServiceTest --- .../api/service/MonitorServiceTest.java | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java new file mode 100644 index 0000000000..ef4b684f57 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import org.apache.dolphinscheduler.api.ApiApplicationServer; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.model.Server; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +import java.util.List; +import java.util.Map; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = ApiApplicationServer.class) +public class MonitorServiceTest { + + private static final Logger logger = LoggerFactory.getLogger(MonitorServiceTest.class); + + @Autowired + private MonitorService monitorService; + + @Test + public void testQueryDatabaseState(){ + + Map result = monitorService.queryDatabaseState(null); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + @Test + public void testQueryMaster(){ + + Map result = monitorService.queryMaster(null); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + @Test + public void testQueryZookeeperState(){ + Map result = monitorService.queryZookeeperState(null); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + + @Test + public void testGetServerListFromZK(){ + List serverList = monitorService.getServerListFromZK(true); + logger.info(serverList.toString()); + } + +} From 16c29a008e8671af58e4356cef54374503a50d7b Mon Sep 17 00:00:00 2001 From: Technoboy- Date: Tue, 24 Dec 2019 22:00:19 +0800 Subject: [PATCH 11/53] add log --- .../server/master/runner/MasterBaseTaskExecThread.java | 4 ++-- .../server/master/runner/MasterTaskExecThread.java | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java index bc524e28f9..6b4b799ef9 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java @@ -134,9 +134,9 @@ public class MasterBaseTaskExecThread implements Callable { return task; } if(!submitDB){ - logger.error("task commit to db failed , task has already retry {} times, please check the database", retryTimes); + logger.error("task commit to db failed , taskId {} has already retry {} times, please check the database", taskInstance.getId(), retryTimes); }else if(!submitQueue){ - logger.error("task commit to queue failed , task has already retry {} times, please check the database", retryTimes); + logger.error("task commit to queue failed , taskId {} has already retry {} times, please check the queue", taskInstance.getId(), retryTimes); } Thread.sleep(commitRetryInterval); } catch (Exception e) { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java index 16bb16de38..7d10591e0d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java @@ -75,6 +75,7 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { Boolean result = false; this.taskInstance = submit(); if(this.taskInstance == null){ + logger.error("submit task instance to mysql and queue failed , please check and fix it"); return result; } if(!this.taskInstance.getState().typeIsFinished()) { From f5de6172729bd67f3f726cf3ed3e121d97724ed2 Mon Sep 17 00:00:00 2001 From: "dk.technoboy" Date: Wed, 25 Dec 2019 11:47:18 +0800 Subject: [PATCH 12/53] delete lombok --- .../common/queue/TaskQueueZkImpl.java | 26 ++----- .../common/zk/ZookeeperConfig.java | 77 +++++++++++++++---- 2 files changed, 69 insertions(+), 34 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java index 9bc7359174..5eca993120 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java @@ -17,26 +17,16 @@ package org.apache.dolphinscheduler.common.queue; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Set; -import java.util.TreeSet; - import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.Bytes; import org.apache.dolphinscheduler.common.utils.IpUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; -import org.apache.dolphinscheduler.common.zk.AbstractZKClient; -import org.apache.curator.framework.CuratorFramework; import org.apache.dolphinscheduler.common.zk.DefaultEnsembleProvider; import org.apache.dolphinscheduler.common.zk.ZookeeperConfig; import org.apache.zookeeper.CreateMode; @@ -44,7 +34,7 @@ import org.apache.zookeeper.data.Stat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.dolphinscheduler.common.utils.Preconditions.checkNotNull; +import java.util.*; /** * A singleton of a task queue implemented with zookeeper @@ -421,12 +411,11 @@ public class TaskQueueZkImpl implements ITaskQueue { logger.error("load zookeeper properties file failed, system exit"); System.exit(-1); } - zookeeperConfig = ZookeeperConfig.getFromConf(conf); - zkClient = CuratorFrameworkFactory.builder().ensembleProvider(new DefaultEnsembleProvider(checkNotNull(zookeeperConfig.getServerList(), "zookeeper quorum can't be null"))) - .retryPolicy(new ExponentialBackoffRetry(zookeeperConfig.getBaseSleepTimeMs(), zookeeperConfig.getMaxRetries(), zookeeperConfig.getMaxSleepMs())) - .sessionTimeoutMs(zookeeperConfig.getSessionTimeoutMs()) - .connectionTimeoutMs(zookeeperConfig.getConnectionTimeoutMs()) + zkClient = CuratorFrameworkFactory.builder().ensembleProvider(new DefaultEnsembleProvider(conf.getString("zookeeper.quorum"))) + .retryPolicy(new ExponentialBackoffRetry(conf.getInt("zookeeper.retry.base.sleep"), conf.getInt("zookeeper.retry.maxtime"), conf.getInt("zookeeper.retry.max.sleep"))) + .sessionTimeoutMs(conf.getInt("zookeeper.session.timeout")) + .connectionTimeoutMs(conf.getInt("zookeeper.connection.timeout")) .build(); zkClient.start(); @@ -472,8 +461,7 @@ public class TaskQueueZkImpl implements ITaskQueue { * @return */ public String getTasksPath(String key){ - return zookeeperConfig.getDsRoot() + Constants.SINGLE_SLASH + key; + return "/dolphinscheduler" + Constants.SINGLE_SLASH + key; } - } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java index 6e471bbbfb..a90a147425 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java @@ -16,11 +16,6 @@ */ package org.apache.dolphinscheduler.common.zk; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.apache.commons.configuration.Configuration; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.PropertySource; import org.springframework.stereotype.Component; @@ -29,10 +24,6 @@ import org.springframework.stereotype.Component; * zookeeper conf */ @Component -@Data -@AllArgsConstructor -@NoArgsConstructor -@Builder @PropertySource("classpath:zookeeper.properties") public class ZookeeperConfig { @@ -58,14 +49,70 @@ public class ZookeeperConfig { @Value("${zookeeper.connection.digest: }") private String digest; - //ds scheduler install config @Value("${zookeeper.dolphinscheduler.root:/dolphinscheduler}") private String dsRoot; - public static ZookeeperConfig getFromConf(Configuration conf){ - return ZookeeperConfig.builder().serverList(conf.getString("zookeeper.quorum")).baseSleepTimeMs(conf.getInt("zookeeper.retry.base.sleep")) - .maxSleepMs(conf.getInt("zookeeper.retry.max.sleep")).maxRetries(conf.getInt("zookeeper.retry.maxtime")) - .sessionTimeoutMs(conf.getInt("zookeeper.session.timeout")).connectionTimeoutMs(conf.getInt("zookeeper.connection.timeout")) - .dsRoot(conf.getString("zookeeper.dolphinscheduler.root")).build(); + public String getServerList() { + return serverList; + } + + public void setServerList(String serverList) { + this.serverList = serverList; + } + + public int getBaseSleepTimeMs() { + return baseSleepTimeMs; + } + + public void setBaseSleepTimeMs(int baseSleepTimeMs) { + this.baseSleepTimeMs = baseSleepTimeMs; + } + + public int getMaxSleepMs() { + return maxSleepMs; + } + + public void setMaxSleepMs(int maxSleepMs) { + this.maxSleepMs = maxSleepMs; + } + + public int getMaxRetries() { + return maxRetries; + } + + public void setMaxRetries(int maxRetries) { + this.maxRetries = maxRetries; + } + + public int getSessionTimeoutMs() { + return sessionTimeoutMs; + } + + public void setSessionTimeoutMs(int sessionTimeoutMs) { + this.sessionTimeoutMs = sessionTimeoutMs; + } + + public int getConnectionTimeoutMs() { + return connectionTimeoutMs; + } + + public void setConnectionTimeoutMs(int connectionTimeoutMs) { + this.connectionTimeoutMs = connectionTimeoutMs; + } + + public String getDigest() { + return digest; + } + + public void setDigest(String digest) { + this.digest = digest; + } + + public String getDsRoot() { + return dsRoot; + } + + public void setDsRoot(String dsRoot) { + this.dsRoot = dsRoot; } } \ No newline at end of file From 373c2aeba4ee51f4b065bda6cbe3d63ad7282a30 Mon Sep 17 00:00:00 2001 From: "dk.technoboy" Date: Thu, 26 Dec 2019 12:28:47 +0800 Subject: [PATCH 13/53] remove duplicate code --- .../server/master/runner/SubProcessTaskExecThread.java | 1 - 1 file changed, 1 deletion(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java index 2eb48cd8d2..5ee5d01b7a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java @@ -72,7 +72,6 @@ public class SubProcessTaskExecThread extends MasterBaseTaskExecThread { this.taskInstance.setState(ExecutionStatus.KILL); }else{ this.taskInstance.setState(subProcessInstance.getState()); - result = true; } } taskInstance.setEndTime(new Date()); From f59bb10c2c770164d4ea4fc34de2a0e43723abbc Mon Sep 17 00:00:00 2001 From: "dk.technoboy" Date: Thu, 26 Dec 2019 19:16:33 +0800 Subject: [PATCH 14/53] refactor TaskQueueZkImpl --- .../common/queue/TaskQueueFactory.java | 3 +- .../common/queue/TaskQueueZkImpl.java | 163 +++--------------- .../utils/SpringApplicationContext.java | 2 +- .../common/zk/ZookeeperOperator.java | 3 - .../dolphinscheduler/dao/ProcessDao.java | 3 +- .../server/master/MasterServer.java | 3 +- .../runner/MasterBaseTaskExecThread.java | 2 +- .../master/runner/MasterExecThread.java | 1 - .../master/runner/MasterSchedulerThread.java | 2 +- .../server/worker/WorkerServer.java | 2 +- .../server/worker/runner/FetchTaskThread.java | 3 +- .../server/worker/task/AbstractYarnTask.java | 2 +- .../task/dependent/DependentExecute.java | 2 +- .../worker/task/dependent/DependentTask.java | 2 +- .../server/worker/task/http/HttpTask.java | 2 +- .../task/processdure/ProcedureTask.java | 2 +- .../server/worker/task/python/PythonTask.java | 2 +- .../server/worker/task/shell/ShellTask.java | 2 +- .../server/worker/task/sql/SqlTask.java | 2 +- .../shell/ShellCommandExecutorTest.java | 2 +- .../server/worker/sql/SqlExecutorTest.java | 2 +- 21 files changed, 44 insertions(+), 163 deletions(-) rename {dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server => dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common}/utils/SpringApplicationContext.java (96%) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueFactory.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueFactory.java index a82a3098f2..0a2d943118 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueFactory.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueFactory.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.common.queue; import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.commons.lang.StringUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,7 +44,7 @@ public class TaskQueueFactory { String queueImplValue = CommonUtils.getQueueImplValue(); if (StringUtils.isNotBlank(queueImplValue)) { logger.info("task queue impl use zookeeper "); - return TaskQueueZkImpl.getInstance(); + return SpringApplicationContext.getBean(TaskQueueZkImpl.class); }else{ logger.error("property dolphinscheduler.queue.impl can't be blank, system will exit "); System.exit(-1); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java index 5eca993120..45c6122341 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java @@ -17,22 +17,14 @@ package org.apache.dolphinscheduler.common.queue; -import org.apache.commons.configuration.Configuration; -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.Bytes; import org.apache.dolphinscheduler.common.utils.IpUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; -import org.apache.dolphinscheduler.common.zk.DefaultEnsembleProvider; -import org.apache.dolphinscheduler.common.zk.ZookeeperConfig; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.data.Stat; +import org.apache.dolphinscheduler.common.zk.ZookeeperOperator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; import java.util.*; @@ -40,35 +32,13 @@ import java.util.*; * A singleton of a task queue implemented with zookeeper * tasks queue implemention */ +@Service public class TaskQueueZkImpl implements ITaskQueue { private static final Logger logger = LoggerFactory.getLogger(TaskQueueZkImpl.class); - private static volatile TaskQueueZkImpl instance; - - private CuratorFramework zkClient; - - private ZookeeperConfig zookeeperConfig; - - private CuratorFramework getZkClient() { - return zkClient; - } - - private TaskQueueZkImpl(){ - init(); - } - - public static TaskQueueZkImpl getInstance(){ - if (null == instance) { - synchronized (TaskQueueZkImpl.class) { - if(null == instance) { - instance = new TaskQueueZkImpl(); - } - } - } - return instance; - } - + @Autowired + private ZookeeperOperator zookeeperOperator; /** * get all tasks from tasks queue @@ -78,14 +48,12 @@ public class TaskQueueZkImpl implements ITaskQueue { @Override public List getAllTasks(String key) { try { - List list = getZkClient().getChildren().forPath(getTasksPath(key)); - + List list = zookeeperOperator.getChildrenKeys(getTasksPath(key)); return list; } catch (Exception e) { logger.error("get all tasks from tasks queue exception",e); } - - return new ArrayList(); + return new ArrayList<>(); } /** @@ -99,22 +67,8 @@ public class TaskQueueZkImpl implements ITaskQueue { public boolean checkTaskExists(String key, String task) { String taskPath = getTasksPath(key) + Constants.SINGLE_SLASH + task; - try { - Stat stat = zkClient.checkExists().forPath(taskPath); - - if(null == stat){ - logger.info("check task:{} not exist in task queue",task); - return false; - }else{ - logger.info("check task {} exists in task queue ",task); - return true; - } + return zookeeperOperator.isExisted(taskPath); - } catch (Exception e) { - logger.info(String.format("task {} check exists in task queue exception ", task), e); - } - - return false; } @@ -128,9 +82,7 @@ public class TaskQueueZkImpl implements ITaskQueue { public boolean add(String key, String value){ try { String taskIdPath = getTasksPath(key) + Constants.SINGLE_SLASH + value; - String result = getZkClient().create().withMode(CreateMode.PERSISTENT).forPath(taskIdPath, Bytes.toBytes(value)); - - logger.info("add task : {} to tasks queue , result success",result); + zookeeperOperator.persist(taskIdPath, value); return true; } catch (Exception e) { logger.error("add task to tasks queue exception",e); @@ -153,8 +105,7 @@ public class TaskQueueZkImpl implements ITaskQueue { @Override public List poll(String key, int tasksNum) { try{ - CuratorFramework zk = getZkClient(); - List list = zk.getChildren().forPath(getTasksPath(key)); + List list = zookeeperOperator.getChildrenKeys(getTasksPath(key)); if(list != null && list.size() > 0){ @@ -277,15 +228,12 @@ public class TaskQueueZkImpl implements ITaskQueue { @Override public void removeNode(String key, String nodeValue){ - CuratorFramework zk = getZkClient(); String tasksQueuePath = getTasksPath(key) + Constants.SINGLE_SLASH; String taskIdPath = tasksQueuePath + nodeValue; - logger.info("consume task {}", taskIdPath); + logger.info("removeNode task {}", taskIdPath); try{ - Stat stat = zk.checkExists().forPath(taskIdPath); - if(stat != null){ - zk.delete().forPath(taskIdPath); - } + zookeeperOperator.remove(taskIdPath); + }catch(Exception e){ logger.error(String.format("delete task:%s from zookeeper fail, exception:" ,nodeValue) ,e); } @@ -307,13 +255,10 @@ public class TaskQueueZkImpl implements ITaskQueue { if(value != null && value.trim().length() > 0){ String path = getTasksPath(key) + Constants.SINGLE_SLASH; - CuratorFramework zk = getZkClient(); - Stat stat = zk.checkExists().forPath(path + value); - - if(null == stat){ - String result = zk.create().withMode(CreateMode.PERSISTENT).forPath(path + value,Bytes.toBytes(value)); - logger.info("add task:{} to tasks set result:{} ",value,result); - }else{ + if(!zookeeperOperator.isExisted(path + value)){ + zookeeperOperator.persist(path + value,value); + logger.info("add task:{} to tasks set ",value); + } else{ logger.info("task {} exists in tasks set ",value); } @@ -336,15 +281,7 @@ public class TaskQueueZkImpl implements ITaskQueue { public void srem(String key, String value) { try{ String path = getTasksPath(key) + Constants.SINGLE_SLASH; - CuratorFramework zk = getZkClient(); - Stat stat = zk.checkExists().forPath(path + value); - - if(null != stat){ - zk.delete().forPath(path + value); - logger.info("delete task:{} from tasks set ",value); - }else{ - logger.info("delete task:{} from tasks set fail, there is no this task",value); - } + zookeeperOperator.remove(path + value); }catch(Exception e){ logger.error(String.format("delete task:" + value + " exception"),e); @@ -363,7 +300,7 @@ public class TaskQueueZkImpl implements ITaskQueue { Set tasksSet = new HashSet<>(); try { - List list = getZkClient().getChildren().forPath(getTasksPath(key)); + List list = zookeeperOperator.getChildrenKeys(getTasksPath(key)); for (String task : list) { tasksSet.add(task); @@ -377,56 +314,6 @@ public class TaskQueueZkImpl implements ITaskQueue { return tasksSet; } - - - /** - * Init the task queue of zookeeper node - */ - private void init(){ - initZkClient(); - try { - String tasksQueuePath = getTasksPath(Constants.DOLPHINSCHEDULER_TASKS_QUEUE); - String tasksCancelPath = getTasksPath(Constants.DOLPHINSCHEDULER_TASKS_KILL); - - for(String taskQueuePath : new String[]{tasksQueuePath,tasksCancelPath}){ - if(zkClient.checkExists().forPath(taskQueuePath) == null){ - // create a persistent parent node - zkClient.create().creatingParentContainersIfNeeded() - .withMode(CreateMode.PERSISTENT).forPath(taskQueuePath); - logger.info("create tasks queue parent node success : {} ",taskQueuePath); - } - } - - } catch (Exception e) { - logger.error("create zk node failure",e); - } - } - - private void initZkClient() { - - Configuration conf = null; - try { - conf = new PropertiesConfiguration(Constants.ZOOKEEPER_PROPERTIES_PATH); - } catch (ConfigurationException ex) { - logger.error("load zookeeper properties file failed, system exit"); - System.exit(-1); - } - - zkClient = CuratorFrameworkFactory.builder().ensembleProvider(new DefaultEnsembleProvider(conf.getString("zookeeper.quorum"))) - .retryPolicy(new ExponentialBackoffRetry(conf.getInt("zookeeper.retry.base.sleep"), conf.getInt("zookeeper.retry.maxtime"), conf.getInt("zookeeper.retry.max.sleep"))) - .sessionTimeoutMs(conf.getInt("zookeeper.session.timeout")) - .connectionTimeoutMs(conf.getInt("zookeeper.connection.timeout")) - .build(); - - zkClient.start(); - try { - zkClient.blockUntilConnected(); - } catch (final Exception ex) { - throw new RuntimeException(ex); - } - } - - /** * Clear the task queue of zookeeper node */ @@ -437,16 +324,12 @@ public class TaskQueueZkImpl implements ITaskQueue { String tasksCancelPath = getTasksPath(Constants.DOLPHINSCHEDULER_TASKS_KILL); for(String taskQueuePath : new String[]{tasksQueuePath,tasksCancelPath}){ - if(zkClient.checkExists().forPath(taskQueuePath) != null){ - - List list = zkClient.getChildren().forPath(taskQueuePath); - + if(zookeeperOperator.isExisted(taskQueuePath)){ + List list = zookeeperOperator.getChildrenKeys(taskQueuePath); for (String task : list) { - zkClient.delete().forPath(taskQueuePath + Constants.SINGLE_SLASH + task); + zookeeperOperator.remove(taskQueuePath + Constants.SINGLE_SLASH + task); logger.info("delete task from tasks queue : {}/{} ",taskQueuePath,task); - } - } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/SpringApplicationContext.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SpringApplicationContext.java similarity index 96% rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/SpringApplicationContext.java rename to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SpringApplicationContext.java index 96087e5a52..97618e1b39 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/SpringApplicationContext.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SpringApplicationContext.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.server.utils; +package org.apache.dolphinscheduler.common.utils; import org.springframework.beans.BeansException; import org.springframework.context.ApplicationContext; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java index f4d72f436e..5e3751b25d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java @@ -127,9 +127,6 @@ public class ZookeeperOperator implements InitializingBean { List values; try { values = zkClient.getChildren().forPath(key); - if (CollectionUtils.isEmpty(values)) { - logger.warn("getChildrenKeys key : {} is empty", key); - } return values; } catch (InterruptedException ex) { logger.error("getChildrenKeys key : {} InterruptedException", key); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java index eb97ad75b2..66948951f6 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java @@ -105,7 +105,8 @@ public class ProcessDao { /** * task queue impl */ - protected ITaskQueue taskQueue = TaskQueueFactory.getTaskQueueInstance(); + @Autowired + private ITaskQueue taskQueue; /** * handle Command (construct ProcessInstance from Command) , wrapped in transaction * @param logger logger diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index 8297cd0403..e8c8b6779e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -23,18 +23,17 @@ import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.runner.MasterSchedulerThread; import org.apache.dolphinscheduler.server.quartz.ProcessScheduleJob; import org.apache.dolphinscheduler.server.quartz.QuartzExecutors; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.zk.ZKMasterClient; import org.quartz.SchedulerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.SpringApplication; import org.springframework.boot.WebApplicationType; import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.context.annotation.ComponentScan; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java index 6b4b799ef9..5c96757072 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java @@ -18,13 +18,13 @@ package org.apache.dolphinscheduler.server.master.runner; import org.apache.dolphinscheduler.common.queue.ITaskQueue; import org.apache.dolphinscheduler.common.queue.TaskQueueFactory; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.utils.BeanContext; import org.apache.dolphinscheduler.server.master.config.MasterConfig; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 5446830780..3481b79caa 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -34,7 +34,6 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.utils.DagHelper; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.utils.AlertManager; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java index 8d7d5a0add..5f594b3fa0 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java @@ -22,12 +22,12 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.common.zk.AbstractZKClient; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.Command; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.server.master.config.MasterConfig; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.zk.ZKMasterClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index 96f5ba0b5d..877d60a33b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -29,12 +29,12 @@ import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.common.zk.AbstractZKClient; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.utils.ProcessUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.runner.FetchTaskThread; import org.apache.dolphinscheduler.server.zk.ZKWorkerClient; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java index 7429050605..ae67716da2 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java @@ -25,12 +25,12 @@ import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.common.zk.AbstractZKClient; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.WorkerGroup; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.zk.ZKWorkerClient; import org.slf4j.Logger; @@ -155,6 +155,7 @@ public class FetchTaskThread implements Runnable{ //whether have tasks, if no tasks , no need lock //get all tasks List tasksQueueList = taskQueue.getAllTasks(Constants.DOLPHINSCHEDULER_TASKS_QUEUE); if (CollectionUtils.isEmpty(tasksQueueList)){ + Thread.sleep(Constants.SLEEP_TIME_MILLIS); continue; } // creating distributed locks, lock path /dolphinscheduler/lock/worker diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java index b9b3ad6824..6846617408 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java @@ -16,10 +16,10 @@ */ package org.apache.dolphinscheduler.server.worker.task; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.utils.ProcessUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.slf4j.Logger; /** diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java index 5dc25b8935..4be65ed49d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java @@ -23,10 +23,10 @@ import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.DateInterval; import org.apache.dolphinscheduler.common.model.DependentItem; import org.apache.dolphinscheduler.common.utils.DependentUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java index b0bb4c6f4c..9af29e01dd 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java @@ -25,9 +25,9 @@ import org.apache.dolphinscheduler.common.task.dependent.DependentParameters; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.utils.DependentUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskProps; import org.slf4j.Logger; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java index 44eef65aba..993310f6ec 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java @@ -30,10 +30,10 @@ import org.apache.dolphinscheduler.common.task.http.HttpParameters; import org.apache.dolphinscheduler.common.utils.Bytes; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.server.utils.ParamUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskProps; import org.apache.http.HttpEntity; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java index 59cf8a6e24..9b4952bbd2 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java @@ -29,10 +29,10 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.procedure.ProcedureParameters; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.server.utils.ParamUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskProps; import org.slf4j.Logger; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java index f6227b15a4..585d62f154 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java @@ -22,9 +22,9 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.python.PythonParameters; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.server.utils.ParamUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.PythonCommandExecutor; import org.apache.dolphinscheduler.server.worker.task.TaskProps; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java index 438d373775..789a0c5302 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java @@ -23,9 +23,9 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.shell.ShellParameters; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.server.utils.ParamUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor; import org.apache.dolphinscheduler.server.worker.task.TaskProps; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index ccfee2efec..08a90c62ce 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -36,6 +36,7 @@ import org.apache.dolphinscheduler.common.task.sql.SqlType; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.DataSource; @@ -43,7 +44,6 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.server.utils.ParamUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.utils.UDFUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskProps; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index 71bebe2990..1117fe0015 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -20,10 +20,10 @@ import com.alibaba.fastjson.JSONObject; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.utils.LoggerUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskManager; import org.apache.dolphinscheduler.server.worker.task.TaskProps; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java index 725f2835e9..7cf4b874d1 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java @@ -21,10 +21,10 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.utils.LoggerUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskManager; import org.apache.dolphinscheduler.server.worker.task.TaskProps; From 49402aa978eb0ee17a0f07108297a4bf0e8c60ca Mon Sep 17 00:00:00 2001 From: "dk.technoboy" Date: Thu, 26 Dec 2019 19:38:21 +0800 Subject: [PATCH 15/53] ignore First , we have to rewrite --- .../dolphinscheduler/common/queue/TaskQueueZKImplTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java index b13f4f63c5..1b44673149 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java @@ -32,11 +32,9 @@ import static org.junit.Assert.*; /** * task queue test */ +@Ignore public class TaskQueueZKImplTest extends BaseTaskQueueTest { - - - @Before public void before(){ From 1610f434afb416c487e42961be9688353fc19d7e Mon Sep 17 00:00:00 2001 From: "dk.technoboy" Date: Thu, 26 Dec 2019 20:03:40 +0800 Subject: [PATCH 16/53] updates --- .../org/apache/dolphinscheduler/dao/mapper/Application.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/Application.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/Application.java index 75b79a4266..02df228f72 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/Application.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/Application.java @@ -21,7 +21,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.ComponentScan; @SpringBootApplication -@ComponentScan("org.apache.dolphinscheduler.dao") +@ComponentScan("org.apache.dolphinscheduler") public class Application { public static void main(String[] args) { From 6bec773792e7828a2fb6416818a479d343e82bfa Mon Sep 17 00:00:00 2001 From: lilin Date: Fri, 27 Dec 2019 14:18:44 +0800 Subject: [PATCH 17/53] mobile phone need 11 number --- .../java/org/apache/dolphinscheduler/api/utils/CheckUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java index 6c9f714721..a7867f1ba2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java @@ -104,7 +104,7 @@ public class CheckUtils { * @return true if phone regex valid, otherwise return false */ public static boolean checkPhone(String phone) { - return StringUtils.isEmpty(phone) || phone.length() <= 11; + return StringUtils.isEmpty(phone) || phone.length() == 11; } From 9c6f9fce20ea2d84c0a191309625f2a61897cfec Mon Sep 17 00:00:00 2001 From: lilin Date: Fri, 27 Dec 2019 14:21:48 +0800 Subject: [PATCH 18/53] delete file --- .../api/service/MonitorServiceTest.java | 71 ------------------- 1 file changed, 71 deletions(-) delete mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java deleted file mode 100644 index ef4b684f57..0000000000 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.api.service; - -import org.apache.dolphinscheduler.api.ApiApplicationServer; -import org.apache.dolphinscheduler.api.enums.Status; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.model.Server; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; - -import java.util.List; -import java.util.Map; - -@RunWith(SpringRunner.class) -@SpringBootTest(classes = ApiApplicationServer.class) -public class MonitorServiceTest { - - private static final Logger logger = LoggerFactory.getLogger(MonitorServiceTest.class); - - @Autowired - private MonitorService monitorService; - - @Test - public void testQueryDatabaseState(){ - - Map result = monitorService.queryDatabaseState(null); - logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); - } - @Test - public void testQueryMaster(){ - - Map result = monitorService.queryMaster(null); - logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); - } - @Test - public void testQueryZookeeperState(){ - Map result = monitorService.queryZookeeperState(null); - logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); - } - - @Test - public void testGetServerListFromZK(){ - List serverList = monitorService.getServerListFromZK(true); - logger.info(serverList.toString()); - } - -} From a526106952005dfa85b7f909019bd57bc5aa7576 Mon Sep 17 00:00:00 2001 From: gaojun2048 <32193458+gaojun2048@users.noreply.github.com> Date: Fri, 27 Dec 2019 14:32:24 +0800 Subject: [PATCH 19/53] Added the maven profile to build rpm packages (#1563) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update quick-start.md * 更新demo的地址 * add rpm plugin * delete the git ignore files * 测试git ignore * 测试ignore * optimize the .gitignore file * add rpm package, delete unuse copy jar action in assembly * delete docs dir * add rpm plugin support * add rpm plugin support * add rpm plugin support * dolphinscheduler-common have not bin dir * dolphinscheduler-common have not bin dir * delete unuse config about assembly * add defineSatement * add rpm plugin support * add rpm plugin support * add rpm plugin support * add rpm support * update install dirname , make front and backend together * update rpm name * update rpm name * update rpm config * add jars excludes * add jars excludes * add rpm plugion * add rpm plugion * add rpm plugion * add auto create /opt/soft --- dolphinscheduler-dist/pom.xml | 270 ++++++++++++++++++ .../main/assembly/dolphinscheduler-binary.xml | 19 +- pom.xml | 16 ++ 3 files changed, 288 insertions(+), 17 deletions(-) diff --git a/dolphinscheduler-dist/pom.xml b/dolphinscheduler-dist/pom.xml index 3d01eff831..dbeddbb6b3 100644 --- a/dolphinscheduler-dist/pom.xml +++ b/dolphinscheduler-dist/pom.xml @@ -101,6 +101,276 @@ + + + rpmbuild + + + + org.apache.maven.plugins + maven-dependency-plugin + + ${project.build.directory}/lib + false + false + true + provided + + + + copy-dependencies + package + + copy-dependencies + + + + + + + org.codehaus.mojo + rpm-maven-plugin + true + + + package + + attached-rpm + + + + + + apache-dolphinscheduler-incubating + 1 + apache dolphinscheduler incubating rpm + apache + dolphinscheduler + + /opt/soft + + + + __os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g') + + + + /opt/soft/${project.build.finalName}/conf + 755 + root + root + + + + ${basedir}/../dolphinscheduler-alert/src/main/resources + + + **/*.properties + **/*.xml + **/*.json + **/*.ftl + + + + + + ${basedir}/../dolphinscheduler-common/src/main/resources + + + **/*.properties + **/*.xml + **/*.json + + + + + + ${basedir}/../dolphinscheduler-dao/src/main/resources + + + **/*.properties + **/*.xml + **/*.json + **/*.yml + + + + + + ${basedir}/../dolphinscheduler-api/src/main/resources + + + **/*.properties + **/*.xml + **/*.json + + + + + + ${basedir}/../dolphinscheduler-server/src/main/resources + + + **/*.properties + **/*.xml + **/*.json + + + + + + ${basedir}/../script + + + config/*.* + env/*.* + + + + + + + + /opt/soft/${project.build.finalName}/lib + 755 + root + root + + + + + ${basedir}/../dolphinscheduler-dist/target/lib + + + *.* + + + servlet-api-*.jar + slf4j-log4j12-${slf4j.log4j12.version}.jar + + + + + + /opt/soft/${project.build.finalName}/bin + 755 + root + root + + + + + ${basedir}/../script + + + start-all.sh + stop-all.sh + dolphinscheduler-daemon.sh + + + + + + /opt/soft/${project.build.finalName} + 755 + root + root + + + + ${basedir}/../ + + + *.sh + *.py + DISCLAIMER + + + + + + ${basedir}/../dolphinscheduler-ui + + + install-dolphinscheduler-ui.sh + + + + + ${basedir}/release-docs + + + **/* + + + + + + + /opt/soft/${project.build.finalName}/dist + 755 + root + root + + + + ${basedir}/../dolphinscheduler-ui/dist + + + **/*.* + + + + + + /opt/soft/${project.build.finalName}/sql + 755 + root + root + + + + ${basedir}/../sql + + + **/*.* + + + + + + + /opt/soft/${project.build.finalName}/script + 755 + root + root + + + + ${basedir}/../script + + + **/*.* + + + + + + + + + + + + + + + + + + + + diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml index 5cddadd4f5..2bf109698b 100644 --- a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml +++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml @@ -97,23 +97,8 @@ conf - - ${basedir}/../dolphinscheduler-common/src/main/resources - - **/*.properties - **/*.xml - **/*.json - - conf - - - ${basedir}/../dolphinscheduler-common/src/main/resources/bin - - *.* - - 755 - bin - + + ${basedir}/../dolphinscheduler-dao/src/main/resources diff --git a/pom.xml b/pom.xml index 83a6b45e10..3d002df1ed 100644 --- a/pom.xml +++ b/pom.xml @@ -108,6 +108,8 @@ 2.10.3 2.4 2.18.1 + 3.1.1 + 2.2.0 0.8.4 1.0 false @@ -524,6 +526,14 @@ + + + org.codehaus.mojo + rpm-maven-plugin + ${rpm-maven-plugion.version} + false + + org.apache.maven.plugins maven-compiler-plugin @@ -568,6 +578,12 @@ ${maven-source-plugin.version} + + org.apache.maven.plugins + maven-dependency-plugin + ${maven-dependency-plugin.version} + + From ef401b98e2072ce2d175f407ce16f4a9c3672075 Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Fri, 27 Dec 2019 14:32:44 +0800 Subject: [PATCH 20/53] add AlertGroupMapperTest UT in github action (#1590) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * remove LogViewServiceGrpc.java file and pom modify * remove kazoo * remove kazoo * remove kazoo * remove common monitor package * add license * remove kazoo modify * remove kazoo modify * remove kazoo modify * remove kazoo modify * remove kazoo modify * remove kazoo modify * install.sh remove python kazoo * add system param whether repeat running * remove kazoo modify * BusinessTimeUtils remove whther repeat running inner param * add AccessTokenMapperTest UT * CI UT yml modify，start postgresql and zookeeper by default * add AlertGroupMapperTest UT in github action * Conflicts reslove --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 3d002df1ed..69f1ff0665 100644 --- a/pom.xml +++ b/pom.xml @@ -673,6 +673,7 @@ **/server/utils/SparkArgsUtilsTest.java **/server/utils/FlinkArgsUtilsTest.java **/dao/mapper/AccessTokenMapperTest.java + **/dao/mapper/AlertGroupMapperTest.java From fe9344d4be3c6b4bcf5e280fca770da8a8f37cee Mon Sep 17 00:00:00 2001 From: samz406 Date: Fri, 27 Dec 2019 14:42:21 +0800 Subject: [PATCH 21/53] mobile phone need 11 number (#1596) * misspell words * modify common queue TaskQueueZKImplTest.java unit test * extends BaseTaskQueueTest get zkServer * modify zk config * add MonitorServiceTest * mobile phone need 11 number * delete file --- .../java/org/apache/dolphinscheduler/api/utils/CheckUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java index 6c9f714721..a7867f1ba2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java @@ -104,7 +104,7 @@ public class CheckUtils { * @return true if phone regex valid, otherwise return false */ public static boolean checkPhone(String phone) { - return StringUtils.isEmpty(phone) || phone.length() <= 11; + return StringUtils.isEmpty(phone) || phone.length() == 11; } From 3cccfa30374a7b3bfef1916f363ff32200d44c69 Mon Sep 17 00:00:00 2001 From: lilin Date: Fri, 27 Dec 2019 14:44:01 +0800 Subject: [PATCH 22/53] updateToken add entity exist check --- .../dolphinscheduler/api/service/AccessTokenService.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java index 4664b59763..897646ba70 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java @@ -154,8 +154,13 @@ public class AccessTokenService extends BaseService { */ public Map updateToken(int id,int userId, String expireTime, String token) { Map result = new HashMap<>(5); - AccessToken accessToken = new AccessToken(); - accessToken.setId(id); + + AccessToken accessToken = accessTokenMapper.selectById(id); + if (accessToken == null) { + logger.error("access token not exist, access token id {}", id); + putMsg(result, Status.ACCESS_TOKEN_NOT_EXIST); + return result; + } accessToken.setUserId(userId); accessToken.setExpireTime(DateUtils.stringToDate(expireTime)); accessToken.setToken(token); From 05a5a413cf9a1dc770848b9f023134665875b4f6 Mon Sep 17 00:00:00 2001 From: lilin Date: Fri, 27 Dec 2019 14:57:55 +0800 Subject: [PATCH 23/53] delAlertgroupById add entity exist check --- .../dolphinscheduler/api/service/AlertGroupService.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java index 40fc65b4dc..63f50f936f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java @@ -193,6 +193,12 @@ public class AlertGroupService extends BaseService{ if (checkAdmin(loginUser, result)){ return result; } + //check exist + AlertGroup alertGroup = alertGroupMapper.selectById(id); + if (alertGroup == null) { + putMsg(result, Status.ALERT_GROUP_NOT_EXIST); + return result; + } userAlertGroupMapper.deleteByAlertgroupId(id); alertGroupMapper.deleteById(id); From 9de928ea042d3159c5e875ae3a75e8de0ce65462 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Fri, 27 Dec 2019 15:00:55 +0800 Subject: [PATCH 24/53] add profile nginx in order to deploy frontend (#1599) * add profile nginx * add dolphinscheduler-nginx.xml --- dolphinscheduler-dist/pom.xml | 57 +++++ .../main/assembly/dolphinscheduler-nginx.xml | 236 ++++++++++++++++++ dolphinscheduler-ui/pom.xml | 165 ++++++++---- 3 files changed, 408 insertions(+), 50 deletions(-) create mode 100644 dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml diff --git a/dolphinscheduler-dist/pom.xml b/dolphinscheduler-dist/pom.xml index dbeddbb6b3..2a7553e3a3 100644 --- a/dolphinscheduler-dist/pom.xml +++ b/dolphinscheduler-dist/pom.xml @@ -102,6 +102,63 @@ + + nginx + + + + maven-assembly-plugin + + + dolphinscheduler-nginx + package + + single + + + + + src/main/assembly/dolphinscheduler-nginx.xml + + true + + + + + src + package + + single + + + + src/main/assembly/dolphinscheduler-src.xml + + true + + + + + + + + org.apache.maven.plugins + maven-source-plugin + + + attach-sources + verify + + jar-no-fork + + + + + + + + + rpmbuild diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml new file mode 100644 index 0000000000..ebe5b2cb3f --- /dev/null +++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml @@ -0,0 +1,236 @@ + + + + dolphinscheduler-nginx + + tar.gz + + true + ${project.build.finalName}-dolphinscheduler-bin + + + + + ${basedir}/../dolphinscheduler-alert/src/main/resources + + **/*.properties + **/*.xml + **/*.json + **/*.ftl + + ./conf + + + + + + src/main/resources + + **/*.properties + **/*.xml + **/*.json + + conf + + + ${basedir}/../dolphinscheduler-common/src/main/resources + + **/*.properties + **/*.xml + **/*.json + + conf + + + ${basedir}/../dolphinscheduler-common/src/main/resources/bin + + *.* + + 755 + bin + + + ${basedir}/../dolphinscheduler-dao/src/main/resources + + **/*.properties + **/*.xml + **/*.json + + conf + + + ${basedir}/../dolphinscheduler-api/src/main/resources + + **/*.properties + **/*.xml + **/*.json + + conf + + + + + + ${basedir}/../dolphinscheduler-server/src/main/resources + + **/*.properties + **/*.xml + **/*.json + + conf + + + ${basedir}/../dolphinscheduler-common/src/main/resources + + **/*.properties + **/*.xml + **/*.json + + conf + + + ${basedir}/../dolphinscheduler-common/src/main/resources/bin + + *.* + + 755 + bin + + + ${basedir}/../dolphinscheduler-dao/src/main/resources + + **/*.properties + **/*.xml + **/*.json + **/*.yml + + conf + + + + + ${basedir}/../dolphinscheduler-server/target/dolphinscheduler-server-${project.version} + + **/*.* + + . + + + + ${basedir}/../dolphinscheduler-api/target/dolphinscheduler-api-${project.version} + + **/*.* + + . + + + + ${basedir}/../dolphinscheduler-alert/target/dolphinscheduler-alert-${project.version} + + **/*.* + + . + + + + ${basedir}/../dolphinscheduler-ui/dist + + **/*.* + + ./ui/dist + + + + ${basedir}/../dolphinscheduler-ui + + install-dolphinscheduler-ui.sh + + ./ui + + + + ${basedir}/../sql + + **/* + + ./sql + + + + ${basedir}/../script + + *.* + + ./script + + + + ${basedir}/../script + + config/*.* + env/*.* + + ./conf + + + + ${basedir}/../script + + start-all.sh + stop-all.sh + dolphinscheduler-daemon.sh + + ./bin + + + + ${basedir}/.././ + + *.sh + *.py + DISCLAIMER + + . + + + + ${basedir}/release-docs + true + + **/* + + . + + + + + + + lib + true + + javax.servlet:servlet-api + org.eclipse.jetty.aggregate:jetty-all + org.slf4j:slf4j-log4j12 + + + + \ No newline at end of file diff --git a/dolphinscheduler-ui/pom.xml b/dolphinscheduler-ui/pom.xml index 9f338a2977..28f584f49a 100644 --- a/dolphinscheduler-ui/pom.xml +++ b/dolphinscheduler-ui/pom.xml @@ -32,55 +32,120 @@ v12.12.0 6.11.3 + + + release + + + + com.github.eirslett + frontend-maven-plugin + ${frontend-maven-plugin.version} + + + install node and npm + + install-node-and-npm + + + ${node.version} + ${npm.version} + + + + npm install node-sass --unsafe-perm + + npm + + generate-resources + + install node-sass --unsafe-perm + + + + npm install + + npm + + generate-resources + + install + + + + npm run build:release + + npm + + + run build:release + + + + + + + + + + + nginx + + + + com.github.eirslett + frontend-maven-plugin + ${frontend-maven-plugin.version} + + + install node and npm + + install-node-and-npm + + + ${node.version} + ${npm.version} + + + + npm install node-sass --unsafe-perm + + npm + + generate-resources + + install node-sass --unsafe-perm + + + + npm install + + npm + + generate-resources + + install + + + + npm run build + + npm + + + run build + + + + + + + + + + + + + - - - - com.github.eirslett - frontend-maven-plugin - ${frontend-maven-plugin.version} - - - install node and npm - - install-node-and-npm - - - ${node.version} - ${npm.version} - - - - npm install node-sass --unsafe-perm - - npm - - generate-resources - - install node-sass --unsafe-perm - - - - npm install - - npm - - generate-resources - - install - - - - npm run build:release - - npm - - - run build:release - - - - - - From 7826843620fb31d312f13a017dbf06e3d7ef5e25 Mon Sep 17 00:00:00 2001 From: samz406 Date: Fri, 27 Dec 2019 15:02:49 +0800 Subject: [PATCH 25/53] update token add exist check (#1598) * misspell words * modify common queue TaskQueueZKImplTest.java unit test * extends BaseTaskQueueTest get zkServer * modify zk config * updateToken add entity exist check --- .../dolphinscheduler/api/service/AccessTokenService.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java index 4664b59763..897646ba70 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java @@ -154,8 +154,13 @@ public class AccessTokenService extends BaseService { */ public Map updateToken(int id,int userId, String expireTime, String token) { Map result = new HashMap<>(5); - AccessToken accessToken = new AccessToken(); - accessToken.setId(id); + + AccessToken accessToken = accessTokenMapper.selectById(id); + if (accessToken == null) { + logger.error("access token not exist, access token id {}", id); + putMsg(result, Status.ACCESS_TOKEN_NOT_EXIST); + return result; + } accessToken.setUserId(userId); accessToken.setExpireTime(DateUtils.stringToDate(expireTime)); accessToken.setToken(token); From 3b653328ef7d441505a3a9d89e187f4d4d1bc1bd Mon Sep 17 00:00:00 2001 From: JinyLeeChina <297061848@qq.com> Date: Fri, 27 Dec 2019 15:20:57 +0800 Subject: [PATCH 26/53] '#1595' --- .../server/utils/AlertManager.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java index 8ae96a3136..f3441edd17 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java @@ -90,15 +90,15 @@ public class AlertManager { * process instance format */ private static final String PROCESS_INSTANCE_FORMAT = - "\"Id:%d\"," + - "\"Name:%s\"," + - "\"Job type: %s\"," + - "\"State: %s\"," + - "\"Recovery:%s\"," + - "\"Run time: %d\"," + - "\"Start time: %s\"," + - "\"End time: %s\"," + - "\"Host: %s\"" ; + "\"id:%d\"," + + "\"name:%s\"," + + "\"job type: %s\"," + + "\"state: %s\"," + + "\"recovery:%s\"," + + "\"run time: %d\"," + + "\"start time: %s\"," + + "\"end time: %s\"," + + "\"host: %s\"" ; /** * get process instance content @@ -234,7 +234,7 @@ public class AlertManager { String cmdName = getCommandCnName(processInstance.getCommandType()); String success = processInstance.getState().typeIsSuccess() ? "success" :"failed"; - alert.setTitle(cmdName + success); + alert.setTitle(cmdName + " " + success); ShowType showType = processInstance.getState().typeIsSuccess() ? ShowType.TEXT : ShowType.TABLE; alert.setShowType(showType); String content = getContentProcessInstance(processInstance, taskInstances); From 67367658442af0ede799afc9db668b8a1af381a4 Mon Sep 17 00:00:00 2001 From: lilin Date: Fri, 27 Dec 2019 15:22:25 +0800 Subject: [PATCH 27/53] 1 word error 2 modify check project perm may NPE 3 modify checkResult may NPE 4 update project add desc check --- .../api/service/ProjectService.java | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectService.java index f33a365912..8f9a62000a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectService.java @@ -41,7 +41,7 @@ import static org.apache.dolphinscheduler.api.utils.CheckUtils.checkDesc; /** * project service *HttpTask./ -**/ + **/ @Service public class ProjectService extends BaseService{ @@ -121,7 +121,7 @@ public class ProjectService extends BaseService{ * @param loginUser login user * @param project project * @param projectName project name - * @return true if the login user havve permission to see the project + * @return true if the login user have permission to see the project */ public Map checkProjectAndAuth(User loginUser, Project project, String projectName) { @@ -143,7 +143,7 @@ public class ProjectService extends BaseService{ public boolean hasProjectAndPerm(User loginUser, Project project, Map result) { boolean checkResult = false; if (project == null) { - putMsg(result, Status.PROJECT_NOT_FOUNT, project.getName()); + putMsg(result, Status.PROJECT_NOT_FOUNT, ""); } else if (!checkReadPermission(loginUser, project)) { putMsg(result, Status.USER_NO_OPERATION_PROJECT_PERM, loginUser.getUserName(), project.getName()); } else { @@ -199,13 +199,14 @@ public class ProjectService extends BaseService{ if (checkResult != null) { return checkResult; } - List processDefinitionList = processDefinitionMapper.queryAllDefinitionList(projectId); if (!hasPerm(loginUser, project.getUserId())) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } + List processDefinitionList = processDefinitionMapper.queryAllDefinitionList(projectId); + if(processDefinitionList.size() > 0){ putMsg(result, Status.DELETE_PROJECT_ERROR_DEFINES_NOT_NULL); return result; @@ -227,7 +228,8 @@ public class ProjectService extends BaseService{ * @return check result */ private Map getCheckResult(User loginUser, Project project) { - Map checkResult = checkProjectAndAuth(loginUser, project, project.getName()); + String projectName = project == null ? null:project.getName(); + Map checkResult = checkProjectAndAuth(loginUser, project, projectName); Status status = (Status) checkResult.get(Constants.STATUS); if (status != Status.SUCCESS) { return checkResult; @@ -247,6 +249,11 @@ public class ProjectService extends BaseService{ public Map update(User loginUser, Integer projectId, String projectName, String desc) { Map result = new HashMap<>(5); + Map descCheck = checkDesc(desc); + if (descCheck.get(Constants.STATUS) != Status.SUCCESS) { + return descCheck; + } + Project project = projectMapper.selectById(projectId); boolean hasProjectAndPerm = hasProjectAndPerm(loginUser, project, result); if (!hasProjectAndPerm) { From c17fe38a6ceea589109fa332909de89ce96b467d Mon Sep 17 00:00:00 2001 From: "DK.Pino" Date: Fri, 27 Dec 2019 16:41:34 +0800 Subject: [PATCH 28/53] remove lombok annocation (#1605) --- .../dolphinscheduler/common/enums/AlertType.java | 10 ++++++++-- .../dolphinscheduler/common/enums/FailureStrategy.java | 10 ++++++++-- .../dolphinscheduler/common/enums/ReleaseState.java | 10 ++++++++-- .../apache/dolphinscheduler/common/enums/RunMode.java | 10 ++++++++-- .../dolphinscheduler/common/enums/TaskDependType.java | 10 ++++++++-- .../dolphinscheduler/common/enums/WarningType.java | 10 ++++++++-- 6 files changed, 48 insertions(+), 12 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertType.java index e7c3b24a6c..3c757f5337 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertType.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * warning message notification method */ -@Getter public enum AlertType { /** * 0 email; 1 SMS @@ -39,4 +37,12 @@ public enum AlertType { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/FailureStrategy.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/FailureStrategy.java index 6582d2056c..c9c0c32930 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/FailureStrategy.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/FailureStrategy.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * failure policy when some task node failed. */ -@Getter public enum FailureStrategy { /** @@ -40,4 +38,12 @@ public enum FailureStrategy { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ReleaseState.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ReleaseState.java index 41662a4f67..51e9a3393b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ReleaseState.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ReleaseState.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * process define release state */ -@Getter public enum ReleaseState { /** @@ -50,4 +48,12 @@ public enum ReleaseState { //For values out of enum scope return null; } + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/RunMode.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/RunMode.java index 1f751ffaeb..44fdb5b1b6 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/RunMode.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/RunMode.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * complement data run mode */ -@Getter public enum RunMode { /** * 0 serial run @@ -39,4 +37,12 @@ public enum RunMode { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskDependType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskDependType.java index 590e178350..401fecf3ea 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskDependType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskDependType.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * task node depend type */ -@Getter public enum TaskDependType { /** * 0 run current tasks only @@ -41,4 +39,12 @@ public enum TaskDependType { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WarningType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WarningType.java index ce9124b8b8..3a65760716 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WarningType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WarningType.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * types for whether to send warning when process ending; */ -@Getter public enum WarningType { /** * 0 do not send warning; @@ -44,4 +42,12 @@ public enum WarningType { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } From 3c6780c3cd155e357b21879bd2c1b99d6e7713c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A9=E5=90=88=E6=B0=91?= Date: Fri, 27 Dec 2019 16:53:51 +0800 Subject: [PATCH 29/53] =?UTF-8?q?fix=EF=BC=9AIn=20a=20cluster=20environmen?= =?UTF-8?q?t,=20verify=20that=20nodes=20have=20matching=20bug.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../apache/dolphinscheduler/common/zk/AbstractZKClient.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java index 5b937ce46d..0e95dddb36 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java @@ -144,7 +144,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ String parentPath = getZNodeParentPath(zkNodeType); String serverPathPrefix = parentPath + "/" + OSUtils.getHost(); String registerPath = zkClient.create().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath( - serverPathPrefix + "_", heartbeatZKInfo.getBytes()); + serverPathPrefix + UNDERLINE, heartbeatZKInfo.getBytes()); logger.info("register {} node {} success" , zkNodeType.toString(), registerPath); return registerPath; } @@ -307,7 +307,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ } Map serverMaps = getServerMaps(zkNodeType); for(String hostKey : serverMaps.keySet()){ - if(hostKey.startsWith(host)){ + if(hostKey.startsWith(host + UNDERLINE)){ return true; } } From aadf0aae46fd6788be59ce410af7e6a0cf59a7c7 Mon Sep 17 00:00:00 2001 From: "DK.Pino" Date: Fri, 27 Dec 2019 16:59:48 +0800 Subject: [PATCH 30/53] remove lombok annocation (#1600) --- .../org/apache/dolphinscheduler/common/enums/Flag.java | 10 ++++++++-- .../apache/dolphinscheduler/common/enums/Priority.java | 9 +++++++-- .../dolphinscheduler/common/enums/ResourceType.java | 10 ++++++++-- .../apache/dolphinscheduler/common/enums/ShowType.java | 10 ++++++++-- .../apache/dolphinscheduler/common/enums/TaskType.java | 9 +++++++-- .../apache/dolphinscheduler/common/enums/UserType.java | 10 ++++++++-- 6 files changed, 46 insertions(+), 12 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Flag.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Flag.java index a4a0d41162..622e9d17d4 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Flag.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Flag.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * have_script @@ -27,7 +26,6 @@ import lombok.Getter; * have_map_variables * have_alert */ -@Getter public enum Flag { /** * 0 no @@ -45,4 +43,12 @@ public enum Flag { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Priority.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Priority.java index 9d20ed8eed..bdd7128eac 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Priority.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Priority.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * define process and task priority */ -@Getter public enum Priority { /** * 0 highest priority @@ -46,4 +44,11 @@ public enum Priority { private final int code; private final String descp; + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ResourceType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ResourceType.java index 1b8c47bf92..043402c2ae 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ResourceType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ResourceType.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * resource type */ -@Getter public enum ResourceType { /** * 0 file, 1 udf @@ -39,4 +37,12 @@ public enum ResourceType { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ShowType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ShowType.java index c7d8e64a3e..19e552d765 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ShowType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ShowType.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * show type for email */ -@Getter public enum ShowType { /** * 0 TABLE; @@ -44,4 +42,12 @@ public enum ShowType { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskType.java index f924825f19..45f36883e3 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskType.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * task node type */ -@Getter public enum TaskType { /** * 0 SHELL @@ -61,4 +59,11 @@ public enum TaskType { return !(taskType == TaskType.SUB_PROCESS || taskType == TaskType.DEPENDENT); } + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UserType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UserType.java index 229a9bccd5..75a5df6fb9 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UserType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UserType.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * user type */ -@Getter public enum UserType { /** * 0 admin user; 1 general user @@ -39,5 +37,13 @@ public enum UserType { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } From 8b2b5ba684090233294e23bbfafc57fbfffd5d4d Mon Sep 17 00:00:00 2001 From: Tboy Date: Fri, 27 Dec 2019 17:00:06 +0800 Subject: [PATCH 31/53] refactor TaskQueueZkImpl (#1591) * fix #1515 * sleep when resource in not satisfy. fix #1522 * add sleep 1s for no command * fix MasterBaseTaskExecThread submit method bug * updates * add log * delete lombok * remove duplicate code * refactor TaskQueueZkImpl * ignore First , we have to rewrite * updates --- .../common/queue/TaskQueueFactory.java | 3 +- .../common/queue/TaskQueueZkImpl.java | 163 +++--------------- .../utils/SpringApplicationContext.java | 2 +- .../common/zk/ZookeeperOperator.java | 3 - .../common/queue/TaskQueueZKImplTest.java | 4 +- .../dolphinscheduler/dao/ProcessDao.java | 3 +- .../dao/mapper/Application.java | 2 +- .../server/master/MasterServer.java | 3 +- .../runner/MasterBaseTaskExecThread.java | 2 +- .../master/runner/MasterExecThread.java | 1 - .../master/runner/MasterSchedulerThread.java | 2 +- .../server/worker/WorkerServer.java | 2 +- .../server/worker/runner/FetchTaskThread.java | 3 +- .../server/worker/task/AbstractYarnTask.java | 2 +- .../task/dependent/DependentExecute.java | 2 +- .../worker/task/dependent/DependentTask.java | 2 +- .../server/worker/task/http/HttpTask.java | 2 +- .../task/processdure/ProcedureTask.java | 2 +- .../server/worker/task/python/PythonTask.java | 2 +- .../server/worker/task/shell/ShellTask.java | 2 +- .../server/worker/task/sql/SqlTask.java | 2 +- .../shell/ShellCommandExecutorTest.java | 2 +- .../server/worker/sql/SqlExecutorTest.java | 2 +- 23 files changed, 46 insertions(+), 167 deletions(-) rename {dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server => dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common}/utils/SpringApplicationContext.java (96%) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueFactory.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueFactory.java index a82a3098f2..0a2d943118 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueFactory.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueFactory.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.common.queue; import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.commons.lang.StringUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,7 +44,7 @@ public class TaskQueueFactory { String queueImplValue = CommonUtils.getQueueImplValue(); if (StringUtils.isNotBlank(queueImplValue)) { logger.info("task queue impl use zookeeper "); - return TaskQueueZkImpl.getInstance(); + return SpringApplicationContext.getBean(TaskQueueZkImpl.class); }else{ logger.error("property dolphinscheduler.queue.impl can't be blank, system will exit "); System.exit(-1); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java index 5eca993120..45c6122341 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java @@ -17,22 +17,14 @@ package org.apache.dolphinscheduler.common.queue; -import org.apache.commons.configuration.Configuration; -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.Bytes; import org.apache.dolphinscheduler.common.utils.IpUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; -import org.apache.dolphinscheduler.common.zk.DefaultEnsembleProvider; -import org.apache.dolphinscheduler.common.zk.ZookeeperConfig; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.data.Stat; +import org.apache.dolphinscheduler.common.zk.ZookeeperOperator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; import java.util.*; @@ -40,35 +32,13 @@ import java.util.*; * A singleton of a task queue implemented with zookeeper * tasks queue implemention */ +@Service public class TaskQueueZkImpl implements ITaskQueue { private static final Logger logger = LoggerFactory.getLogger(TaskQueueZkImpl.class); - private static volatile TaskQueueZkImpl instance; - - private CuratorFramework zkClient; - - private ZookeeperConfig zookeeperConfig; - - private CuratorFramework getZkClient() { - return zkClient; - } - - private TaskQueueZkImpl(){ - init(); - } - - public static TaskQueueZkImpl getInstance(){ - if (null == instance) { - synchronized (TaskQueueZkImpl.class) { - if(null == instance) { - instance = new TaskQueueZkImpl(); - } - } - } - return instance; - } - + @Autowired + private ZookeeperOperator zookeeperOperator; /** * get all tasks from tasks queue @@ -78,14 +48,12 @@ public class TaskQueueZkImpl implements ITaskQueue { @Override public List getAllTasks(String key) { try { - List list = getZkClient().getChildren().forPath(getTasksPath(key)); - + List list = zookeeperOperator.getChildrenKeys(getTasksPath(key)); return list; } catch (Exception e) { logger.error("get all tasks from tasks queue exception",e); } - - return new ArrayList(); + return new ArrayList<>(); } /** @@ -99,22 +67,8 @@ public class TaskQueueZkImpl implements ITaskQueue { public boolean checkTaskExists(String key, String task) { String taskPath = getTasksPath(key) + Constants.SINGLE_SLASH + task; - try { - Stat stat = zkClient.checkExists().forPath(taskPath); - - if(null == stat){ - logger.info("check task:{} not exist in task queue",task); - return false; - }else{ - logger.info("check task {} exists in task queue ",task); - return true; - } + return zookeeperOperator.isExisted(taskPath); - } catch (Exception e) { - logger.info(String.format("task {} check exists in task queue exception ", task), e); - } - - return false; } @@ -128,9 +82,7 @@ public class TaskQueueZkImpl implements ITaskQueue { public boolean add(String key, String value){ try { String taskIdPath = getTasksPath(key) + Constants.SINGLE_SLASH + value; - String result = getZkClient().create().withMode(CreateMode.PERSISTENT).forPath(taskIdPath, Bytes.toBytes(value)); - - logger.info("add task : {} to tasks queue , result success",result); + zookeeperOperator.persist(taskIdPath, value); return true; } catch (Exception e) { logger.error("add task to tasks queue exception",e); @@ -153,8 +105,7 @@ public class TaskQueueZkImpl implements ITaskQueue { @Override public List poll(String key, int tasksNum) { try{ - CuratorFramework zk = getZkClient(); - List list = zk.getChildren().forPath(getTasksPath(key)); + List list = zookeeperOperator.getChildrenKeys(getTasksPath(key)); if(list != null && list.size() > 0){ @@ -277,15 +228,12 @@ public class TaskQueueZkImpl implements ITaskQueue { @Override public void removeNode(String key, String nodeValue){ - CuratorFramework zk = getZkClient(); String tasksQueuePath = getTasksPath(key) + Constants.SINGLE_SLASH; String taskIdPath = tasksQueuePath + nodeValue; - logger.info("consume task {}", taskIdPath); + logger.info("removeNode task {}", taskIdPath); try{ - Stat stat = zk.checkExists().forPath(taskIdPath); - if(stat != null){ - zk.delete().forPath(taskIdPath); - } + zookeeperOperator.remove(taskIdPath); + }catch(Exception e){ logger.error(String.format("delete task:%s from zookeeper fail, exception:" ,nodeValue) ,e); } @@ -307,13 +255,10 @@ public class TaskQueueZkImpl implements ITaskQueue { if(value != null && value.trim().length() > 0){ String path = getTasksPath(key) + Constants.SINGLE_SLASH; - CuratorFramework zk = getZkClient(); - Stat stat = zk.checkExists().forPath(path + value); - - if(null == stat){ - String result = zk.create().withMode(CreateMode.PERSISTENT).forPath(path + value,Bytes.toBytes(value)); - logger.info("add task:{} to tasks set result:{} ",value,result); - }else{ + if(!zookeeperOperator.isExisted(path + value)){ + zookeeperOperator.persist(path + value,value); + logger.info("add task:{} to tasks set ",value); + } else{ logger.info("task {} exists in tasks set ",value); } @@ -336,15 +281,7 @@ public class TaskQueueZkImpl implements ITaskQueue { public void srem(String key, String value) { try{ String path = getTasksPath(key) + Constants.SINGLE_SLASH; - CuratorFramework zk = getZkClient(); - Stat stat = zk.checkExists().forPath(path + value); - - if(null != stat){ - zk.delete().forPath(path + value); - logger.info("delete task:{} from tasks set ",value); - }else{ - logger.info("delete task:{} from tasks set fail, there is no this task",value); - } + zookeeperOperator.remove(path + value); }catch(Exception e){ logger.error(String.format("delete task:" + value + " exception"),e); @@ -363,7 +300,7 @@ public class TaskQueueZkImpl implements ITaskQueue { Set tasksSet = new HashSet<>(); try { - List list = getZkClient().getChildren().forPath(getTasksPath(key)); + List list = zookeeperOperator.getChildrenKeys(getTasksPath(key)); for (String task : list) { tasksSet.add(task); @@ -377,56 +314,6 @@ public class TaskQueueZkImpl implements ITaskQueue { return tasksSet; } - - - /** - * Init the task queue of zookeeper node - */ - private void init(){ - initZkClient(); - try { - String tasksQueuePath = getTasksPath(Constants.DOLPHINSCHEDULER_TASKS_QUEUE); - String tasksCancelPath = getTasksPath(Constants.DOLPHINSCHEDULER_TASKS_KILL); - - for(String taskQueuePath : new String[]{tasksQueuePath,tasksCancelPath}){ - if(zkClient.checkExists().forPath(taskQueuePath) == null){ - // create a persistent parent node - zkClient.create().creatingParentContainersIfNeeded() - .withMode(CreateMode.PERSISTENT).forPath(taskQueuePath); - logger.info("create tasks queue parent node success : {} ",taskQueuePath); - } - } - - } catch (Exception e) { - logger.error("create zk node failure",e); - } - } - - private void initZkClient() { - - Configuration conf = null; - try { - conf = new PropertiesConfiguration(Constants.ZOOKEEPER_PROPERTIES_PATH); - } catch (ConfigurationException ex) { - logger.error("load zookeeper properties file failed, system exit"); - System.exit(-1); - } - - zkClient = CuratorFrameworkFactory.builder().ensembleProvider(new DefaultEnsembleProvider(conf.getString("zookeeper.quorum"))) - .retryPolicy(new ExponentialBackoffRetry(conf.getInt("zookeeper.retry.base.sleep"), conf.getInt("zookeeper.retry.maxtime"), conf.getInt("zookeeper.retry.max.sleep"))) - .sessionTimeoutMs(conf.getInt("zookeeper.session.timeout")) - .connectionTimeoutMs(conf.getInt("zookeeper.connection.timeout")) - .build(); - - zkClient.start(); - try { - zkClient.blockUntilConnected(); - } catch (final Exception ex) { - throw new RuntimeException(ex); - } - } - - /** * Clear the task queue of zookeeper node */ @@ -437,16 +324,12 @@ public class TaskQueueZkImpl implements ITaskQueue { String tasksCancelPath = getTasksPath(Constants.DOLPHINSCHEDULER_TASKS_KILL); for(String taskQueuePath : new String[]{tasksQueuePath,tasksCancelPath}){ - if(zkClient.checkExists().forPath(taskQueuePath) != null){ - - List list = zkClient.getChildren().forPath(taskQueuePath); - + if(zookeeperOperator.isExisted(taskQueuePath)){ + List list = zookeeperOperator.getChildrenKeys(taskQueuePath); for (String task : list) { - zkClient.delete().forPath(taskQueuePath + Constants.SINGLE_SLASH + task); + zookeeperOperator.remove(taskQueuePath + Constants.SINGLE_SLASH + task); logger.info("delete task from tasks queue : {}/{} ",taskQueuePath,task); - } - } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/SpringApplicationContext.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SpringApplicationContext.java similarity index 96% rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/SpringApplicationContext.java rename to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SpringApplicationContext.java index 96087e5a52..97618e1b39 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/SpringApplicationContext.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SpringApplicationContext.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.server.utils; +package org.apache.dolphinscheduler.common.utils; import org.springframework.beans.BeansException; import org.springframework.context.ApplicationContext; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java index f4d72f436e..5e3751b25d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java @@ -127,9 +127,6 @@ public class ZookeeperOperator implements InitializingBean { List values; try { values = zkClient.getChildren().forPath(key); - if (CollectionUtils.isEmpty(values)) { - logger.warn("getChildrenKeys key : {} is empty", key); - } return values; } catch (InterruptedException ex) { logger.error("getChildrenKeys key : {} InterruptedException", key); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java index b13f4f63c5..1b44673149 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/queue/TaskQueueZKImplTest.java @@ -32,11 +32,9 @@ import static org.junit.Assert.*; /** * task queue test */ +@Ignore public class TaskQueueZKImplTest extends BaseTaskQueueTest { - - - @Before public void before(){ diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java index eb97ad75b2..66948951f6 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java @@ -105,7 +105,8 @@ public class ProcessDao { /** * task queue impl */ - protected ITaskQueue taskQueue = TaskQueueFactory.getTaskQueueInstance(); + @Autowired + private ITaskQueue taskQueue; /** * handle Command (construct ProcessInstance from Command) , wrapped in transaction * @param logger logger diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/Application.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/Application.java index 75b79a4266..02df228f72 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/Application.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/Application.java @@ -21,7 +21,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.ComponentScan; @SpringBootApplication -@ComponentScan("org.apache.dolphinscheduler.dao") +@ComponentScan("org.apache.dolphinscheduler") public class Application { public static void main(String[] args) { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index 8297cd0403..e8c8b6779e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -23,18 +23,17 @@ import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.runner.MasterSchedulerThread; import org.apache.dolphinscheduler.server.quartz.ProcessScheduleJob; import org.apache.dolphinscheduler.server.quartz.QuartzExecutors; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.zk.ZKMasterClient; import org.quartz.SchedulerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.SpringApplication; import org.springframework.boot.WebApplicationType; import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.context.annotation.ComponentScan; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java index 6b4b799ef9..5c96757072 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java @@ -18,13 +18,13 @@ package org.apache.dolphinscheduler.server.master.runner; import org.apache.dolphinscheduler.common.queue.ITaskQueue; import org.apache.dolphinscheduler.common.queue.TaskQueueFactory; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.utils.BeanContext; import org.apache.dolphinscheduler.server.master.config.MasterConfig; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 5446830780..3481b79caa 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -34,7 +34,6 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.utils.DagHelper; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.utils.AlertManager; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java index 8d7d5a0add..5f594b3fa0 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerThread.java @@ -22,12 +22,12 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.common.zk.AbstractZKClient; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.Command; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.server.master.config.MasterConfig; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.zk.ZKMasterClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index 96f5ba0b5d..877d60a33b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -29,12 +29,12 @@ import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.common.zk.AbstractZKClient; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.utils.ProcessUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.runner.FetchTaskThread; import org.apache.dolphinscheduler.server.zk.ZKWorkerClient; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java index 7429050605..ae67716da2 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java @@ -25,12 +25,12 @@ import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.common.zk.AbstractZKClient; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.WorkerGroup; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.zk.ZKWorkerClient; import org.slf4j.Logger; @@ -155,6 +155,7 @@ public class FetchTaskThread implements Runnable{ //whether have tasks, if no tasks , no need lock //get all tasks List tasksQueueList = taskQueue.getAllTasks(Constants.DOLPHINSCHEDULER_TASKS_QUEUE); if (CollectionUtils.isEmpty(tasksQueueList)){ + Thread.sleep(Constants.SLEEP_TIME_MILLIS); continue; } // creating distributed locks, lock path /dolphinscheduler/lock/worker diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java index b9b3ad6824..6846617408 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractYarnTask.java @@ -16,10 +16,10 @@ */ package org.apache.dolphinscheduler.server.worker.task; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.utils.ProcessUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.slf4j.Logger; /** diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java index 5dc25b8935..4be65ed49d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java @@ -23,10 +23,10 @@ import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.DateInterval; import org.apache.dolphinscheduler.common.model.DependentItem; import org.apache.dolphinscheduler.common.utils.DependentUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java index b0bb4c6f4c..9af29e01dd 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java @@ -25,9 +25,9 @@ import org.apache.dolphinscheduler.common.task.dependent.DependentParameters; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.utils.DependentUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskProps; import org.slf4j.Logger; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java index 44eef65aba..993310f6ec 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java @@ -30,10 +30,10 @@ import org.apache.dolphinscheduler.common.task.http.HttpParameters; import org.apache.dolphinscheduler.common.utils.Bytes; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.server.utils.ParamUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskProps; import org.apache.http.HttpEntity; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java index 59cf8a6e24..9b4952bbd2 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java @@ -29,10 +29,10 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.procedure.ProcedureParameters; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.server.utils.ParamUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskProps; import org.slf4j.Logger; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java index f6227b15a4..585d62f154 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java @@ -22,9 +22,9 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.python.PythonParameters; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.server.utils.ParamUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.PythonCommandExecutor; import org.apache.dolphinscheduler.server.worker.task.TaskProps; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java index 438d373775..789a0c5302 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java @@ -23,9 +23,9 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.shell.ShellParameters; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.server.utils.ParamUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor; import org.apache.dolphinscheduler.server.worker.task.TaskProps; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index ccfee2efec..08a90c62ce 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -36,6 +36,7 @@ import org.apache.dolphinscheduler.common.task.sql.SqlType; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.DataSource; @@ -43,7 +44,6 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.server.utils.ParamUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.utils.UDFUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskProps; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index 71bebe2990..1117fe0015 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -20,10 +20,10 @@ import com.alibaba.fastjson.JSONObject; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.utils.LoggerUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskManager; import org.apache.dolphinscheduler.server.worker.task.TaskProps; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java index 725f2835e9..7cf4b874d1 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java @@ -21,10 +21,10 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.utils.LoggerUtils; -import org.apache.dolphinscheduler.server.utils.SpringApplicationContext; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskManager; import org.apache.dolphinscheduler.server.worker.task.TaskProps; From 597d685955a3d3028039103edeb1b723077b3b7d Mon Sep 17 00:00:00 2001 From: samz406 Date: Fri, 27 Dec 2019 17:00:33 +0800 Subject: [PATCH 32/53] delAlertgroupById add entity exist check (#1602) * misspell words * modify common queue TaskQueueZKImplTest.java unit test * extends BaseTaskQueueTest get zkServer * modify zk config * add MonitorServiceTest * mobile phone need 11 number * delete file * delAlertgroupById add entity exist check --- .../dolphinscheduler/api/service/AlertGroupService.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java index 40fc65b4dc..63f50f936f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java @@ -193,6 +193,12 @@ public class AlertGroupService extends BaseService{ if (checkAdmin(loginUser, result)){ return result; } + //check exist + AlertGroup alertGroup = alertGroupMapper.selectById(id); + if (alertGroup == null) { + putMsg(result, Status.ALERT_GROUP_NOT_EXIST); + return result; + } userAlertGroupMapper.deleteByAlertgroupId(id); alertGroupMapper.deleteById(id); From 63f396b61f77cc44e462d35928f46ef2710adfd2 Mon Sep 17 00:00:00 2001 From: "DK.Pino" Date: Fri, 27 Dec 2019 17:03:10 +0800 Subject: [PATCH 33/53] refactor zk tree cache (#1577) * refactor zk tree cache * refactor zk tree cache --- .../common/zk/AbstractListener.java | 35 ------- .../common/zk/ZookeeperCachedOperator.java | 48 ++++++---- .../common/zk/ZookeeperOperator.java | 8 +- .../server/zk/ZKMasterClient.java | 96 +++++++++---------- .../server/zk/ZKWorkerClient.java | 61 ++++++------ 5 files changed, 107 insertions(+), 141 deletions(-) delete mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractListener.java diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractListener.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractListener.java deleted file mode 100644 index d84b9f7e11..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractListener.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.common.zk; - -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.recipes.cache.TreeCacheEvent; -import org.apache.curator.framework.recipes.cache.TreeCacheListener; - -public abstract class AbstractListener implements TreeCacheListener { - - @Override - public final void childEvent(final CuratorFramework client, final TreeCacheEvent event) throws Exception { - String path = null == event.getData() ? "" : event.getData().getPath(); - if (path.isEmpty()) { - return; - } - dataChanged(client, event, path); - } - - protected abstract void dataChanged(final CuratorFramework client, final TreeCacheEvent event, final String path); -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperCachedOperator.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperCachedOperator.java index cf4980147e..daec765315 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperCachedOperator.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperCachedOperator.java @@ -16,8 +16,10 @@ */ package org.apache.dolphinscheduler.common.zk; +import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.TreeCache; +import org.apache.curator.framework.recipes.cache.TreeCacheEvent; import org.apache.curator.framework.recipes.cache.TreeCacheListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,30 +36,37 @@ public class ZookeeperCachedOperator extends ZookeeperOperator { private final Logger logger = LoggerFactory.getLogger(ZookeeperCachedOperator.class); - //kay is zk path, value is TreeCache - private ConcurrentHashMap allCaches = new ConcurrentHashMap<>(); + TreeCache treeCache; /** - * @param cachePath zk path - * @param listener operator + * register a unified listener of /${dsRoot}, */ - public void registerListener(final String cachePath, final TreeCacheListener listener) { - TreeCache newCache = new TreeCache(zkClient, cachePath); - logger.info("add listener to zk path: {}", cachePath); + @Override + protected void registerListener() { + treeCache = new TreeCache(zkClient, getZookeeperConfig().getDsRoot()); + logger.info("add listener to zk path: {}", getZookeeperConfig().getDsRoot()); try { - newCache.start(); + treeCache.start(); } catch (Exception e) { - logger.error("add listener to zk path: {} failed", cachePath); + logger.error("add listener to zk path: {} failed", getZookeeperConfig().getDsRoot()); throw new RuntimeException(e); } - newCache.getListenable().addListener(listener); + treeCache.getListenable().addListener((client, event) -> { + String path = null == event.getData() ? "" : event.getData().getPath(); + if (path.isEmpty()) { + return; + } + dataChanged(client, event, path); + }); - allCaches.put(cachePath, newCache); } + //for sub class + protected void dataChanged(final CuratorFramework client, final TreeCacheEvent event, final String path){} + public String getFromCache(final String cachePath, final String key) { - ChildData resultInCache = allCaches.get(checkNotNull(cachePath)).getCurrentData(key); + ChildData resultInCache = treeCache.getCurrentData(key); if (null != resultInCache) { return null == resultInCache.getData() ? null : new String(resultInCache.getData(), StandardCharsets.UTF_8); } @@ -65,18 +74,15 @@ public class ZookeeperCachedOperator extends ZookeeperOperator { } public TreeCache getTreeCache(final String cachePath) { - return allCaches.get(checkNotNull(cachePath)); + return treeCache; } public void close() { - - allCaches.forEach((path, cache) -> { - cache.close(); - try { - Thread.sleep(500); - } catch (InterruptedException ignore) { - } - }); + treeCache.close(); + try { + Thread.sleep(500); + } catch (InterruptedException ignore) { + } super.close(); } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java index 5e3751b25d..c6faec2b78 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java @@ -57,11 +57,13 @@ public class ZookeeperOperator implements InitializingBean { public void afterPropertiesSet() throws Exception { this.zkClient = buildClient(); initStateLister(); - //init(); + registerListener(); } - //for subclass - //protected void init(){} + /** + * this method is for sub class, + */ + protected void registerListener(){} public void initStateLister() { checkNotNull(zkClient); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java index 2aec6ecaf6..a26a217665 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java @@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ZKNodeType; import org.apache.dolphinscheduler.common.model.Server; -import org.apache.dolphinscheduler.common.zk.AbstractListener; import org.apache.dolphinscheduler.common.zk.AbstractZKClient; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.DaoFactory; @@ -31,9 +30,6 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.utils.ProcessUtils; import org.apache.commons.lang.StringUtils; import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.recipes.cache.PathChildrenCache; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.apache.curator.utils.ThreadUtils; import org.slf4j.Logger; @@ -101,12 +97,6 @@ public class ZKMasterClient extends AbstractZKClient { // init system znode this.initSystemZNode(); - // monitor master - this.listenerMaster(); - - // monitor worker - this.listenerWorker(); - // register master this.registerMaster(); @@ -158,31 +148,22 @@ public class ZKMasterClient extends AbstractZKClient { } } - /** - * monitor master + * handle path events that this class cares about + * @param client zkClient + * @param event path event + * @param path zk path */ - public void listenerMaster(){ - registerListener(getZNodeParentPath(ZKNodeType.MASTER), new AbstractListener() { - @Override - protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) { - switch (event.getType()) { - case NODE_ADDED: - logger.info("master node added : {}", path); - break; - case NODE_REMOVED: - String serverHost = getHostByEventDataPath(path); - if (checkServerSelfDead(serverHost, ZKNodeType.MASTER)) { - return; - } - removeZKNodePath(path, ZKNodeType.MASTER, true); - break; - default: - break; - } - } - }); -} + @Override + protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) { + if(path.startsWith(getZNodeParentPath(ZKNodeType.MASTER)+Constants.SINGLE_SLASH)){ //monitor master + handleMasterEvent(event,path); + + }else if(path.startsWith(getZNodeParentPath(ZKNodeType.WORKER)+Constants.SINGLE_SLASH)){ //monitor worker + handleWorkerEvent(event,path); + } + //other path event, ignore + } /** * remove zookeeper node path @@ -273,25 +254,40 @@ public class ZKMasterClient extends AbstractZKClient { } /** - * monitor worker + * monitor master */ - public void listenerWorker(){ - registerListener(getZNodeParentPath(ZKNodeType.WORKER), new AbstractListener() { - @Override - protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) { - switch (event.getType()) { - case NODE_ADDED: - logger.info("worker node added : {}", path); - break; - case NODE_REMOVED: - logger.info("worker node deleted : {}", path); - removeZKNodePath(path, ZKNodeType.WORKER, true); - break; - default: - break; + public void handleMasterEvent(TreeCacheEvent event, String path){ + switch (event.getType()) { + case NODE_ADDED: + logger.info("master node added : {}", path); + break; + case NODE_REMOVED: + String serverHost = getHostByEventDataPath(path); + if (checkServerSelfDead(serverHost, ZKNodeType.MASTER)) { + return; } - } - }); + removeZKNodePath(path, ZKNodeType.MASTER, true); + break; + default: + break; + } + } + + /** + * monitor worker + */ + public void handleWorkerEvent(TreeCacheEvent event, String path){ + switch (event.getType()) { + case NODE_ADDED: + logger.info("worker node added : {}", path); + break; + case NODE_REMOVED: + logger.info("worker node deleted : {}", path); + removeZKNodePath(path, ZKNodeType.WORKER, true); + break; + default: + break; + } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java index 0dd1cf15be..2e063d50d5 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java @@ -19,20 +19,13 @@ package org.apache.dolphinscheduler.server.zk; import org.apache.curator.framework.recipes.cache.TreeCacheEvent; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ZKNodeType; -import org.apache.dolphinscheduler.common.zk.AbstractListener; import org.apache.dolphinscheduler.common.zk.AbstractZKClient; import org.apache.commons.lang.StringUtils; import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.recipes.cache.PathChildrenCache; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; -import org.apache.curator.utils.ThreadUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; -import java.util.concurrent.ThreadFactory; - /** * zookeeper worker client @@ -61,9 +54,6 @@ public class ZKWorkerClient extends AbstractZKClient { // init system znode this.initSystemZNode(); - // monitor worker - this.listenerWorker(); - // register worker this.registWorker(); } @@ -83,31 +73,38 @@ public class ZKWorkerClient extends AbstractZKClient { System.exit(-1); } } - + /** - * monitor worker + * handle path events that this class cares about + * @param client zkClient + * @param event path event + * @param path zk path */ - private void listenerWorker(){ - registerListener(getZNodeParentPath(ZKNodeType.WORKER), new AbstractListener() { - @Override - protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) { - switch (event.getType()) { - case NODE_ADDED: - logger.info("worker node added : {}", path); - break; - case NODE_REMOVED: - //find myself dead - String serverHost = getHostByEventDataPath(path); - if(checkServerSelfDead(serverHost, ZKNodeType.WORKER)){ - return; - } - break; - default: - break; - } - } - }); + @Override + protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) { + if(path.startsWith(getZNodeParentPath(ZKNodeType.WORKER)+Constants.SINGLE_SLASH)){ + handleWorkerEvent(event,path); + } + } + /** + * monitor worker + */ + public void handleWorkerEvent(TreeCacheEvent event, String path){ + switch (event.getType()) { + case NODE_ADDED: + logger.info("worker node added : {}", path); + break; + case NODE_REMOVED: + //find myself dead + String serverHost = getHostByEventDataPath(path); + if(checkServerSelfDead(serverHost, ZKNodeType.WORKER)){ + return; + } + break; + default: + break; + } } /** From b6cca46d8f39993b54f22ae362884537d9499eeb Mon Sep 17 00:00:00 2001 From: "DK.Pino" Date: Fri, 27 Dec 2019 17:04:25 +0800 Subject: [PATCH 34/53] remove lombok annocation (#1603) --- .../common/enums/AlertStatus.java | 10 +++- .../common/enums/CommandType.java | 10 +++- .../dolphinscheduler/common/enums/DbType.java | 60 ++++++++++--------- .../common/enums/ExecutionStatus.java | 8 ++- .../common/enums/SparkVersion.java | 10 +++- .../common/enums/UdfType.java | 10 +++- 6 files changed, 71 insertions(+), 37 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertStatus.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertStatus.java index 7543dc48cd..42ea05f75d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertStatus.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertStatus.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * alert status */ -@Getter public enum AlertStatus { /** * 0 waiting executed; 1 execute successfully，2 execute failed @@ -40,4 +38,12 @@ public enum AlertStatus { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CommandType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CommandType.java index 352f6ef812..1ee79156dc 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CommandType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CommandType.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * command types */ -@Getter public enum CommandType { /** @@ -59,4 +57,12 @@ public enum CommandType { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbType.java index 4637771eda..5fb245afef 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbType.java @@ -17,38 +17,44 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * data base types */ -@Getter public enum DbType { - /** - * 0 mysql - * 1 postgresql - * 2 hive - * 3 spark - * 4 clickhouse - * 5 oracle - * 6 sqlserver - * 7 db2 - */ - MYSQL(0, "mysql"), - POSTGRESQL(1, "postgresql"), - HIVE(2, "hive"), - SPARK(3, "spark"), - CLICKHOUSE(4, "clickhouse"), - ORACLE(5, "oracle"), - SQLSERVER(6, "sqlserver"), - DB2(7, "db2"); + /** + * 0 mysql + * 1 postgresql + * 2 hive + * 3 spark + * 4 clickhouse + * 5 oracle + * 6 sqlserver + * 7 db2 + */ + MYSQL(0, "mysql"), + POSTGRESQL(1, "postgresql"), + HIVE(2, "hive"), + SPARK(3, "spark"), + CLICKHOUSE(4, "clickhouse"), + ORACLE(5, "oracle"), + SQLSERVER(6, "sqlserver"), + DB2(7, "db2"); - DbType(int code, String descp){ - this.code = code; - this.descp = descp; - } + DbType(int code, String descp) { + this.code = code; + this.descp = descp; + } - @EnumValue - private final int code; - private final String descp; + @EnumValue + private final int code; + private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java index f29b8a54aa..12702527f0 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java @@ -18,13 +18,11 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * running status for workflow and task nodes * */ -@Getter public enum ExecutionStatus { /** @@ -123,5 +121,11 @@ public enum ExecutionStatus { return this == KILL || this == STOP ; } + public int getCode() { + return code; + } + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SparkVersion.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SparkVersion.java index e3f7c73797..867d063a64 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SparkVersion.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SparkVersion.java @@ -17,9 +17,7 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; -@Getter public enum SparkVersion { /** @@ -37,4 +35,12 @@ public enum SparkVersion { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UdfType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UdfType.java index a667c05878..22f6752689 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UdfType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UdfType.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.Getter; /** * UDF type */ -@Getter public enum UdfType { /** * 0 hive; 1 spark @@ -38,4 +36,12 @@ public enum UdfType { @EnumValue private final int code; private final String descp; + + public int getCode() { + return code; + } + + public String getDescp() { + return descp; + } } From 7f27c8a86e8d10f30875a74edf0854c68283c873 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Fri, 27 Dec 2019 17:05:28 +0800 Subject: [PATCH 35/53] update assemble and rpmbuild because config/*.* remove script to dolphinscheduler-server/src/main/resources (#1609) --- dolphinscheduler-dist/pom.xml | 2 +- .../src/main/assembly/dolphinscheduler-binary.xml | 2 +- .../src/main/assembly/dolphinscheduler-nginx.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-dist/pom.xml b/dolphinscheduler-dist/pom.xml index 2a7553e3a3..b43daff41b 100644 --- a/dolphinscheduler-dist/pom.xml +++ b/dolphinscheduler-dist/pom.xml @@ -271,6 +271,7 @@ **/*.properties **/*.xml **/*.json + config/*.* @@ -279,7 +280,6 @@ ${basedir}/../script - config/*.* env/*.* diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml index 2bf109698b..b4326c6795 100644 --- a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml +++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml @@ -94,6 +94,7 @@ **/*.properties **/*.xml **/*.json + config/*.* conf @@ -162,7 +163,6 @@ ${basedir}/../script - config/*.* env/*.* ./conf diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml index ebe5b2cb3f..f4e403e4b4 100644 --- a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml +++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml @@ -94,6 +94,7 @@ **/*.properties **/*.xml **/*.json + config/*.* conf @@ -185,7 +186,6 @@ ${basedir}/../script - config/*.* env/*.* ./conf From be5fc116effe456cb9b74b277a3fb5c3988e6870 Mon Sep 17 00:00:00 2001 From: samz406 Date: Fri, 27 Dec 2019 17:20:34 +0800 Subject: [PATCH 36/53] update resource may NPE (#1610) * misspell words * modify common queue TaskQueueZKImplTest.java unit test * extends BaseTaskQueueTest get zkServer * modify zk config * update resource may NPE --- .../apache/dolphinscheduler/api/service/ResourcesService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index 97db9ee1d7..66bf214608 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -210,7 +210,6 @@ public class ResourcesService extends BaseService { } Resource resource = resourcesMapper.selectById(resourceId); - String originResourceName = resource.getAlias(); if (resource == null) { putMsg(result, Status.RESOURCE_NOT_EXIST); return result; @@ -236,6 +235,7 @@ public class ResourcesService extends BaseService { } //get the file suffix + String originResourceName = resource.getAlias(); String suffix = originResourceName.substring(originResourceName.lastIndexOf(".")); //if the name without suffix then add it ,else use the origin name From d91e0155c749ad01dc7ffdb8aa2743f7ba1b0b5c Mon Sep 17 00:00:00 2001 From: lilin Date: Fri, 27 Dec 2019 17:35:15 +0800 Subject: [PATCH 37/53] 1when queue or queName is null,return info param is null 2 update queue may NPE --- .../api/service/QueueService.java | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java index f860517d0f..862c895c92 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java @@ -115,12 +115,12 @@ public class QueueService extends BaseService { } if (StringUtils.isEmpty(queue)) { - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queue); + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queue"); return result; } if (StringUtils.isEmpty(queueName)) { - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queueName); + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queueName"); return result; } @@ -163,6 +163,16 @@ public class QueueService extends BaseService { return result; } + if (StringUtils.isEmpty(queue)) { + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queue"); + return result; + } + + if (StringUtils.isEmpty(queueName)) { + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queueName"); + return result; + } + Queue queueObj = queueMapper.selectById(id); if (queueObj == null) { putMsg(result, Status.QUEUE_NOT_EXIST, id); @@ -222,12 +232,12 @@ public class QueueService extends BaseService { Result result = new Result(); if (StringUtils.isEmpty(queue)) { - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queue); + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queue"); return result; } if (StringUtils.isEmpty(queueName)) { - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queueName); + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "queueName"); return result; } From ce76f3eab763e944a8a8149a5babbc90a085b215 Mon Sep 17 00:00:00 2001 From: lilin Date: Fri, 27 Dec 2019 18:11:48 +0800 Subject: [PATCH 38/53] 1 Modify judgment order 2 before delete check entity exist --- .../api/service/UdfFuncService.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java index 8694f4e7b3..7b4ce63e24 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java @@ -187,6 +187,12 @@ public class UdfFuncService extends BaseService{ // verify udfFunc is exist UdfFunc udf = udfFuncMapper.selectUdfById(udfFuncId); + if (udf == null) { + result.put(Constants.STATUS, Status.UDF_FUNCTION_NOT_EXIST); + result.put(Constants.MSG, Status.UDF_FUNCTION_NOT_EXIST.getMsg()); + return result; + } + // if resource upload startup if (!PropertyUtils.getResUploadStartupState()){ logger.error("resource upload startup state: {}", PropertyUtils.getResUploadStartupState()); @@ -194,12 +200,6 @@ public class UdfFuncService extends BaseService{ return result; } - if (udf == null) { - result.put(Constants.STATUS, Status.UDF_FUNCTION_NOT_EXIST); - result.put(Constants.MSG, Status.UDF_FUNCTION_NOT_EXIST.getMsg()); - return result; - } - // verify udfFuncName is exist if (!funcName.equals(udf.getFuncName())) { if (checkUdfFuncNameExists(funcName)) { @@ -303,7 +303,12 @@ public class UdfFuncService extends BaseService{ @Transactional(rollbackFor = Exception.class) public Result delete(int id) { Result result = new Result(); - + //check exist + UdfFunc udfFunc = udfFuncMapper.selectById(id); + if (udfFunc == null) { + putMsg(result, Status.RESOURCE_NOT_EXIST); + return result; + } udfFuncMapper.deleteById(id); udfUserMapper.deleteByUdfFuncId(id); putMsg(result, Status.SUCCESS); From 62f54eed7f25ee207bee1b4731b08263ab68aadc Mon Sep 17 00:00:00 2001 From: lilin Date: Fri, 27 Dec 2019 18:21:21 +0800 Subject: [PATCH 39/53] =?UTF-8?q?1=20Modify=20judgment=20order(=E8=B0=83?= =?UTF-8?q?=E6=95=B4=E9=A1=BA=E5=BA=8F=EF=BC=8C=E5=85=88=E5=88=A4=E6=96=AD?= =?UTF-8?q?=E6=98=AF=E5=90=A6=E5=AD=98=E5=9C=A8)=202=20before=20delete=20c?= =?UTF-8?q?heck=20entity=20exist?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../org/apache/dolphinscheduler/api/service/UdfFuncService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java index 7b4ce63e24..df439ce38f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java @@ -306,7 +306,7 @@ public class UdfFuncService extends BaseService{ //check exist UdfFunc udfFunc = udfFuncMapper.selectById(id); if (udfFunc == null) { - putMsg(result, Status.RESOURCE_NOT_EXIST); + putMsg(result, Status.UDF_FUNCTION_NOT_EXIST); return result; } udfFuncMapper.deleteById(id); From 04df353b4c7074fd1ba797fa2d1df207aea3c823 Mon Sep 17 00:00:00 2001 From: Technoboy- Date: Sat, 28 Dec 2019 19:24:38 +0800 Subject: [PATCH 40/53] refactor getKillProcessThread method --- .../server/worker/WorkerServer.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index 877d60a33b..fe9c4991ac 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -290,22 +290,20 @@ public class WorkerServer implements IStoppable { Runnable killProcessThread = new Runnable() { @Override public void run() { - Set taskInfoSet = taskQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_KILL); while (Stopper.isRunning()){ - try { - Thread.sleep(Constants.SLEEP_TIME_MILLIS); - } catch (InterruptedException e) { - logger.error("interrupted exception",e); - } - // if set is null , return + Set taskInfoSet = taskQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_KILL); if (CollectionUtils.isNotEmpty(taskInfoSet)){ for (String taskInfo : taskInfoSet){ killTask(taskInfo, processDao); removeKillInfoFromQueue(taskInfo); } } - - taskInfoSet = taskQueue.smembers(Constants.DOLPHINSCHEDULER_TASKS_KILL); + try { + Thread.sleep(Constants.SLEEP_TIME_MILLIS); + } catch (InterruptedException e) { + logger.error("interrupted exception",e); + Thread.currentThread().interrupt(); + } } } }; From e9fcf6128e051a642808c7815d1ebc30aa32aad6 Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Sun, 29 Dec 2019 09:59:49 +0800 Subject: [PATCH 41/53] AlertMappert UT modify #1465 (#1616) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * remove LogViewServiceGrpc.java file and pom modify * remove kazoo * remove kazoo * remove kazoo * remove common monitor package * add license * remove kazoo modify * remove kazoo modify * remove kazoo modify * remove kazoo modify * remove kazoo modify * remove kazoo modify * install.sh remove python kazoo * add system param whether repeat running * remove kazoo modify * BusinessTimeUtils remove whther repeat running inner param * add AccessTokenMapperTest UT * CI UT yml modify，start postgresql and zookeeper by default * add AlertGroupMapperTest UT in github action * Conflicts reslove * AlertMappert UT modify * AlertMappert UT modify * AlertMappert UT modify --- .../dolphinscheduler/dao/entity/Alert.java | 71 +++++++- .../dao/mapper/AlertMapperTest.java | 153 ++++++++++++------ pom.xml | 1 + 3 files changed, 177 insertions(+), 48 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Alert.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Alert.java index d989f82fa2..dac96e8695 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Alert.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Alert.java @@ -32,7 +32,6 @@ import java.util.Map; /** * alert */ -@Data @TableName("t_ds_alert") public class Alert { @@ -217,6 +216,72 @@ public class Alert { this.updateTime = updateTime; } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + Alert alert = (Alert) o; + + if (id != alert.id) { + return false; + } + if (alertGroupId != alert.alertGroupId) { + return false; + } + if (!title.equals(alert.title)) { + return false; + } + if (showType != alert.showType) { + return false; + } + if (!content.equals(alert.content)) { + return false; + } + if (alertType != alert.alertType) { + return false; + } + if (alertStatus != alert.alertStatus) { + return false; + } + if (!log.equals(alert.log)) { + return false; + } + if (!receivers.equals(alert.receivers)) { + return false; + } + if (!receiversCc.equals(alert.receiversCc)) { + return false; + } + if (!createTime.equals(alert.createTime)) { + return false; + } + return updateTime.equals(alert.updateTime) && info.equals(alert.info); + + } + + @Override + public int hashCode() { + int result = id; + result = 31 * result + title.hashCode(); + result = 31 * result + showType.hashCode(); + result = 31 * result + content.hashCode(); + result = 31 * result + alertType.hashCode(); + result = 31 * result + alertStatus.hashCode(); + result = 31 * result + log.hashCode(); + result = 31 * result + alertGroupId; + result = 31 * result + receivers.hashCode(); + result = 31 * result + receiversCc.hashCode(); + result = 31 * result + createTime.hashCode(); + result = 31 * result + updateTime.hashCode(); + result = 31 * result + info.hashCode(); + return result; + } + @Override public String toString() { return "Alert{" + @@ -228,10 +293,10 @@ public class Alert { ", alertStatus=" + alertStatus + ", log='" + log + '\'' + ", alertGroupId=" + alertGroupId + - ", createTime=" + createTime + - ", updateTime=" + updateTime + ", receivers='" + receivers + '\'' + ", receiversCc='" + receiversCc + '\'' + + ", createTime=" + createTime + + ", updateTime=" + updateTime + ", info=" + info + '}'; } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java index 958cd39c18..5f10a7586d 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java @@ -19,44 +19,52 @@ package org.apache.dolphinscheduler.dao.mapper; import org.apache.dolphinscheduler.common.enums.AlertStatus; import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.common.enums.ShowType; +import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.Alert; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.Rollback; import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.transaction.annotation.Transactional; -import java.util.Date; -import java.util.List; - +import java.util.*; +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.*; @RunWith(SpringRunner.class) @SpringBootTest +@Transactional +@Rollback(true) public class AlertMapperTest { @Autowired - AlertMapper alertMapper; + private AlertMapper alertMapper; /** - * insert - * @return Alert + * test insert + * @return */ - private Alert insertOne(){ - //insertOne - Alert alert = new Alert(); - alert.setContent("[{'type':'WORKER','host':'192.168.xx.xx','event':'server down','warning level':'serious'}]"); - alert.setLog("success"); - alert.setReceivers("xx@aa.com"); - alert.setAlertType(AlertType.EMAIL); - alert.setShowType(ShowType.TABLE); - alert.setAlertGroupId(1); - alert.setAlertStatus(AlertStatus.EXECUTION_SUCCESS); - alert.setCreateTime(new Date()); - alert.setUpdateTime(new Date()); - alertMapper.insert(alert); - return alert; + @Test + public void testInsert(){ + Alert expectedAlert = createAlert(); + assertNotNull(expectedAlert.getId()); + assertThat(expectedAlert.getId(), greaterThan(0)); + } + + + /** + * test select by id + * @return + */ + @Test + public void testSelectById(){ + Alert expectedAlert = createAlert(); + Alert actualAlert = alertMapper.selectById(expectedAlert.getId()); + assertEquals(expectedAlert, actualAlert); } /** @@ -64,13 +72,18 @@ public class AlertMapperTest { */ @Test public void testUpdate(){ - //insertOne - Alert alert = insertOne(); - //update - alert.setTitle("hello"); - int update = alertMapper.updateById(alert); - Assert.assertEquals(update, 1); - alertMapper.deleteById(alert.getId()); + + Alert expectedAlert = createAlert(); + + expectedAlert.setAlertStatus(AlertStatus.EXECUTION_FAILURE); + expectedAlert.setLog("error"); + expectedAlert.setUpdateTime(DateUtils.getCurrentDate()); + + alertMapper.updateById(expectedAlert); + + Alert actualAlert = alertMapper.selectById(expectedAlert.getId()); + + assertEquals(expectedAlert, actualAlert); } /** @@ -78,33 +91,83 @@ public class AlertMapperTest { */ @Test public void testDelete(){ + Alert expectedAlert = createAlert(); - Alert alert = insertOne(); - int delete = alertMapper.deleteById(alert.getId()); - Assert.assertEquals(delete, 1); + alertMapper.deleteById(expectedAlert.getId()); + + Alert actualAlert = alertMapper.selectById(expectedAlert.getId()); + + assertNull(actualAlert); } + /** - * test query + * test list alert by status */ @Test - public void testQuery() { - Alert alert = insertOne(); - //query - List alerts = alertMapper.selectList(null); - Assert.assertNotEquals(alerts.size(), 0); - alertMapper.deleteById(alert.getId()); + public void testListAlertByStatus() { + Integer count = 10; + AlertStatus waitExecution = AlertStatus.WAIT_EXECUTION; + + Map expectedAlertMap = createAlertMap(count, waitExecution); + + List actualAlerts = alertMapper.listAlertByStatus(waitExecution); + + for (Alert actualAlert : actualAlerts){ + Alert expectedAlert = expectedAlertMap.get(actualAlert.getId()); + if (expectedAlert != null){ + assertEquals(expectedAlert,actualAlert); + } + } } /** - * test list alert by status + * create alert map + * @param count alert count + * @param alertStatus alert status + * @return alert map */ - @Test - public void testListAlertByStatus() { - Alert alert = insertOne(); - //query - List alerts = alertMapper.listAlertByStatus(AlertStatus.EXECUTION_SUCCESS); - Assert.assertNotEquals(alerts.size(), 0); - alertMapper.deleteById(alert.getId()); + private Map createAlertMap(Integer count,AlertStatus alertStatus){ + Map alertMap = new HashMap<>(); + + for (int i = 0 ; i < count ;i++){ + Alert alert = createAlert(alertStatus); + alertMap.put(alert.getId(),alert); + } + + return alertMap; + + } + + + /** + * create alert + * @return alert + * @throws Exception + */ + private Alert createAlert(){ + return createAlert(AlertStatus.WAIT_EXECUTION); + } + + /** + * create alert + * @param alertStatus alert status + * @return alert + */ + private Alert createAlert(AlertStatus alertStatus){ + Alert alert = new Alert(); + alert.setShowType(ShowType.TABLE); + alert.setTitle("test alert"); + alert.setContent("[{'type':'WORKER','host':'192.168.xx.xx','event':'server down','warning level':'serious'}]"); + alert.setAlertType(AlertType.EMAIL); + alert.setAlertStatus(alertStatus); + alert.setLog("success"); + alert.setReceivers("aa@aa.com"); + alert.setReceiversCc("bb@aa.com"); + alert.setCreateTime(DateUtils.getCurrentDate()); + alert.setUpdateTime(DateUtils.getCurrentDate()); + + alertMapper.insert(alert); + return alert; } } \ No newline at end of file diff --git a/pom.xml b/pom.xml index 69f1ff0665..796d2b8228 100644 --- a/pom.xml +++ b/pom.xml @@ -674,6 +674,7 @@ **/server/utils/FlinkArgsUtilsTest.java **/dao/mapper/AccessTokenMapperTest.java **/dao/mapper/AlertGroupMapperTest.java + **/dao/mapper/AlertMapperTest.java From 2a0dab14ee56015f692208a4ef92900d6deec605 Mon Sep 17 00:00:00 2001 From: Tboy Date: Sun, 29 Dec 2019 10:11:41 +0800 Subject: [PATCH 42/53] we should insert alert DB once , and trigger this type of alert 3 times (#1622) --- .../apache/dolphinscheduler/server/master/MasterServer.java | 6 ++---- .../apache/dolphinscheduler/server/worker/WorkerServer.java | 4 +--- .../apache/dolphinscheduler/server/zk/ZKMasterClient.java | 6 ++---- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index e8c8b6779e..1f5b16a25a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -152,10 +152,8 @@ public class MasterServer implements IStoppable { @Override public void run() { if (zkMasterClient.getActiveMasterNum() <= 1) { - for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) { - zkMasterClient.getAlertDao().sendServerStopedAlert( - 1, OSUtils.getHost(), "Master-Server"); - } + zkMasterClient.getAlertDao().sendServerStopedAlert( + 1, OSUtils.getHost(), "Master-Server"); } stop("shutdownhook"); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index fe9c4991ac..ea8330e18a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -177,9 +177,7 @@ public class WorkerServer implements IStoppable { public void run() { // worker server exit alert if (zkWorkerClient.getActiveMasterNum() <= 1) { - for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) { - alertDao.sendServerStopedAlert(1, OSUtils.getHost(), "Worker-Server"); - } + alertDao.sendServerStopedAlert(1, OSUtils.getHost(), "Worker-Server"); } stop("shutdownhook"); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java index a26a217665..1c7b737f5b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java @@ -247,10 +247,8 @@ public class ZKMasterClient extends AbstractZKClient { */ private void alertServerDown(String serverHost, ZKNodeType zkNodeType) { - String serverType = zkNodeType.toString(); - for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) { - alertDao.sendServerStopedAlert(1, serverHost, serverType); - } + String serverType = zkNodeType.toString(); + alertDao.sendServerStopedAlert(1, serverHost, serverType); } /** From b03632bc6de211935a3531cd0ee3a08b14802666 Mon Sep 17 00:00:00 2001 From: samz406 Date: Sun, 29 Dec 2019 10:13:14 +0800 Subject: [PATCH 43/53] User update not check params (#1619) * misspell words * modify common queue TaskQueueZKImplTest.java unit test * extends BaseTaskQueueTest get zkServer * modify zk config * User update not check params * user phone update when noteEmpty --- .../api/service/UsersService.java | 49 +++++++++++++++++-- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java index 92319e71a1..a4664b6c3e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java @@ -221,9 +221,13 @@ public class UsersService extends BaseService { return result; } - Date now = new Date(); - if (StringUtils.isNotEmpty(userName)) { + + if (!CheckUtils.checkUserName(userName)){ + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,userName); + return result; + } + User tempUser = userMapper.queryByUserNameAccurately(userName); if (tempUser != null && tempUser.getId() != userId) { putMsg(result, Status.USER_NAME_EXIST); @@ -233,14 +237,30 @@ public class UsersService extends BaseService { } if (StringUtils.isNotEmpty(userPassword)) { + if (!CheckUtils.checkPassword(userPassword)){ + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,userPassword); + return result; + } user.setUserPassword(EncryptionUtils.getMd5(userPassword)); } if (StringUtils.isNotEmpty(email)) { + if (!CheckUtils.checkEmail(email)){ + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,email); + return result; + } user.setEmail(email); } + + if (StringUtils.isNotEmpty(phone)) { + if (!CheckUtils.checkPhone(phone)){ + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR,phone); + return result; + } + user.setPhone(phone); + } user.setQueue(queue); - user.setPhone(phone); + Date now = new Date(); user.setUpdateTime(now); //if user switches the tenant, the user's resources need to be copied to the new tenant @@ -318,10 +338,15 @@ public class UsersService extends BaseService { Map result = new HashMap<>(5); //only admin can operate if (!isAdmin(loginUser)) { + putMsg(result, Status.USER_NO_OPERATION_PERM, id); + return result; + } + //check exist + User tempUser = userMapper.selectById(id); + if (tempUser == null) { putMsg(result, Status.USER_NOT_EXIST, id); return result; } - // delete user User user = userMapper.queryTenantCodeByUserId(id); @@ -357,6 +382,12 @@ public class UsersService extends BaseService { return result; } + //check exist + User tempUser = userMapper.selectById(userId); + if (tempUser == null) { + putMsg(result, Status.USER_NOT_EXIST, userId); + return result; + } //if the selected projectIds are empty, delete all items associated with the user projectUserMapper.deleteProjectRelation(0, userId); @@ -443,6 +474,11 @@ public class UsersService extends BaseService { if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; } + User user = userMapper.selectById(userId); + if(user == null){ + putMsg(result, Status.USER_NOT_EXIST, userId); + return result; + } udfUserMapper.deleteByUserId(userId); @@ -485,6 +521,11 @@ public class UsersService extends BaseService { if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; } + User user = userMapper.selectById(userId); + if(user == null){ + putMsg(result, Status.USER_NOT_EXIST, userId); + return result; + } datasourceUserMapper.deleteByUserId(userId); From 779decc96dd2e832f9cd14b8a0f44b483080683a Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Sun, 29 Dec 2019 10:23:10 +0800 Subject: [PATCH 44/53] =?UTF-8?q?Increase=20script=20text=20box=20to=20zoo?= =?UTF-8?q?m=20in=20and=20fix=20log=20loading=20#1547=20=E3=80=81#1247=20(?= =?UTF-8?q?#1604)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Increase script text box to zoom in and fix log loading * Timing operation adds default value for start and end dates --- .../home/pages/dag/_source/formModel/log.vue | 8 +- .../formModel/tasks/_source/scriptBox.vue | 124 ++++++++++++++++++ .../dag/_source/formModel/tasks/shell.vue | 64 ++++++++- .../definition/pages/list/_source/timing.vue | 16 +++ 4 files changed, 202 insertions(+), 10 deletions(-) create mode 100644 dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/scriptBox.vue diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/log.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/log.vue index 4c1acff5e6..7ec9cdfa7a 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/log.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/log.vue @@ -257,8 +257,8 @@ _rtParam () { return { taskInstId: this.stateId || this.logId, - skipLineNum: parseInt(`${this.loadingIndex ? this.loadingIndex + '0000' : 0}`), - limit: parseInt(`${this.loadingIndex ? this.loadingIndex + 1 : 1}0000`) + skipLineNum: parseInt(`${this.loadingIndex ? this.loadingIndex + '000' : 0}`), + limit: parseInt(`${this.loadingIndex ? this.loadingIndex + 1 : 1}000`) } } }, @@ -316,7 +316,7 @@ } .refresh-log { >i { - font-size: 24px; + font-size: 20px; vertical-align: middle; transform: scale(1); } @@ -353,7 +353,7 @@ font-weight: bold; resize:none; line-height: 1.6; - padding: 6px; + padding: 0px; } } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/scriptBox.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/scriptBox.vue new file mode 100644 index 0000000000..c16d18cc38 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/scriptBox.vue @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue index 230b43d446..8b45e7c3d1 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue @@ -21,10 +21,13 @@
+ + +
@@ -55,6 +58,7 @@ import _ from 'lodash' import i18n from '@/module/i18n' import mListBox from './_source/listBox' + import mScriptBox from './_source/scriptBox' import mResources from './_source/resources' import mLocalParams from './_source/localParams' import disabledState from '@/module/mixin/disabledState' @@ -85,8 +89,37 @@ _onLocalParams (a) { this.localParams = a }, + setEditorVal() { + let self = this + let modal = self.$modal.dialog({ + className: 'scriptModal', + closable: false, + showMask: true, + maskClosable: true, + onClose: function() { + + }, + render (h) { + return h(mScriptBox, { + on: { + getSriptBoxValue (val) { + editor.setValue(val) + }, + closeAble () { + // this.$modal.destroy() + modal.remove() + } + }, + props: { + item: editor.getValue() + } + }) + } + }) + }, /** * return resourceList + * */ _onResourcesData (a) { this.resourceList = a @@ -109,7 +142,6 @@ if (!this.$refs.refLocalParams._verifProp()) { return false } - // storage this.$emit('on-params', { resourceList: this.resourceList, @@ -138,7 +170,6 @@ // Monitor keyboard editor.on('keypress', this.keypress) - editor.setValue(this.rawScript) return editor @@ -176,6 +207,27 @@ editor.off($('.code-shell-mirror'), 'keypress', this.keypress) } }, - components: { mLocalParams, mListBox, mResources } + components: { mLocalParams, mListBox, mResources, mScriptBox } } + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue index 6ed78980f1..a15999edbd 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue @@ -313,7 +313,23 @@ this.crontab = this.item.crontab } if(this.type == 'timing') { + let date = new Date() + let year = date.getFullYear() + let month = date.getMonth() + 1 + let day = date.getDate() + if (month < 10) { + month = "0" + month; + } + if (day < 10) { + day = "0" + day; + } + let startDate = year + "-" + month + "-" + day + ' ' + '00:00:00' + let endDate = (year+100) + "-" + month + "-" + day + ' ' + '00:00:00' + let times = [] + times[0] = startDate + times[1] = endDate this.crontab = '0 0 * * * ? *' + this.scheduleTime = times } this.receivers = _.cloneDeep(this.receiversD) this.receiversCc = _.cloneDeep(this.receiversCcD) From e61586cd0665f94618d0d0489052dc87a1aea61c Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Sun, 29 Dec 2019 13:12:29 +0800 Subject: [PATCH 45/53] CommandMapperTest UT modify #1465 (#1625) * CommandMapperTest UT modify --- .../dolphinscheduler/dao/entity/Command.java | 74 ++++- .../dao/entity/CommandCount.java | 40 ++- .../dao/mapper/AlertMapperTest.java | 3 + .../dao/mapper/CommandMapperTest.java | 264 ++++++++++++------ pom.xml | 1 + 5 files changed, 292 insertions(+), 90 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java index a180a433bc..2d47b7874a 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java @@ -28,7 +28,6 @@ import java.util.Date; /** * command */ -@Data @TableName("t_ds_command") public class Command { @@ -265,6 +264,79 @@ public class Command { this.workerGroupId = workerGroupId; } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + Command command = (Command) o; + + if (id != command.id) { + return false; + } + if (processDefinitionId != command.processDefinitionId) { + return false; + } + if (executorId != command.executorId) { + return false; + } + if (workerGroupId != command.workerGroupId) { + return false; + } + if (commandType != command.commandType) { + return false; + } + if (commandParam != null ? !commandParam.equals(command.commandParam) : command.commandParam != null) { + return false; + } + if (taskDependType != command.taskDependType) { + return false; + } + if (failureStrategy != command.failureStrategy) { + return false; + } + if (warningType != command.warningType) { + return false; + } + if (warningGroupId != null ? !warningGroupId.equals(command.warningGroupId) : command.warningGroupId != null) { + return false; + } + if (scheduleTime != null ? !scheduleTime.equals(command.scheduleTime) : command.scheduleTime != null) { + return false; + } + if (startTime != null ? !startTime.equals(command.startTime) : command.startTime != null) { + return false; + } + if (processInstancePriority != command.processInstancePriority) { + return false; + } + return !(updateTime != null ? !updateTime.equals(command.updateTime) : command.updateTime != null); + + } + + @Override + public int hashCode() { + int result = id; + result = 31 * result + (commandType != null ? commandType.hashCode() : 0); + result = 31 * result + processDefinitionId; + result = 31 * result + executorId; + result = 31 * result + (commandParam != null ? commandParam.hashCode() : 0); + result = 31 * result + (taskDependType != null ? taskDependType.hashCode() : 0); + result = 31 * result + (failureStrategy != null ? failureStrategy.hashCode() : 0); + result = 31 * result + (warningType != null ? warningType.hashCode() : 0); + result = 31 * result + (warningGroupId != null ? warningGroupId.hashCode() : 0); + result = 31 * result + (scheduleTime != null ? scheduleTime.hashCode() : 0); + result = 31 * result + (startTime != null ? startTime.hashCode() : 0); + result = 31 * result + (processInstancePriority != null ? processInstancePriority.hashCode() : 0); + result = 31 * result + (updateTime != null ? updateTime.hashCode() : 0); + result = 31 * result + workerGroupId; + return result; + } + @Override public String toString() { return "Command{" + diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CommandCount.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CommandCount.java index 158169b727..df99b54dbe 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CommandCount.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CommandCount.java @@ -33,13 +33,6 @@ public class CommandCount { private int count; - @Override - public String toString(){ - return "command count:" + - " commandType: "+ commandType.toString() + - " count: "+ count; - } - public CommandType getCommandType() { return commandType; } @@ -55,4 +48,37 @@ public class CommandCount { public void setCount(int count) { this.count = count; } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + CommandCount that = (CommandCount) o; + + if (count != that.count) { + return false; + } + return commandType == that.commandType; + + } + + @Override + public int hashCode() { + int result = commandType != null ? commandType.hashCode() : 0; + result = 31 * result + count; + return result; + } + + @Override + public String toString() { + return "CommandCount{" + + "commandType=" + commandType + + ", count=" + count + + '}'; + } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java index 5f10a7586d..4e298e3aa3 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java @@ -35,6 +35,9 @@ import java.util.*; import static org.hamcrest.Matchers.*; import static org.junit.Assert.*; +/** + * alert mapper test + */ @RunWith(SpringRunner.class) @SpringBootTest @Transactional diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java index 2af44576b5..c35ce7e8ce 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.dao.mapper; +import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.Command; import org.apache.dolphinscheduler.dao.entity.CommandCount; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; @@ -25,13 +26,25 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.Rollback; import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.transaction.annotation.Transactional; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.*; + +/** + * command mapper test + */ @RunWith(SpringRunner.class) @SpringBootTest +@Transactional +@Rollback(true) public class CommandMapperTest { @@ -41,71 +54,85 @@ public class CommandMapperTest { @Autowired ProcessDefinitionMapper processDefinitionMapper; + /** - * insert - * @return Command + * test insert */ - private Command insertOne(){ - //insertOne - Command command = new Command(); - command.setCommandType(CommandType.START_PROCESS); - command.setProcessDefinitionId(1); - command.setExecutorId(4); - command.setProcessInstancePriority(Priority.MEDIUM); - command.setFailureStrategy(FailureStrategy.CONTINUE); - command.setWorkerGroupId(-1); - command.setWarningGroupId(1); - command.setUpdateTime(new Date()); - commandMapper.insert(command); - return command; + @Test + public void testInsert(){ + Command command = createCommand(); + assertNotNull(command.getId()); + assertThat(command.getId(),greaterThan(0)); } + /** - * test update + * test select by id */ @Test - public void testUpdate(){ - //insertOne - Command command = insertOne(); - //update - command.setStartTime(new Date()); - int update = commandMapper.updateById(command); - Assert.assertEquals(update, 1); - commandMapper.deleteById(command.getId()); + public void testSelectById() { + Command expectedCommand = createCommand(); + //query + Command actualCommand = commandMapper.selectById(expectedCommand.getId()); + + assertEquals(expectedCommand, actualCommand); } /** - * test delete + * test update */ @Test - public void testDelete(){ + public void testUpdate(){ + + Command expectedCommand = createCommand(); + + // update the command time if current command if recover from waiting + expectedCommand.setUpdateTime(DateUtils.getCurrentDate()); + + commandMapper.updateById(expectedCommand); + + Command actualCommand = commandMapper.selectById(expectedCommand.getId()); + + assertEquals(expectedCommand,actualCommand); - Command Command = insertOne(); - int delete = commandMapper.deleteById(Command.getId()); - Assert.assertEquals(delete, 1); } /** - * test query + * test delete */ @Test - public void testQuery() { - Command command = insertOne(); - //query - List commands = commandMapper.selectList(null); - Assert.assertNotEquals(commands.size(), 0); - commandMapper.deleteById(command.getId()); + public void testDelete(){ + Command expectedCommand = createCommand(); + + commandMapper.deleteById(expectedCommand.getId()); + + Command actualCommand = commandMapper.selectById(expectedCommand.getId()); + + assertNull(actualCommand); } + + /** * test query all */ @Test public void testGetAll() { - Command command = insertOne(); - List commands = commandMapper.selectList(null); - Assert.assertNotEquals(commands.size(), 0); - commandMapper.deleteById(command.getId()); + Integer count = 10; + + Map commandMap = createCommandMap(count); + + + List actualCommands = commandMapper.selectList(null); + + assertThat(actualCommands.size(), greaterThanOrEqualTo(count)); + + for (Command actualCommand : actualCommands){ + Command expectedCommand = commandMap.get(actualCommand.getId()); + if (expectedCommand != null){ + assertEquals(expectedCommand,actualCommand); + } + } } /** @@ -113,28 +140,14 @@ public class CommandMapperTest { */ @Test public void testGetOneToRun() { - ProcessDefinition processDefinition = new ProcessDefinition(); - processDefinition.setReleaseState(ReleaseState.ONLINE); - processDefinition.setName("ut test"); - processDefinition.setProjectId(1); - processDefinition.setFlag(Flag.YES); - processDefinitionMapper.insert(processDefinition); - Command command = new Command(); - command.setCommandType(CommandType.START_PROCESS); - command.setProcessDefinitionId(processDefinition.getId()); - command.setExecutorId(4); - command.setProcessInstancePriority(Priority.MEDIUM); - command.setFailureStrategy(FailureStrategy.CONTINUE); - command.setWorkerGroupId(-1); - command.setWarningGroupId(1); - command.setUpdateTime(new Date()); - commandMapper.insert(command); + ProcessDefinition processDefinition = createProcessDefinition(); + + Command expectedCommand = createCommand(CommandType.START_PROCESS,processDefinition.getId()); - Command command2 = commandMapper.getOneToRun(); - Assert.assertNotEquals(command2, null); - commandMapper.deleteById(command.getId()); - processDefinitionMapper.deleteById(processDefinition.getId()); + Command actualCommand = commandMapper.getOneToRun(); + + assertEquals(expectedCommand, actualCommand); } /** @@ -142,35 +155,122 @@ public class CommandMapperTest { */ @Test public void testCountCommandState() { - Command command = insertOne(); + Integer count = 10; + + ProcessDefinition processDefinition = createProcessDefinition(); + + CommandCount expectedCommandCount = createCommandMap(count, CommandType.START_PROCESS, processDefinition.getId()); + + Integer[] projectIdArray = {processDefinition.getProjectId()}; + + Date startTime = DateUtils.stringToDate("2019-12-29 00:10:00"); + + Date endTime = DateUtils.stringToDate("2019-12-29 23:59:59"); + + List actualCommandCounts = commandMapper.countCommandState(0, startTime, endTime, projectIdArray); + + assertThat(actualCommandCounts.size(),greaterThanOrEqualTo(1)); + + Boolean flag = false; + for (CommandCount actualCommandCount : actualCommandCounts){ + if (actualCommandCount.getCommandType().equals(expectedCommandCount.getCommandType())){ + assertEquals(expectedCommandCount,actualCommandCount); + flag = true; + } + } + + assertTrue(flag); + } - //insertOne + + /** + * create command map + * @param count map count + * @param commandType comman type + * @param processDefinitionId process definition id + * @return command map + */ + private CommandCount createCommandMap( + Integer count, + CommandType commandType, + Integer processDefinitionId){ + + CommandCount commandCount = new CommandCount(); + + for (int i = 0 ;i < count ;i++){ + createCommand(commandType,processDefinitionId); + } + commandCount.setCommandType(commandType); + commandCount.setCount(count); + + return commandCount; + } + + /** + * create process definition + * @return process definition + */ + private ProcessDefinition createProcessDefinition(){ ProcessDefinition processDefinition = new ProcessDefinition(); - processDefinition.setName("def 1"); - processDefinition.setProjectId(1010); - processDefinition.setUserId(101); - processDefinition.setUpdateTime(new Date()); - processDefinition.setCreateTime(new Date()); + processDefinition.setReleaseState(ReleaseState.ONLINE); + processDefinition.setName("ut test"); + processDefinition.setProjectId(1); + processDefinition.setFlag(Flag.YES); + processDefinitionMapper.insert(processDefinition); - command.setProcessDefinitionId(processDefinition.getId()); - commandMapper.updateById(command); + return processDefinition; + } + /** + * create command map + * @param count map count + * @return command map + */ + private Map createCommandMap(Integer count){ + Map commandMap = new HashMap<>(); + + for (int i = 0; i < count ;i++){ + Command command = createCommand(); + commandMap.put(command.getId(),command); + } + return commandMap; + } - List commandCounts = commandMapper.countCommandState( - 4, null, null, new Integer[0] - ); - Integer[] projectIdArray = new Integer[2]; - projectIdArray[0] = processDefinition.getProjectId(); - projectIdArray[1] = 200; - List commandCounts2 = commandMapper.countCommandState( - 4, null, null, projectIdArray - ); + /** + * create command + * @return + */ + private Command createCommand(){ + return createCommand(CommandType.START_PROCESS,1); + } + + /** + * create command + * @return Command + */ + private Command createCommand(CommandType commandType,Integer processDefinitionId){ + + Command command = new Command(); + command.setCommandType(commandType); + command.setProcessDefinitionId(processDefinitionId); + command.setExecutorId(4); + command.setCommandParam("test command param"); + command.setTaskDependType(TaskDependType.TASK_ONLY); + command.setFailureStrategy(FailureStrategy.CONTINUE); + command.setWarningType(WarningType.ALL); + command.setWarningGroupId(1); + command.setScheduleTime(DateUtils.stringToDate("2019-12-29 12:10:00")); + command.setProcessInstancePriority(Priority.MEDIUM); + command.setStartTime(DateUtils.stringToDate("2019-12-29 10:10:00")); + command.setUpdateTime(DateUtils.stringToDate("2019-12-29 10:10:00")); + command.setWorkerGroupId(-1); + commandMapper.insert(command); - commandMapper.deleteById(command.getId()); - processDefinitionMapper.deleteById(processDefinition.getId()); - Assert.assertNotEquals(commandCounts.size(), 0); - Assert.assertNotEquals(commandCounts2.size(), 0); + return command; } + + + } \ No newline at end of file diff --git a/pom.xml b/pom.xml index 796d2b8228..7936bf7dc8 100644 --- a/pom.xml +++ b/pom.xml @@ -675,6 +675,7 @@ **/dao/mapper/AccessTokenMapperTest.java **/dao/mapper/AlertGroupMapperTest.java **/dao/mapper/AlertMapperTest.java + **/dao/mapper/CommandMapperTest.java From d3d5726aebdeab98e10abbff725ca7be9be1a911 Mon Sep 17 00:00:00 2001 From: Jave-Chen Date: Sun, 29 Dec 2019 13:15:29 +0800 Subject: [PATCH 46/53] Add github action to analyze code with sonarcloud (#1624) * Create .sonarcloud.properties * add License description --- .github/workflows/SonarCloud.yml | 51 ++++++++++++++++++++++++++++++++ README.md | 3 +- README_zh_CN.md | 2 +- 3 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/SonarCloud.yml diff --git a/.github/workflows/SonarCloud.yml b/.github/workflows/SonarCloud.yml new file mode 100644 index 0000000000..fd5b0585db --- /dev/null +++ b/.github/workflows/SonarCloud.yml @@ -0,0 +1,51 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: SonarCloud +on: [push, pull_request] +jobs: + sonarCloudTrigger: + name: SonarCloud Trigger + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - uses: actions/cache@v1 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-sonarqube + restore-keys: | + ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-sonarqube + ${{ runner.os }}-maven- + - name: Maven clean + run: mvn clean + - name: Run SonarCloud analyse + run: > + mvn clean --batch-mode + org.jacoco:jacoco-maven-plugin:prepare-agent + verify + org.sonarsource.scanner.maven:sonar-maven-plugin:sonar + -Dmaven.test.skip=true + -Dsonar.host.url=https://sonarcloud.io + -Dsonar.organization=apache + -Dsonar.projectKey=apache-dolphinscheduler + -Dsonar.login=e4058004bc6be89decf558ac819aa1ecbee57682 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/README.md b/README.md index aee0e3b2e3..bc7dba684f 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ Dolphin Scheduler Official Website [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) [![Total Lines](https://tokei.rs/b1/github/apache/Incubator-DolphinScheduler?category=lines)](https://github.com/apache/Incubator-DolphinScheduler) [![codecov](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev) -![Codacy Badge](https://api.codacy.com/project/badge/Grade/ae514bfb7bb049658e916da091928e62) +[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache-dolphinscheduler&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler) + > Dolphin Scheduler for Big Data diff --git a/README_zh_CN.md b/README_zh_CN.md index f8855aca6c..8141763c33 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -4,7 +4,7 @@ Dolphin Scheduler Official Website [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) [![Total Lines](https://tokei.rs/b1/github/apache/Incubator-DolphinScheduler?category=lines)](https://github.com/apache/Incubator-DolphinScheduler) [![codecov](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev) -![Codacy Badge](https://api.codacy.com/project/badge/Grade/ae514bfb7bb049658e916da091928e62) +[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache-dolphinscheduler&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler) > Dolphin Scheduler for Big Data From dad75727ff939b7f0d73acdafe0251921deac50e Mon Sep 17 00:00:00 2001 From: Jave-Chen Date: Mon, 30 Dec 2019 10:28:47 +0800 Subject: [PATCH 47/53] fix bug (#1633) dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&issues=AW9PvkP_YzPBRjkobY7y&open=AW9PvkP_YzPBRjkobY7y https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&issues=AW9PvkP_YzPBRjkobY7z&open=AW9PvkP_YzPBRjkobY7z dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&issues=AW9PvkRwYzPBRjkobY87&open=AW9PvkRwYzPBRjkobY87 --- .../dolphinscheduler/alert/AlertServer.java | 11 ++-- .../alert/utils/EnterpriseWeChatUtils.java | 53 +++++++++++-------- 2 files changed, 35 insertions(+), 29 deletions(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java index ee38f3d3c9..a41498e258 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java @@ -39,19 +39,15 @@ public class AlertServer { private AlertSender alertSender; - private static volatile AlertServer instance; + private static AlertServer instance; public AlertServer() { } - public static AlertServer getInstance(){ + public synchronized static AlertServer getInstance(){ if (null == instance) { - synchronized (AlertServer.class) { - if(null == instance) { - instance = new AlertServer(); - } - } + instance = new AlertServer(); } return instance; } @@ -63,6 +59,7 @@ public class AlertServer { Thread.sleep(Constants.ALERT_SCAN_INTERVEL); } catch (InterruptedException e) { logger.error(e.getMessage(),e); + Thread.currentThread().interrupt(); } List alerts = alertDao.listWaitExecutionAlert(); alertSender = new AlertSender(alerts, alertDao); diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java index a11d37f3d3..15e7e18855 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java @@ -86,20 +86,24 @@ public class EnterpriseWeChatUtils { String resp; CloseableHttpClient httpClient = HttpClients.createDefault(); - HttpGet httpGet = new HttpGet(enterpriseWeChatTokenUrlReplace); - CloseableHttpResponse response = httpClient.execute(httpGet); try { - HttpEntity entity = response.getEntity(); - resp = EntityUtils.toString(entity, Constants.UTF_8); - EntityUtils.consume(entity); + HttpGet httpGet = new HttpGet(enterpriseWeChatTokenUrlReplace); + CloseableHttpResponse response = httpClient.execute(httpGet); + try { + HttpEntity entity = response.getEntity(); + resp = EntityUtils.toString(entity, Constants.UTF_8); + EntityUtils.consume(entity); + } finally { + response.close(); + } + + Map map = JSON.parseObject(resp, + new TypeToken>() { + }.getType()); + return map.get("access_token").toString(); } finally { - response.close(); + httpClient.close(); } - - Map map = JSON.parseObject(resp, - new TypeToken>() { - }.getType()); - return map.get("access_token").toString(); } /** @@ -167,20 +171,25 @@ public class EnterpriseWeChatUtils { public static String sendEnterpriseWeChat(String charset, String data, String token) throws IOException { String enterpriseWeChatPushUrlReplace = enterpriseWeChatPushUrl.replaceAll("\\$token", token); - CloseableHttpClient httpclient = HttpClients.createDefault(); - HttpPost httpPost = new HttpPost(enterpriseWeChatPushUrlReplace); - httpPost.setEntity(new StringEntity(data, charset)); - CloseableHttpResponse response = httpclient.execute(httpPost); - String resp; + CloseableHttpClient httpClient = HttpClients.createDefault(); try { - HttpEntity entity = response.getEntity(); - resp = EntityUtils.toString(entity, charset); - EntityUtils.consume(entity); + HttpPost httpPost = new HttpPost(enterpriseWeChatPushUrlReplace); + httpPost.setEntity(new StringEntity(data, charset)); + CloseableHttpResponse response = httpClient.execute(httpPost); + String resp; + try { + HttpEntity entity = response.getEntity(); + resp = EntityUtils.toString(entity, charset); + EntityUtils.consume(entity); + } finally { + response.close(); + } + logger.info("Enterprise WeChat send [{}], param:{}, resp:{}", + enterpriseWeChatPushUrl, data, resp); + return resp; } finally { - response.close(); + httpClient.close(); } - logger.info("Enterprise WeChat send [{}], param:{}, resp:{}", enterpriseWeChatPushUrl, data, resp); - return resp; } /** From 6d4a3410c86feed171063b760aa4b504662c75a2 Mon Sep 17 00:00:00 2001 From: samz406 Date: Mon, 30 Dec 2019 10:31:35 +0800 Subject: [PATCH 48/53] modify saveWorkerGroup may NPE (#1629) * User update not check params * user phone update when noteEmpty * modify saveWorkerGroup may NPE --- .../dolphinscheduler/api/service/WorkerGroupService.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java index 925d01bd09..2d0743a5ab 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java @@ -76,6 +76,11 @@ public class WorkerGroupService extends BaseService { WorkerGroup workerGroup = null; if(id != 0){ workerGroup = workerGroupMapper.selectById(id); + //check exist + if (workerGroup == null){ + workerGroup = new WorkerGroup(); + workerGroup.setCreateTime(now); + } }else{ workerGroup = new WorkerGroup(); workerGroup.setCreateTime(now); From 503be5f4007cc05bdcab268eb6f63c5cd72344fb Mon Sep 17 00:00:00 2001 From: samz406 Date: Mon, 30 Dec 2019 17:24:34 +0800 Subject: [PATCH 49/53] add service UT (#1637) * User update not check params * user phone update when noteEmpty * modify saveWorkerGroup may NPE * add some service UT * add service ut include --- .../api/service/AccessTokenServiceTest.java | 181 ++++++++ .../api/service/AlertGroupServiceTest.java | 219 +++++++++ .../api/service/MonitorServiceTest.java | 101 ++++ .../api/service/ProjectServiceTest.java | 356 ++++++++++++++ .../api/service/QueueServiceTest.java | 208 +++++++++ .../api/service/SessionServiceTest.java | 121 ++++- .../api/service/TenantServiceTest.java | 225 ++++++++- .../api/service/UsersServiceTest.java | 434 +++++++++++++++++- .../api/service/WorkerGroupServiceTest.java | 173 +++++++ pom.xml | 9 + 10 files changed, 1990 insertions(+), 37 deletions(-) create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/QueueServiceTest.java create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java new file mode 100644 index 0000000000..a9a5f67b0b --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.commons.lang3.time.DateUtils; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.dao.entity.AccessToken; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.AccessTokenMapper; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class AccessTokenServiceTest { + + + private static final Logger logger = LoggerFactory.getLogger(AccessTokenServiceTest.class); + + + @InjectMocks + private AccessTokenService accessTokenService ; + + @Mock + private AccessTokenMapper accessTokenMapper; + + @Before + public void setUp() { + + } + + + @After + public void after(){ + + } + + + + @Test + public void testQueryAccessTokenList(){ + + IPage tokenPage = new Page<>(); + tokenPage.setRecords(getList()); + tokenPage.setTotal(1L); + when(accessTokenMapper.selectAccessTokenPage(any(Page.class),eq("zhangsan"),eq(0))).thenReturn(tokenPage); + + User user =new User(); + Map result = accessTokenService.queryAccessTokenList(user,"zhangsan",1,10); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); + Assert.assertTrue(pageInfo.getTotalCount()>0); + } + + @Test + public void testCreateToken(){ + + + when(accessTokenMapper.insert(any(AccessToken.class))).thenReturn(2); + Map result = accessTokenService.createToken(1,getDate(),"AccessTokenServiceTest"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + + @Test + public void testGenerateToken(){ + + Map result = accessTokenService.generateToken(Integer.MAX_VALUE,getDate()); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + String token = (String) result.get(Constants.DATA_LIST); + Assert.assertNotNull(token); + } + + @Test + public void testDelAccessTokenById(){ + + when(accessTokenMapper.selectById(1)).thenReturn(getEntity()); + User userLogin = new User(); + // not exist + Map result = accessTokenService.delAccessTokenById(userLogin,0); + logger.info(result.toString()); + Assert.assertEquals(Status.ACCESS_TOKEN_NOT_EXIST,result.get(Constants.STATUS)); + // no operate + result = accessTokenService.delAccessTokenById(userLogin,1); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + //success + userLogin.setId(1); + userLogin.setUserType(UserType.ADMIN_USER); + result = accessTokenService.delAccessTokenById(userLogin,1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + + @Test + public void testUpdateToken(){ + + when(accessTokenMapper.selectById(1)).thenReturn(getEntity()); + Map result = accessTokenService.updateToken(1,Integer.MAX_VALUE,getDate(),"token"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + // not exist + result = accessTokenService.updateToken(2,Integer.MAX_VALUE,getDate(),"token"); + logger.info(result.toString()); + Assert.assertEquals(Status.ACCESS_TOKEN_NOT_EXIST,result.get(Constants.STATUS)); + + } + + /** + * create entity + * @return + */ + private AccessToken getEntity(){ + AccessToken accessToken = new AccessToken(); + accessToken.setId(1); + accessToken.setUserId(1); + accessToken.setToken("AccessTokenServiceTest"); + Date date = DateUtils.addDays(new Date(),30); + accessToken.setExpireTime(date); + return accessToken; + } + + /** + * entity list + * @return + */ + private List getList(){ + + List list = new ArrayList<>(); + list.add(getEntity()); + return list; + } + + + + /** + * get dateStr + * @return + */ + private String getDate(){ + Date date = DateUtils.addDays(new Date(),30); + return org.apache.dolphinscheduler.common.utils.DateUtils.dateToString(date); + } +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java new file mode 100644 index 0000000000..4a31902af9 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.AlertType; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.dao.entity.AlertGroup; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper; +import org.apache.dolphinscheduler.dao.mapper.UserAlertGroupMapper; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; + +@RunWith(MockitoJUnitRunner.class) +public class AlertGroupServiceTest { + + private static final Logger logger = LoggerFactory.getLogger(AlertGroupServiceTest.class); + + @InjectMocks + private AlertGroupService alertGroupService; + @Mock + private AlertGroupMapper alertGroupMapper; + @Mock + private UserAlertGroupMapper userAlertGroupMapper; + + private String groupName = "AlertGroupServiceTest"; + + @Before + public void setUp() { + } + + + @After + public void after(){ + + } + + + + @Test + public void testQueryAlertgroup(){ + + Mockito.when(alertGroupMapper.queryAllGroupList()).thenReturn(getList()); + HashMap result= alertGroupService.queryAlertgroup(); + logger.info(result.toString()); + List alertGroups = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(alertGroups)); + } + @Test + public void testListPaging(){ + IPage page = new Page<>(1,10); + page.setTotal(1L); + page.setRecords(getList()); + Mockito.when(alertGroupMapper.queryAlertGroupPage(any(Page.class),eq(groupName))).thenReturn(page); + User user = new User(); + // no operate + Map result = alertGroupService.listPaging(user,groupName,1,10); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + //success + user.setUserType(UserType.ADMIN_USER); + result = alertGroupService.listPaging(user,groupName,1,10); + logger.info(result.toString()); + PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists())); + + } + @Test + public void testCreateAlertgroup(){ + + + Mockito.when(alertGroupMapper.insert(any(AlertGroup.class))).thenReturn(2); + User user = new User(); + //no operate + Map result = alertGroupService.createAlertgroup(user,groupName, AlertType.EMAIL,groupName); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + user.setUserType(UserType.ADMIN_USER); + //success + result = alertGroupService.createAlertgroup(user,groupName, AlertType.EMAIL,groupName); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + @Test + public void testUpdateAlertgroup(){ + + User user = new User(); + // no operate + Map result = alertGroupService.updateAlertgroup(user,1,groupName, AlertType.SMS,groupName); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + user.setUserType(UserType.ADMIN_USER); + // not exist + result = alertGroupService.updateAlertgroup(user,1,groupName, AlertType.SMS,groupName); + logger.info(result.toString()); + Assert.assertEquals(Status.ALERT_GROUP_NOT_EXIST,result.get(Constants.STATUS)); + //success + Mockito.when(alertGroupMapper.selectById(2)).thenReturn(getEntity()); + result = alertGroupService.updateAlertgroup(user,2,groupName, AlertType.SMS,groupName); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + } + @Test + public void testDelAlertgroupById(){ + + User user = new User(); + // no operate + Map result = alertGroupService.delAlertgroupById(user,1); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + user.setUserType(UserType.ADMIN_USER); + // not exist + result = alertGroupService.delAlertgroupById(user,2); + logger.info(result.toString()); + Assert.assertEquals(Status.ALERT_GROUP_NOT_EXIST,result.get(Constants.STATUS)); + //success + Mockito.when(alertGroupMapper.selectById(2)).thenReturn(getEntity()); + result = alertGroupService.delAlertgroupById(user,2); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + + } + @Test + public void testGrantUser(){ + + Map result = alertGroupService.grantUser(getLoginUser(),1,"123,321"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + @Test + public void testVerifyGroupName(){ + //group name not exist + Result result = alertGroupService.verifyGroupName(getLoginUser(), groupName); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + Mockito.when(alertGroupMapper.queryByGroupName(groupName)).thenReturn(getList()); + + //group name exist + result = alertGroupService.verifyGroupName(getLoginUser(), groupName); + logger.info(result.toString()); + Assert.assertEquals(Status.ALERT_GROUP_EXIST.getMsg(),result.getMsg()); + } + + + /** + * create admin user + * @return + */ + private User getLoginUser(){ + + User loginUser = new User(); + loginUser.setUserType(UserType.ADMIN_USER); + loginUser.setId(99999999); + return loginUser; + } + + /** + * get list + * @return + */ + private List getList(){ + List alertGroups = new ArrayList<>(); + alertGroups.add(getEntity()); + return alertGroups; + } + + /** + * get entity + * @return + */ + private AlertGroup getEntity(){ + AlertGroup alertGroup = new AlertGroup(); + alertGroup.setId(1); + alertGroup.setGroupName(groupName); + alertGroup.setGroupType(AlertType.EMAIL); + return alertGroup; + } + +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java new file mode 100644 index 0000000000..b155d5959a --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.model.Server; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.dao.MonitorDBDao; +import org.apache.dolphinscheduler.dao.entity.MonitorRecord; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@RunWith(MockitoJUnitRunner.class) +public class MonitorServiceTest { + + private static final Logger logger = LoggerFactory.getLogger(MonitorServiceTest.class); + + @InjectMocks + private MonitorService monitorService; + @Mock + private MonitorDBDao monitorDBDao; + + + @Test + public void testQueryDatabaseState(){ + + Mockito.when(monitorDBDao.queryDatabaseState()).thenReturn(getList()); + Map result = monitorService.queryDatabaseState(null); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + List monitorRecordList = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(monitorRecordList)); + } + @Test + public void testQueryMaster(){ + //TODO need zk +// Map result = monitorService.queryMaster(null); +// logger.info(result.toString()); +// Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + @Test + public void testQueryZookeeperState(){ + //TODO need zk +// Map result = monitorService.queryZookeeperState(null); +// logger.info(result.toString()); +// Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + + @Test + public void testGetServerListFromZK(){ + //TODO need zk +// List serverList = monitorService.getServerListFromZK(true); +// logger.info(serverList.toString()); + } + + private List getList(){ + List monitorRecordList = new ArrayList<>(); + monitorRecordList.add(getEntity()); + return monitorRecordList; + } + + private MonitorRecord getEntity(){ + MonitorRecord monitorRecord = new MonitorRecord(); + monitorRecord.setDbType(DbType.MYSQL); + return monitorRecord; + } + + private List getServerList(){ + List servers = new ArrayList<>(); + servers.add(new Server()); + return servers; + } + +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java new file mode 100644 index 0000000000..51f9e148d1 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java @@ -0,0 +1,356 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.entity.ProjectUser; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; +import org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper; +import org.apache.dolphinscheduler.dao.mapper.UserMapper; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@RunWith(MockitoJUnitRunner.class) +public class ProjectServiceTest { + + + private static final Logger logger = LoggerFactory.getLogger(ProjectServiceTest.class); + + @InjectMocks + private ProjectService projectService; + @Mock + private ProjectMapper projectMapper; + @Mock + private UserMapper userMapper; + @Mock + private ProjectUserMapper projectUserMapper; + @Mock + private ProcessDefinitionMapper processDefinitionMapper; + + + + private String projectName = "ProjectServiceTest"; + + private String userName = "ProjectServiceTest"; + + @Before + public void setUp() { + + } + + + @After + public void after(){ + + } + + @Test + public void testCreateProject(){ + + User loginUser = getLoginUser(); + loginUser.setId(1); + Map result = projectService.createProject(loginUser, projectName, getDesc()); + logger.info(result.toString()); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR,result.get(Constants.STATUS)); + + //project name exist + Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject()); + result = projectService.createProject(loginUser, projectName, projectName); + logger.info(result.toString()); + Assert.assertEquals(Status.PROJECT_ALREADY_EXISTS,result.get(Constants.STATUS)); + + //success + Mockito.when(projectMapper.insert(Mockito.any(Project.class))).thenReturn(1); + result = projectService.createProject(loginUser, "test", "test"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + + } + @Test + public void testQueryById(){ + + //not exist + Map result = projectService.queryById(Integer.MAX_VALUE); + Assert.assertEquals(Status.PROJECT_NOT_FOUNT,result.get(Constants.STATUS)); + logger.info(result.toString()); + + //success + Mockito.when(projectMapper.selectById(1)).thenReturn(getProject()); + result = projectService.queryById(1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + } + @Test + public void testCheckProjectAndAuth(){ + + Mockito.when(projectUserMapper.queryProjectRelation(1, 1)).thenReturn(getProjectUser()); + User loginUser = getLoginUser(); + + Map result = projectService.checkProjectAndAuth(loginUser,null,projectName); + logger.info(result.toString()); + Status status = (Status)result.get(Constants.STATUS); + Assert.assertEquals(Status.PROJECT_NOT_FOUNT,result.get(Constants.STATUS)); + + Project project = getProject(); + //USER_NO_OPERATION_PROJECT_PERM + project.setUserId(2); + result = projectService.checkProjectAndAuth(loginUser,project,projectName); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PROJECT_PERM,result.get(Constants.STATUS)); + + //success + project.setUserId(1); + result = projectService.checkProjectAndAuth(loginUser,project,projectName); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + } + + @Test + public void testHasProjectAndPerm(){ + + // Mockito.when(projectUserMapper.queryProjectRelation(1, 1)).thenReturn(getProjectUser()); + User loginUser = getLoginUser(); + Project project = getProject(); + Map result = new HashMap<>(); + // not exist user + User tempUser = new User(); + tempUser.setId(Integer.MAX_VALUE); + boolean checkResult = projectService.hasProjectAndPerm(tempUser,project,result); + logger.info(result.toString()); + Assert.assertFalse(checkResult); + + //success + result = new HashMap<>(); + project.setUserId(1); + checkResult = projectService.hasProjectAndPerm(loginUser,project,result); + logger.info(result.toString()); + Assert.assertTrue(checkResult); + } + @Test + public void testQueryProjectListPaging(){ + + IPage page = new Page<>(1,10); + page.setRecords(getList()); + page.setTotal(1L); + Mockito.when(projectMapper.queryProjectListPaging(Mockito.any(Page.class), Mockito.eq(1), Mockito.eq(projectName))).thenReturn(page); + User loginUser = getLoginUser(); + + // project owner + Map result = projectService.queryProjectListPaging(loginUser,10,1,projectName); + logger.info(result.toString()); + PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists())); + + //admin + Mockito.when(projectMapper.queryProjectListPaging(Mockito.any(Page.class), Mockito.eq(0), Mockito.eq(projectName))).thenReturn(page); + loginUser.setUserType(UserType.ADMIN_USER); + result = projectService.queryProjectListPaging(loginUser,10,1,projectName); + logger.info(result.toString()); + pageInfo = (PageInfo) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists())); + } + @Test + public void testDeleteProject(){ + + Mockito.when(projectMapper.selectById(1)).thenReturn(getProject()); + User loginUser = getLoginUser(); + //PROJECT_NOT_FOUNT + Map result= projectService.deleteProject(loginUser,12); + logger.info(result.toString()); + Assert.assertEquals(Status.PROJECT_NOT_FOUNT,result.get(Constants.STATUS)); + loginUser.setId(2); + //USER_NO_OPERATION_PROJECT_PERM + result= projectService.deleteProject(loginUser,1); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PROJECT_PERM,result.get(Constants.STATUS)); + + //DELETE_PROJECT_ERROR_DEFINES_NOT_NULL + Mockito.when(processDefinitionMapper.queryAllDefinitionList(1)).thenReturn(getProcessDefinitions()); + loginUser.setUserType(UserType.ADMIN_USER); + result= projectService.deleteProject(loginUser,1); + logger.info(result.toString()); + Assert.assertEquals(Status.DELETE_PROJECT_ERROR_DEFINES_NOT_NULL,result.get(Constants.STATUS)); + + //success + Mockito.when(projectMapper.deleteById(1)).thenReturn(1); + Mockito.when(processDefinitionMapper.queryAllDefinitionList(1)).thenReturn(new ArrayList<>()); + result= projectService.deleteProject(loginUser,1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + + } + + @Test + public void testUpdate(){ + + User loginUser = getLoginUser(); + Project project = getProject(); + project.setId(2); + Mockito.when(projectMapper.queryByName(projectName)).thenReturn(project); + Mockito.when( projectMapper.selectById(1)).thenReturn(getProject()); + // PROJECT_NOT_FOUNT + Map result = projectService.update(loginUser,12,projectName,"desc"); + logger.info(result.toString()); + Assert.assertEquals(Status.PROJECT_NOT_FOUNT,result.get(Constants.STATUS)); + + //PROJECT_ALREADY_EXISTS + result = projectService.update(loginUser,1,projectName,"desc"); + logger.info(result.toString()); + Assert.assertEquals(Status.PROJECT_ALREADY_EXISTS,result.get(Constants.STATUS)); + + //success + project.setUserId(1); + Mockito.when(projectMapper.updateById(Mockito.any(Project.class))).thenReturn(1); + result = projectService.update(loginUser,1,"test","desc"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + } + @Test + public void testQueryAuthorizedProject(){ + + User loginUser = getLoginUser(); + + Mockito.when(projectMapper.queryAuthedProjectListByUserId(1)).thenReturn(getList()); + //USER_NO_OPERATION_PERM + Map result = projectService.queryAuthorizedProject(loginUser,3); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + + //success + loginUser.setUserType(UserType.ADMIN_USER); + result = projectService.queryAuthorizedProject(loginUser,1); + logger.info(result.toString()); + List projects = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(projects)); + + } + @Test + public void testQueryAllProjectList(){ + + Mockito.when(projectMapper.selectList(null)).thenReturn(getList()); + Mockito.when(processDefinitionMapper.selectList(null)).thenReturn(getProcessDefinitions()); + + Map result = projectService.queryAllProjectList(); + logger.info(result.toString()); + List projects = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(projects)); + + } + @Test + public void testQueryUnauthorizedProject(){ + // Mockito.when(projectMapper.queryAuthedProjectListByUserId(1)).thenReturn(getList()); + Mockito.when(projectMapper.queryProjectExceptUserId(2)).thenReturn(getList()); + + User loginUser = new User(); + loginUser.setUserType(UserType.ADMIN_USER); + + Map result = projectService.queryUnauthorizedProject(loginUser,2); + logger.info(result.toString()); + List projects = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(projects)); + } + + + private Project getProject(){ + Project project = new Project(); + project.setId(1); + project.setName(projectName); + project.setUserId(1); + return project; + } + + private List getList(){ + List list = new ArrayList<>(); + list.add(getProject()); + return list; + } + + + /** + * create admin user + * @return + */ + private User getLoginUser(){ + + User loginUser = new User(); + loginUser.setUserType(UserType.GENERAL_USER); + loginUser.setUserName(userName); + loginUser.setId(1); + return loginUser; + + } + + /** + * get project user + + */ + private ProjectUser getProjectUser(){ + ProjectUser projectUser = new ProjectUser(); + projectUser.setProjectId(1); + projectUser.setUserId(1); + return projectUser; + } + + private List getProcessDefinitions(){ + List list = new ArrayList<>(); + ProcessDefinition processDefinition = new ProcessDefinition(); + processDefinition.setProjectId(1); + list.add(processDefinition); + return list; + } + + + + + private String getDesc(){ + return "projectUserMapper.deleteProjectRelation(projectId,userId)projectUserMappe" + + ".deleteProjectRelation(projectId,userId)projectUserMappe" + + "r.deleteProjectRelation(projectId,userId)projectUserMapper" + + ".deleteProjectRelation(projectId,userId)projectUserMapper.deleteProjectRelation(projectId,userId)"; + } + + +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/QueueServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/QueueServiceTest.java new file mode 100644 index 0000000000..dbae95b181 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/QueueServiceTest.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.dao.entity.Queue; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.QueueMapper; +import org.apache.dolphinscheduler.dao.mapper.UserMapper; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@RunWith(MockitoJUnitRunner.class) +public class QueueServiceTest { + + private static final Logger logger = LoggerFactory.getLogger(QueueServiceTest.class); + + @InjectMocks + private QueueService queueService; + @Mock + private QueueMapper queueMapper; + @Mock + private UserMapper userMapper; + private String queueName = "QueueServiceTest"; + + @Before + public void setUp() { + } + + + @After + public void after(){ + } + + @Test + public void testQueryList(){ + + Mockito.when(queueMapper.selectList(null)).thenReturn(getQueueList()); + Map result = queueService.queryList(getLoginUser()); + logger.info(result.toString()); + List queueList = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(queueList)); + + } + @Test + public void testQueryListPage(){ + + IPage page = new Page<>(1,10); + page.setTotal(1L); + page.setRecords(getQueueList()); + Mockito.when(queueMapper.queryQueuePaging(Mockito.any(Page.class), Mockito.eq(queueName))).thenReturn(page); + Map result = queueService.queryList(getLoginUser(),queueName,1,10); + logger.info(result.toString()); + PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists())); + } + @Test + public void testCreateQueue(){ + + // queue is null + Map result = queueService.createQueue(getLoginUser(),null,queueName); + logger.info(result.toString()); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR,result.get(Constants.STATUS)); + // queueName is null + result = queueService.createQueue(getLoginUser(),queueName,null); + logger.info(result.toString()); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR,result.get(Constants.STATUS)); + // correct + result = queueService.createQueue(getLoginUser(),queueName,queueName); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + } + @Test + public void testUpdateQueue(){ + + Mockito.when(queueMapper.selectById(1)).thenReturn(getQueue()); + Mockito.when(queueMapper.queryAllQueueList("test", null)).thenReturn(getQueueList()); + Mockito.when(queueMapper.queryAllQueueList(null, "test")).thenReturn(getQueueList()); + Mockito.when(userMapper.queryUserListByQueue(queueName)).thenReturn(getUserList()); + + // not exist + Map result = queueService.updateQueue(getLoginUser(),0,"queue",queueName); + logger.info(result.toString()); + Assert.assertEquals(Status.QUEUE_NOT_EXIST.getCode(),((Status)result.get(Constants.STATUS)).getCode()); + //no need update + result = queueService.updateQueue(getLoginUser(),1,queueName,queueName); + logger.info(result.toString()); + Assert.assertEquals(Status.NEED_NOT_UPDATE_QUEUE.getCode(),((Status)result.get(Constants.STATUS)).getCode()); + //queue exist + result = queueService.updateQueue(getLoginUser(),1,"test",queueName); + logger.info(result.toString()); + Assert.assertEquals(Status.QUEUE_VALUE_EXIST.getCode(),((Status)result.get(Constants.STATUS)).getCode()); + // queueName exist + result = queueService.updateQueue(getLoginUser(),1,"test1","test"); + logger.info(result.toString()); + Assert.assertEquals(Status.QUEUE_NAME_EXIST.getCode(),((Status)result.get(Constants.STATUS)).getCode()); + //success + result = queueService.updateQueue(getLoginUser(),1,"test1","test1"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getCode(),((Status)result.get(Constants.STATUS)).getCode()); + + } + @Test + public void testVerifyQueue(){ + + Mockito.when(queueMapper.queryAllQueueList(queueName, null)).thenReturn(getQueueList()); + Mockito.when(queueMapper.queryAllQueueList(null, queueName)).thenReturn(getQueueList()); + + //queue null + Result result = queueService.verifyQueue(null,queueName); + logger.info(result.toString()); + Assert.assertEquals(result.getCode().intValue(), Status.REQUEST_PARAMS_NOT_VALID_ERROR.getCode()); + + //queueName null + result = queueService.verifyQueue(queueName,null); + logger.info(result.toString()); + Assert.assertEquals(result.getCode().intValue(), Status.REQUEST_PARAMS_NOT_VALID_ERROR.getCode()); + + //exist queueName + result = queueService.verifyQueue(queueName,queueName); + logger.info(result.toString()); + Assert.assertEquals(result.getCode().intValue(), Status.QUEUE_NAME_EXIST.getCode()); + + //exist queue + result = queueService.verifyQueue(queueName,"test"); + logger.info(result.toString()); + Assert.assertEquals(result.getCode().intValue(), Status.QUEUE_VALUE_EXIST.getCode()); + + // success + result = queueService.verifyQueue("test","test"); + logger.info(result.toString()); + Assert.assertEquals(result.getCode().intValue(), Status.SUCCESS.getCode()); + + + } + /** + * create admin user + * @return + */ + private User getLoginUser(){ + + User loginUser = new User(); + loginUser.setUserType(UserType.ADMIN_USER); + loginUser.setId(99999999); + return loginUser; + } + + private List getUserList(){ + List list = new ArrayList<>(); + list.add(getLoginUser()); + return list; + } + + + /** + * get queue + * @return + */ + private Queue getQueue(){ + Queue queue = new Queue(); + queue.setId(1); + queue.setQueue(queueName); + queue.setQueueName(queueName); + return queue; + } + + private List getQueueList(){ + List queueList = new ArrayList<>(); + queueList.add(getQueue()); + return queueList; + } + +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java index 1920586706..72545a347c 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java @@ -16,37 +16,130 @@ */ package org.apache.dolphinscheduler.api.service; -import org.apache.dolphinscheduler.api.ApiApplicationServer; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateUtils; +import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.dao.entity.Session; import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.dao.mapper.SessionMapper; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.mock.web.MockCookie; +import org.springframework.mock.web.MockHttpServletRequest; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + -@RunWith(SpringRunner.class) -@SpringBootTest(classes = ApiApplicationServer.class) +@RunWith(MockitoJUnitRunner.class) public class SessionServiceTest { private static final Logger logger = LoggerFactory.getLogger(SessionServiceTest.class); - @Autowired + @InjectMocks private SessionService sessionService; + @Mock + private SessionMapper sessionMapper; + + private String sessionId ="aaaaaaaaaaaaaaaaaa"; + + @Before + public void setUp() { + } + + + @After + public void after(){ + } + + /** + * create session + */ + @Test + public void testGetSession(){ + + + Mockito.when(sessionMapper.selectById(sessionId)).thenReturn(getSession()); + // get sessionId from header + MockHttpServletRequest mockHttpServletRequest = new MockHttpServletRequest(); + mockHttpServletRequest.addHeader(Constants.SESSION_ID,sessionId); + mockHttpServletRequest.addHeader("HTTP_X_FORWARDED_FOR","127.0.0.1"); + //query + Session session = sessionService.getSession(mockHttpServletRequest); + Assert.assertNotNull(session); + logger.info("session ip {}",session.getIp()); + + // get sessionId from cookie + mockHttpServletRequest = new MockHttpServletRequest(); + mockHttpServletRequest.addHeader("HTTP_X_FORWARDED_FOR","127.0.0.1"); + MockCookie mockCookie = new MockCookie(Constants.SESSION_ID,sessionId); + mockHttpServletRequest.setCookies(mockCookie); + //query + session = sessionService.getSession(mockHttpServletRequest); + Assert.assertNotNull(session); + logger.info("session ip {}",session.getIp()); + Assert.assertEquals(session.getIp(),"127.0.0.1"); + + + } + + /** + * create session + */ + @Test + public void testCreateSession(){ + + String ip = "127.0.0.1"; + User user = new User(); + user.setUserType(UserType.GENERAL_USER); + user.setId(1); + Mockito.when(sessionMapper.queryByUserId(1)).thenReturn(getSessions()); + String sessionId = sessionService.createSession(user, ip); + logger.info("createSessionId is "+sessionId); + Assert.assertTrue(StringUtils.isNotEmpty(sessionId)); + } + /** + * sign out + * remove ip restrictions + */ @Test - public void createSession(){ + public void testSignOut(){ + + int userId = 88888888; + String ip = "127.0.0.1"; + User user = new User(); + user.setId(userId); + sessionService.signOut(ip ,user); - User loginUser = new User(); - loginUser.setId(1); - loginUser.setUserType(UserType.GENERAL_USER); + } - String session = sessionService.createSession(loginUser, "127.0.0.1"); - Assert.assertTrue(StringUtils.isNotEmpty(session)); + private Session getSession(){ + Session session = new Session(); + session.setId(sessionId); + session.setIp("127.0.0.1"); + session.setLastLoginTime(DateUtils.addDays(new Date(),40)); + session.setUserId(1); + return session; } + + private List getSessions(){ + List sessionList = new ArrayList<>(); + sessionList.add(getSession()); + return sessionList; + } + + } \ No newline at end of file diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java index ddd604ea88..31c8c0222d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java @@ -16,38 +16,241 @@ */ package org.apache.dolphinscheduler.api.service; -import org.apache.dolphinscheduler.api.ApiApplicationServer; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; +import org.apache.dolphinscheduler.dao.mapper.TenantMapper; +import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; +import java.util.ArrayList; +import java.util.List; import java.util.Map; -@RunWith(SpringRunner.class) -@SpringBootTest(classes = ApiApplicationServer.class) +@RunWith(MockitoJUnitRunner.class) public class TenantServiceTest { private static final Logger logger = LoggerFactory.getLogger(TenantServiceTest.class); - @Autowired + @InjectMocks private TenantService tenantService; + @Mock + private TenantMapper tenantMapper; + @Mock + private ProcessDefinitionMapper processDefinitionMapper; + @Mock + private ProcessInstanceMapper processInstanceMapper; + @Mock + private UserMapper userMapper; + + private String tenantCode ="TenantServiceTest"; + private String tenantName ="TenantServiceTest"; + + + @Test + public void testCreateTenant(){ + + User loginUser = getLoginUser(); + Mockito.when(tenantMapper.queryByTenantCode(tenantCode)).thenReturn(getList()); + try { + //check tenantCode + Map result = tenantService.createTenant(getLoginUser(), "%!1111", tenantName, 1, "TenantServiceTest"); + logger.info(result.toString()); + Assert.assertEquals(Status.VERIFY_TENANT_CODE_ERROR,result.get(Constants.STATUS)); + + //check exist + result = tenantService.createTenant(loginUser, tenantCode, tenantName, 1, "TenantServiceTest"); + logger.info(result.toString()); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR,result.get(Constants.STATUS)); + + // success + result = tenantService.createTenant(loginUser, "test", "test", 1, "TenantServiceTest"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } catch (Exception e) { + logger.error("create tenant error",e); + Assert.assertTrue(false); + } + } + + @Test + public void testQueryTenantListPage(){ + + IPage page = new Page<>(1,10); + page.setRecords(getList()); + page.setTotal(1L); + Mockito.when(tenantMapper.queryTenantPaging(Mockito.any(Page.class), Mockito.eq("TenantServiceTest"))).thenReturn(page); + Map result = tenantService.queryTenantList(getLoginUser(), "TenantServiceTest", 1, 10); + logger.info(result.toString()); + PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists())); + + } + + @Test + public void testUpdateTenant(){ + + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + try { + // id not exist + Map result = tenantService.updateTenant(getLoginUser(), 912222, tenantCode, tenantName, 1, "desc"); + logger.info(result.toString()); + // success + Assert.assertEquals(Status.TENANT_NOT_EXIST,result.get(Constants.STATUS)); + result = tenantService.updateTenant(getLoginUser(), 1, tenantCode, "TenantServiceTest001", 1, "desc"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } catch (Exception e) { + logger.error("update tenant error",e); + Assert.assertTrue(false); + } + + } @Test - public void queryTenantList(){ + public void testDeleteTenantById(){ + + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + Mockito.when(processInstanceMapper.queryByTenantIdAndStatus(1, Constants.NOT_TERMINATED_STATES)).thenReturn(getInstanceList()); + Mockito.when(processDefinitionMapper.queryDefinitionListByTenant(2)).thenReturn(getDefinitionsList()); + Mockito.when( userMapper.queryUserListByTenant(3)).thenReturn(getUserList()); + + try { + //TENANT_NOT_EXIST + Map result = tenantService.deleteTenantById(getLoginUser(),12); + logger.info(result.toString()); + Assert.assertEquals(Status.TENANT_NOT_EXIST,result.get(Constants.STATUS)); + + //DELETE_TENANT_BY_ID_FAIL + result = tenantService.deleteTenantById(getLoginUser(),1); + logger.info(result.toString()); + Assert.assertEquals(Status.DELETE_TENANT_BY_ID_FAIL,result.get(Constants.STATUS)); + + //DELETE_TENANT_BY_ID_FAIL_DEFINES + Mockito.when(tenantMapper.queryById(2)).thenReturn(getTenant(2)); + result = tenantService.deleteTenantById(getLoginUser(),2); + logger.info(result.toString()); + Assert.assertEquals(Status.DELETE_TENANT_BY_ID_FAIL_DEFINES,result.get(Constants.STATUS)); + + //DELETE_TENANT_BY_ID_FAIL_USERS + Mockito.when(tenantMapper.queryById(3)).thenReturn(getTenant(3)); + result = tenantService.deleteTenantById(getLoginUser(),3); + logger.info(result.toString()); + Assert.assertEquals(Status.DELETE_TENANT_BY_ID_FAIL_USERS,result.get(Constants.STATUS)); + + // success + Mockito.when(tenantMapper.queryById(4)).thenReturn(getTenant(4)); + result = tenantService.deleteTenantById(getLoginUser(),4); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } catch (Exception e) { + logger.error("delete tenant error",e); + Assert.assertTrue(false); + } + } + + @Test + public void testQueryTenantList(){ + + Mockito.when( tenantMapper.selectList(null)).thenReturn(getList()); + Map result = tenantService.queryTenantList(getLoginUser()); + logger.info(result.toString()); + List tenantList = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(tenantList)); + } + + @Test + public void testVerifyTenantCode(){ + + Mockito.when(tenantMapper.queryByTenantCode(tenantCode)).thenReturn(getList()); + // tenantCode not exist + Result result = tenantService.verifyTenantCode("s00000000000l887888885554444sfjdskfjslakslkdf"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + // tenantCode exist + result = tenantService.verifyTenantCode(getTenant().getTenantCode()); + logger.info(result.toString()); + Assert.assertEquals(Status.TENANT_NAME_EXIST.getMsg(),result.getMsg()); + } + + + /** + * get user + * @return + */ + private User getLoginUser(){ User loginUser = new User(); loginUser.setUserType(UserType.ADMIN_USER); - Map map = tenantService.queryTenantList(loginUser); - Assert.assertEquals(Status.SUCCESS, map.get(Constants.STATUS)); - logger.info(map.toString()); + return loginUser; + } + /** + * get list + * @return + */ + private List getList(){ + List tenantList = new ArrayList<>(); + tenantList.add(getTenant()); + return tenantList; } + + /** + * get tenant + * @return + */ + private Tenant getTenant(){ + return getTenant(1); + } + /** + * get tenant + * @return + */ + private Tenant getTenant(int id){ + Tenant tenant = new Tenant(); + tenant.setId(id); + tenant.setTenantCode(tenantCode); + tenant.setTenantName(tenantName); + return tenant; + } + + private List getUserList(){ + List userList = new ArrayList<>(); + userList.add(getLoginUser()); + return userList; + } + + private List getInstanceList(){ + List processInstances = new ArrayList<>(); + ProcessInstance processInstance = new ProcessInstance(); + processInstances.add(processInstance); + return processInstances; + } + + private List getDefinitionsList(){ + List processDefinitions = new ArrayList<>(); + ProcessDefinition processDefinition = new ProcessDefinition(); + processDefinitions.add(processDefinition); + return processDefinitions; + } + + } \ No newline at end of file diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java index 21f0f7f3b3..30aabe93f2 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java @@ -16,39 +16,449 @@ */ package org.apache.dolphinscheduler.api.service; -import org.apache.dolphinscheduler.api.ApiApplicationServer; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.EncryptionUtils; +import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.*; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; +import java.util.ArrayList; +import java.util.List; import java.util.Map; -@RunWith(SpringRunner.class) -@SpringBootTest(classes = ApiApplicationServer.class) +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) public class UsersServiceTest { private static final Logger logger = LoggerFactory.getLogger(UsersServiceTest.class); - @Autowired + @InjectMocks private UsersService usersService; + @Mock + private UserMapper userMapper; + @Mock + private TenantMapper tenantMapper; + @Mock + private ProjectUserMapper projectUserMapper; + @Mock + private ResourceUserMapper resourcesUserMapper; + @Mock + private UDFUserMapper udfUserMapper; + @Mock + private DataSourceUserMapper datasourceUserMapper; + @Mock + private AlertGroupMapper alertGroupMapper; + + private String queueName ="UsersServiceTestQueue"; + + + @Before + public void before(){ + + + } + @After + public void after(){ + + } + + + @Test + public void testCreateUser(){ + + User user = new User(); + user.setUserType(UserType.ADMIN_USER); + String userName = "userTest0001~"; + String userPassword = "userTest"; + String email = "123@qq.com"; + int tenantId = Integer.MAX_VALUE; + String phone= "13456432345"; + try { + //userName error + Map result = usersService.createUser(user, userName, userPassword, email, tenantId, phone, queueName); + logger.info(result.toString()); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); + + userName = "userTest0001"; + userPassword = "userTest000111111111111111"; + //password error + result = usersService.createUser(user, userName, userPassword, email, tenantId, phone, queueName); + logger.info(result.toString()); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); + + userPassword = "userTest0001"; + email = "1q.com"; + //email error + result = usersService.createUser(user, userName, userPassword, email, tenantId, phone, queueName); + logger.info(result.toString()); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); + + email = "122222@qq.com"; + phone ="2233"; + //phone error + result = usersService.createUser(user, userName, userPassword, email, tenantId, phone, queueName); + logger.info(result.toString()); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); + + phone = "13456432345"; + //tenantId not exists + result = usersService.createUser(user, userName, userPassword, email, tenantId, phone, queueName); + logger.info(result.toString()); + Assert.assertEquals(Status.TENANT_NOT_EXIST, result.get(Constants.STATUS)); + //success + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + result = usersService.createUser(user, userName, userPassword, email, 1, phone, queueName); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + + } catch (Exception e) { + logger.error(Status.CREATE_USER_ERROR.getMsg(),e); + Assert.assertTrue(false); + } + } + + @Test + public void testQueryUser(){ + + String userName = "userTest0001"; + String userPassword = "userTest0001"; + when(userMapper.queryUserByNamePassword(userName,EncryptionUtils.getMd5(userPassword))).thenReturn(getGeneralUser()); + User queryUser = usersService.queryUser(userName, userPassword); + logger.info(queryUser.toString()); + Assert.assertTrue(queryUser!=null); + } + + + + @Test + public void testQueryUserList(){ + + + User user = new User(); + + //no operate + Map result = usersService.queryUserList(user); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + + //success + user.setUserType(UserType.ADMIN_USER); + when(userMapper.selectList(null )).thenReturn(getUserList()); + result = usersService.queryUserList(user); + List userList = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(userList.size()>0); + } + + @Test + public void testQueryUserListPage(){ + + + User user = new User(); + IPage page = new Page<>(1,10); + page.setRecords(getUserList()); + when(userMapper.queryUserPaging(any(Page.class), eq("userTest"))).thenReturn(page); + + //no operate + Map result = usersService.queryUserList(user,"userTest",1,10); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + + //success + user.setUserType(UserType.ADMIN_USER); + result = usersService.queryUserList(user,"userTest",1,10); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); + Assert.assertTrue(pageInfo.getLists().size()>0); + } + + @Test + public void testUpdateUser(){ + + String userName = "userTest0001"; + String userPassword = "userTest0001"; + try { + //user not exist + Map result = usersService.updateUser(0,userName,userPassword,"3443@qq.com",1,"13457864543","queue"); + Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS)); + logger.info(result.toString()); + + //success + when(userMapper.selectById(1)).thenReturn(getUser()); + result = usersService.updateUser(1,userName,userPassword,"32222s@qq.com",1,"13457864543","queue"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + } catch (Exception e) { + logger.error("update user error",e); + Assert.assertTrue(false); + } + } + + @Test + public void testDeleteUserById(){ + + User loginUser = new User(); + try { + when(userMapper.queryTenantCodeByUserId(1)).thenReturn(getUser()); + when(userMapper.selectById(1)).thenReturn(getUser()); + + //no operate + Map result = usersService.deleteUserById(loginUser,3); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + + // user not exist + loginUser.setUserType(UserType.ADMIN_USER); + result = usersService.deleteUserById(loginUser,3); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS)); + + //success + result = usersService.deleteUserById(loginUser,1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + } catch (Exception e) { + logger.error("delete user error",e); + Assert.assertTrue(false); + } + + + } + + @Test + public void testGrantProject(){ + + when(userMapper.selectById(1)).thenReturn(getUser()); + User loginUser = new User(); + String projectIds= "100000,120000"; + Map result = usersService.grantProject(loginUser, 1, projectIds); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + //user not exist + loginUser.setUserType(UserType.ADMIN_USER); + result = usersService.grantProject(loginUser, 2, projectIds); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS)); + //success + result = usersService.grantProject(loginUser, 1, projectIds); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + } + + @Test + public void testGrantResources(){ + + String resourceIds = "100000,120000"; + when(userMapper.selectById(1)).thenReturn(getUser()); + User loginUser = new User(); + Map result = usersService.grantResources(loginUser, 1, resourceIds); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + //user not exist + loginUser.setUserType(UserType.ADMIN_USER); + result = usersService.grantResources(loginUser, 2, resourceIds); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS)); + //success + result = usersService.grantResources(loginUser, 1, resourceIds); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + } + + + @Test + public void testGrantUDFFunction(){ + + String udfIds = "100000,120000"; + when(userMapper.selectById(1)).thenReturn(getUser()); + User loginUser = new User(); + Map result = usersService.grantUDFFunction(loginUser, 1, udfIds); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + //user not exist + loginUser.setUserType(UserType.ADMIN_USER); + result = usersService.grantUDFFunction(loginUser, 2, udfIds); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS)); + //success + result = usersService.grantUDFFunction(loginUser, 1, udfIds); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + } + + @Test + public void testGrantDataSource(){ + + String datasourceIds = "100000,120000"; + when(userMapper.selectById(1)).thenReturn(getUser()); + User loginUser = new User(); + Map result = usersService.grantDataSource(loginUser, 1, datasourceIds); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + //user not exist + loginUser.setUserType(UserType.ADMIN_USER); + result = usersService.grantDataSource(loginUser, 2, datasourceIds); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS)); + //success + result = usersService.grantDataSource(loginUser, 1, datasourceIds); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + + } + @Test public void getUserInfo(){ User loginUser = new User(); - loginUser.setId(19); - loginUser.setUserType(UserType.GENERAL_USER); - Map map = usersService.getUserInfo(loginUser); - Assert.assertEquals(Status.SUCCESS, map.get(Constants.STATUS)); - logger.info(map.toString()); + loginUser.setUserName("admin"); + loginUser.setUserType(UserType.ADMIN_USER); + // get admin user + Map result = usersService.getUserInfo(loginUser); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + User tempUser = (User) result.get(Constants.DATA_LIST); + //check userName + Assert.assertEquals("admin",tempUser.getUserName()); + //get general user + loginUser.setUserType(null); + loginUser.setId(1); + when(userMapper.queryDetailsById(1)).thenReturn(getGeneralUser()); + result = usersService.getUserInfo(loginUser); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + tempUser = (User) result.get(Constants.DATA_LIST); + //check userName + Assert.assertEquals("userTest0001",tempUser.getUserName()); } + + + @Test + public void testQueryAllGeneralUsers(){ + + User loginUser = new User(); + //no operate + Map result = usersService.queryAllGeneralUsers(loginUser); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + //success + loginUser.setUserType(UserType.ADMIN_USER); + when(userMapper.queryAllGeneralUser()).thenReturn(getUserList()); + result = usersService.queryAllGeneralUsers(loginUser); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + List userList = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(userList)); + } + + @Test + public void testVerifyUserName(){ + + //not exist user + Result result = usersService.verifyUserName("admin89899"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); + //exist user + when(userMapper.queryByUserNameAccurately("userTest0001")).thenReturn(getUser()); + result = usersService.verifyUserName("userTest0001"); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NAME_EXIST.getMsg(), result.getMsg()); + } + + @Test + public void testUnauthorizedUser(){ + + User loginUser = new User(); + when(userMapper.selectList(null )).thenReturn(getUserList()); + when( userMapper.queryUserListByAlertGroupId(2)).thenReturn(getUserList()); + //no operate + Map result = usersService.unauthorizedUser(loginUser, 2); + logger.info(result.toString()); + loginUser.setUserType(UserType.ADMIN_USER); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + //success + result = usersService.unauthorizedUser(loginUser, 2); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + } + + + @Test + public void testAuthorizedUser(){ + + User loginUser = new User(); + when(userMapper.queryUserListByAlertGroupId(2)).thenReturn(getUserList()); + //no operate + Map result = usersService.authorizedUser(loginUser, 2); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + //success + loginUser.setUserType(UserType.ADMIN_USER); + result = usersService.authorizedUser(loginUser, 2); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + List userList = (List) result.get(Constants.DATA_LIST); + logger.info(result.toString()); + Assert.assertTrue(CollectionUtils.isNotEmpty(userList)); + } + + /** + * get user + * @return + */ + private User getGeneralUser(){ + + User user = new User(); + user.setUserType(UserType.GENERAL_USER); + user.setUserName("userTest0001"); + user.setUserPassword("userTest0001"); + return user; + } + + + private List getUserList(){ + List userList = new ArrayList<>(); + userList.add(getGeneralUser()); + return userList; + } + + /** + * get user + */ + private User getUser(){ + + User user = new User(); + user.setUserType(UserType.ADMIN_USER); + user.setUserName("userTest0001"); + user.setUserPassword("userTest0001"); + return user; + } + + + private Tenant getTenant(){ + Tenant tenant = new Tenant(); + tenant.setId(1); + return tenant; + } + } \ No newline at end of file diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java new file mode 100644 index 0000000000..2c535054a7 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.entity.WorkerGroup; +import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; +import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@RunWith(MockitoJUnitRunner.class) +public class WorkerGroupServiceTest { + + private static final Logger logger = LoggerFactory.getLogger(WorkerGroupServiceTest.class); + + @InjectMocks + private WorkerGroupService workerGroupService; + @Mock + private WorkerGroupMapper workerGroupMapper; + @Mock + private ProcessInstanceMapper processInstanceMapper; + + + private String groupName="groupName000001"; + + + /** + * create or update a worker group + */ + @Test + public void testSaveWorkerGroup(){ + + User user = new User(); + // general user add + user.setUserType(UserType.GENERAL_USER); + Map result = workerGroupService.saveWorkerGroup(user, 0, groupName, "127.0.0.1"); + logger.info(result.toString()); + Assert.assertEquals( Status.USER_NO_OPERATION_PERM.getMsg(),(String) result.get(Constants.MSG)); + + //success + user.setUserType(UserType.ADMIN_USER); + result = workerGroupService.saveWorkerGroup(user, 0, groupName, "127.0.0.1"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),(String)result.get(Constants.MSG)); + // group name exist + Mockito.when(workerGroupMapper.selectById(2)).thenReturn(getWorkerGroup(2)); + Mockito.when(workerGroupMapper.queryWorkerGroupByName(groupName)).thenReturn(getList()); + result = workerGroupService.saveWorkerGroup(user, 2, groupName, "127.0.0.1"); + logger.info(result.toString()); + Assert.assertEquals(Status.NAME_EXIST,result.get(Constants.STATUS)); + + } + + /** + * query worker group paging + */ + @Test + public void testQueryAllGroupPaging(){ + + User user = new User(); + // general user add + user.setUserType(UserType.GENERAL_USER); + Map result = workerGroupService.queryAllGroupPaging(user, 1, 10, groupName); + logger.info(result.toString()); + Assert.assertEquals((String) result.get(Constants.MSG), Status.USER_NO_OPERATION_PERM.getMsg()); + //success + user.setUserType(UserType.ADMIN_USER); + Page page = new Page<>(1,10); + page.setRecords(getList()); + page.setSize(1L); + Mockito.when(workerGroupMapper.queryListPaging(Mockito.any(Page.class), Mockito.eq(groupName))).thenReturn(page); + result = workerGroupService.queryAllGroupPaging(user, 1, 10, groupName); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),(String)result.get(Constants.MSG)); + PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists())); + } + + /** + * delete group by id + */ + @Test + public void testDeleteWorkerGroupById(){ + + //DELETE_WORKER_GROUP_BY_ID_FAIL + Mockito.when(processInstanceMapper.queryByWorkerGroupIdAndStatus(1, Constants.NOT_TERMINATED_STATES)).thenReturn(getProcessInstanceList()); + Map result = workerGroupService.deleteWorkerGroupById(1); + logger.info(result.toString()); + Assert.assertEquals(Status.DELETE_WORKER_GROUP_BY_ID_FAIL.getCode(),((Status) result.get(Constants.STATUS)).getCode()); + + //correct + result = workerGroupService.deleteWorkerGroupById(2); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),(String)result.get(Constants.MSG)); + + } + + @Test + public void testQueryAllGroup(){ + Mockito.when(workerGroupMapper.queryAllWorkerGroup()).thenReturn(getList()); + Map result = workerGroupService.queryAllGroup(); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),(String)result.get(Constants.MSG)); + List workerGroupList = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(workerGroupList.size()>0); + } + + + /** + * get processInstances + * @return + */ + private List getProcessInstanceList(){ + + List processInstances = new ArrayList<>(); + processInstances.add(new ProcessInstance()); + return processInstances; + } + /** + * get Group + * @return + */ + private WorkerGroup getWorkerGroup(int id){ + WorkerGroup workerGroup = new WorkerGroup(); + workerGroup.setName(groupName); + workerGroup.setId(id); + return workerGroup; + } + private WorkerGroup getWorkerGroup(){ + + return getWorkerGroup(1); + } + + private List getList(){ + List list = new ArrayList<>(); + list.add(getWorkerGroup()); + return list; + } + +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 7936bf7dc8..fafa7be6ef 100644 --- a/pom.xml +++ b/pom.xml @@ -666,6 +666,15 @@ **/api/utils/CheckUtilsTest.java **/api/utils/FileUtilsTest.java **/api/enums/*.java + **/api/service/AccessTokenServiceTest.java + **/api/service/QueueServiceTest.java + **/api/service/MonitorServiceTest.java + **/api/service/SessionServiceTest.java + **/api/service/UsersServiceTest.java + **/api/service/TenantServiceTest.java + **/api/service/WorkerGroupServiceTest.java + **/api/service/AlertGroupServiceTest.java + **/api/service/ProjectServiceTest.java **/alert/utils/ExcelUtilsTest.java **/alert/utils/FuncUtilsTest.java **/alert/utils/JSONUtilsTest.java From 5e935d579eec25701a6ce1d29fea95b161173ba3 Mon Sep 17 00:00:00 2001 From: Tboy Date: Mon, 30 Dec 2019 17:59:06 +0800 Subject: [PATCH 50/53] refactor AbstractZKClient (#1627) * we should insert alert DB once , and trigger this type of alert 3 times * refactor AbstractZKClient --- .../common/zk/AbstractZKClient.java | 96 +++++++------------ 1 file changed, 36 insertions(+), 60 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java index 0e95dddb36..c3ba718270 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java @@ -16,6 +16,22 @@ */ package org.apache.dolphinscheduler.common.zk; +import static org.apache.dolphinscheduler.common.Constants.ADD_ZK_OP; +import static org.apache.dolphinscheduler.common.Constants.DELETE_ZK_OP; +import static org.apache.dolphinscheduler.common.Constants.MASTER_PREFIX; +import static org.apache.dolphinscheduler.common.Constants.SINGLE_SLASH; +import static org.apache.dolphinscheduler.common.Constants.UNDERLINE; +import static org.apache.dolphinscheduler.common.Constants.WORKER_PREFIX; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.imps.CuratorFrameworkState; +import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.IStoppable; import org.apache.dolphinscheduler.common.enums.ZKNodeType; @@ -23,26 +39,9 @@ import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.ResInfo; -import org.apache.commons.configuration.Configuration; -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.commons.lang3.StringUtils; -import org.apache.curator.RetryPolicy; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.framework.imps.CuratorFrameworkState; -import org.apache.curator.framework.recipes.locks.InterProcessMutex; -import org.apache.curator.framework.state.ConnectionState; -import org.apache.curator.framework.state.ConnectionStateListener; -import org.apache.curator.retry.ExponentialBackoffRetry; -import org.apache.zookeeper.CreateMode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; - -import static org.apache.dolphinscheduler.common.Constants.*; - /** * abstract zookeeper client @@ -70,8 +69,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ return; } - byte[] bytes = zkClient.getData().forPath(znode); - String resInfoStr = new String(bytes); + String resInfoStr = super.get(znode); String[] splits = resInfoStr.split(Constants.COMMA); if (splits.length != Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH){ return; @@ -107,8 +105,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ String type = serverType.equals(MASTER_PREFIX) ? MASTER_PREFIX : WORKER_PREFIX; String deadServerPath = getDeadZNodeParentPath() + SINGLE_SLASH + type + UNDERLINE + ipSeqNo; - if(zkClient.checkExists().forPath(zNode) == null || - zkClient.checkExists().forPath(deadServerPath) != null ){ + if(!isExisted(zNode) || isExisted(deadServerPath)){ return true; } @@ -118,14 +115,12 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ public void removeDeadServerByHost(String host, String serverType) throws Exception { - List deadServers = zkClient.getChildren().forPath(getDeadZNodeParentPath()); + List deadServers = super.getChildrenKeys(getDeadZNodeParentPath()); for(String serverPath : deadServers){ if(serverPath.startsWith(serverType+UNDERLINE+host)){ - String server = getDeadZNodeParentPath() + SINGLE_SLASH + serverPath; - if(zkClient.checkExists().forPath(server) != null){ - zkClient.delete().forPath(server); - logger.info("{} server {} deleted from zk dead server path success" , serverType , host); - } + String server = getDeadZNodeParentPath() + SINGLE_SLASH + serverPath; + super.remove(server); + logger.info("{} server {} deleted from zk dead server path success" , serverType , host); } } } @@ -143,8 +138,8 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ // create temporary sequence nodes for master znode String parentPath = getZNodeParentPath(zkNodeType); String serverPathPrefix = parentPath + "/" + OSUtils.getHost(); - String registerPath = zkClient.create().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath( - serverPathPrefix + UNDERLINE, heartbeatZKInfo.getBytes()); + String registerPath = serverPathPrefix + UNDERLINE; + super.persistEphemeral(registerPath, heartbeatZKInfo); logger.info("register {} node {} success" , zkNodeType.toString(), registerPath); return registerPath; } @@ -165,7 +160,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ } registerPath = createZNodePath(zkNodeType); - // handle dead server + // handle dead server handleDeadServer(registerPath, zkNodeType, Constants.DELETE_ZK_OP); return registerPath; @@ -196,10 +191,10 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ }else if(opType.equals(ADD_ZK_OP)){ String deadServerPath = getDeadZNodeParentPath() + SINGLE_SLASH + type + UNDERLINE + ipSeqNo; - if(zkClient.checkExists().forPath(deadServerPath) == null){ + if(!super.isExisted(deadServerPath)){ //add dead server info to zk dead server path : /dead-servers/ - zkClient.create().forPath(deadServerPath,(type + UNDERLINE + ipSeqNo).getBytes()); + super.persist(deadServerPath,(type + UNDERLINE + ipSeqNo)); logger.info("{} server dead , and {} added to zk dead server path success" , zkNodeType.toString(), zNode); @@ -226,19 +221,13 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ List childrenList = new ArrayList<>(); try { // read master node parent path from conf - if(zkClient.checkExists().forPath(getZNodeParentPath(ZKNodeType.MASTER)) != null){ - childrenList = zkClient.getChildren().forPath(getZNodeParentPath(ZKNodeType.MASTER)); + if(super.isExisted(getZNodeParentPath(ZKNodeType.MASTER))){ + childrenList = super.getChildrenKeys(getZNodeParentPath(ZKNodeType.MASTER)); } } catch (Exception e) { - if(e.getMessage().contains("java.lang.IllegalStateException: instance must be started")){ - logger.error("zookeeper service not started",e); - }else{ - logger.error(e.getMessage(),e); - } - - }finally { - return childrenList.size(); + logger.error("getActiveMasterNum error",e); } + return childrenList.size(); } /** @@ -280,10 +269,9 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ Map masterMap = new HashMap<>(); try { String path = getZNodeParentPath(zkNodeType); - List serverList = getZkClient().getChildren().forPath(path); + List serverList = super.getChildrenKeys(path); for(String server : serverList){ - byte[] bytes = getZkClient().getData().forPath(path + "/" + server); - masterMap.putIfAbsent(server, new String(bytes)); + masterMap.putIfAbsent(server, super.get(path + "/" + server)); } } catch (Exception e) { logger.error("get server list failed : " + e.getMessage(), e); @@ -430,27 +418,15 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ */ protected void initSystemZNode(){ try { - createNodePath(getMasterZNodeParentPath()); - createNodePath(getWorkerZNodeParentPath()); - createNodePath(getDeadZNodeParentPath()); + persist(getMasterZNodeParentPath(), ""); + persist(getWorkerZNodeParentPath(), ""); + persist(getDeadZNodeParentPath(), ""); } catch (Exception e) { logger.error("init system znode failed : " + e.getMessage(),e); } } - /** - * create zookeeper node path if not exists - * @param zNodeParentPath zookeeper parent path - * @throws Exception errors - */ - private void createNodePath(String zNodeParentPath) throws Exception { - if(null == zkClient.checkExists().forPath(zNodeParentPath)){ - zkClient.create().creatingParentContainersIfNeeded() - .withMode(CreateMode.PERSISTENT).forPath(zNodeParentPath); - } - } - /** * server self dead, stop all threads * @param serverHost server host From 426c027ddf738f3a6671195f2afd6a655ebc166a Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Mon, 30 Dec 2019 18:11:14 +0800 Subject: [PATCH 51/53] Click the run button to add the workflow name echo #1339 Point 11 (#1638) * Increase script text box to zoom in and fix log loading * Timing operation adds default value for start and end dates * Click the run button to add the workflow name echo --- .../pages/definition/pages/list/_source/start.vue | 8 ++++++++ .../pages/projects/pages/index/_source/chartConfig.js | 2 +- .../projects/pages/index/_source/processStateCount.vue | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue index dfc7f49c20..2049b6b16e 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue @@ -19,6 +19,12 @@
{{$t('Please set the parameters before starting')}}
+
+
+ {{$t('Process Name')}} +
+
{{workflowName}}
+
{{$t('Failure Strategy')}} @@ -174,6 +180,7 @@ processDefinitionId: 0, failureStrategy: 'CONTINUE', warningTypeList: warningTypeList, + workflowName: '', warningType: '', notifyGroupList: [], warningGroupId: '', @@ -276,6 +283,7 @@ this.warningGroupId = '' }) }) + this.workflowName = this.item.name }, computed: {}, components: { mEmail, mPriority, mWorkerGroups } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/chartConfig.js b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/chartConfig.js index f693ed578e..b8601a69c9 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/chartConfig.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/chartConfig.js @@ -27,7 +27,7 @@ let pie = { avoidLabelOverlap: true, // Whether to prevent the label overlap policy hoverAnimation: true, // Whether to enable hover to enlarge the animation on the sector. radius: ['30%', '60%'], - center: ['50%', '50%'], + center: ['53%', '60%'], label: { align: 'left', normal: { diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/processStateCount.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/processStateCount.vue index c21579a434..d121034932 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/processStateCount.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/processStateCount.vue @@ -29,7 +29,7 @@ {{$t('Number')}} {{$t('State')}} - + {{$index+1}} {{item.value}} {{item.key}} From d16500e8bee48f63b80235ce97a576af1b5a150d Mon Sep 17 00:00:00 2001 From: bao liang <29528966+lenboo@users.noreply.github.com> Date: Mon, 30 Dec 2019 18:14:31 +0800 Subject: [PATCH 52/53] [Feature] merge some configurations #1635 (#1636) * merge hadoop.properties into common.properties * merge hadoop,zookeeper.properties into common.properties remove combined.properties/master.properties/worker.properties * change db user/pwd to test/test --- .../resources/application-combined.properties | 40 ------- .../dolphinscheduler/common/Constants.java | 21 +--- .../common/utils/PropertyUtils.java | 3 +- .../common/zk/ZookeeperConfig.java | 2 +- .../resources/{common => }/common.properties | 51 ++++++-- .../resources/common/hadoop/hadoop.properties | 35 ------ .../src/main/resources/quartz.properties | 18 +-- .../src/main/resources/zookeeper.properties | 43 ------- .../dao/utils/PropertyUtils.java | 2 +- ...-dao.properties => application.properties} | 0 .../resources/application-master.properties | 18 --- .../resources/application-worker.properties | 18 --- install.sh | 109 ++++++------------ script/dolphinscheduler-daemon.sh | 8 +- 14 files changed, 100 insertions(+), 268 deletions(-) delete mode 100644 dolphinscheduler-api/src/main/resources/application-combined.properties rename dolphinscheduler-common/src/main/resources/{common => }/common.properties (67%) delete mode 100644 dolphinscheduler-common/src/main/resources/common/hadoop/hadoop.properties delete mode 100644 dolphinscheduler-common/src/main/resources/zookeeper.properties rename dolphinscheduler-dao/src/main/resources/{application-dao.properties => application.properties} (100%) delete mode 100644 dolphinscheduler-server/src/main/resources/application-master.properties delete mode 100644 dolphinscheduler-server/src/main/resources/application-worker.properties diff --git a/dolphinscheduler-api/src/main/resources/application-combined.properties b/dolphinscheduler-api/src/main/resources/application-combined.properties deleted file mode 100644 index 0264c4f4b6..0000000000 --- a/dolphinscheduler-api/src/main/resources/application-combined.properties +++ /dev/null @@ -1,40 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -logging.config=classpath:combined_logback.xml - -# server port -server.port=12345 - -# session config -server.servlet.session.timeout=7200 - -server.servlet.context-path=/dolphinscheduler/ - -# file size limit for upload -spring.servlet.multipart.max-file-size=1024MB -spring.servlet.multipart.max-request-size=1024MB - -#post content -server.jetty.max-http-post-size=5000000 - -spring.messages.encoding=UTF-8 - -#i18n classpath folder , file prefix messages， if have many files, use "," seperator -spring.messages.basename=i18n/messages - -server.is-combined-server=true diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 6073f218a4..1222faa1a7 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -26,25 +26,10 @@ import java.util.regex.Pattern; */ public final class Constants { - /** - * zookeeper properties path - */ - public static final String ZOOKEEPER_PROPERTIES_PATH = "zookeeper.properties"; - - /** - * hadoop properties path - */ - public static final String HADOOP_PROPERTIES_PATH = "/common/hadoop/hadoop.properties"; - /** * common properties path */ - public static final String COMMON_PROPERTIES_PATH = "/common/common.properties"; - - /** - * dao properties path - */ - public static final String DAO_PROPERTIES_PATH = "application.properties"; + public static final String COMMON_PROPERTIES_PATH = "/common.properties"; /** * fs.defaultFS @@ -227,6 +212,8 @@ public final class Constants { public static final String ZOOKEEPER_CONNECTION_TIMEOUT = "zookeeper.connection.timeout"; public static final String ZOOKEEPER_RETRY_SLEEP = "zookeeper.retry.sleep"; + public static final String ZOOKEEPER_RETRY_BASE_SLEEP = "zookeeper.retry.base.sleep"; + public static final String ZOOKEEPER_RETRY_MAX_SLEEP = "zookeeper.retry.max.sleep"; public static final String ZOOKEEPER_RETRY_MAXTIME = "zookeeper.retry.maxtime"; @@ -469,7 +456,7 @@ public final class Constants { /** * task record configuration path */ - public static final String APPLICATION_PROPERTIES = "application-dao.properties"; + public static final String APPLICATION_PROPERTIES = "application.properties"; public static final String TASK_RECORD_URL = "task.record.datasource.url"; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index ea01d025a7..c3e8197079 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -29,7 +29,6 @@ import java.util.Map; import java.util.Properties; import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH; -import static org.apache.dolphinscheduler.common.Constants.HADOOP_PROPERTIES_PATH; /** * property utils @@ -51,7 +50,7 @@ public class PropertyUtils { } private void init(){ - String[] propertyFiles = new String[]{HADOOP_PROPERTIES_PATH,COMMON_PROPERTIES_PATH}; + String[] propertyFiles = new String[]{COMMON_PROPERTIES_PATH}; for (String fileName : propertyFiles) { InputStream fis = null; try { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java index a90a147425..75a9f6c5f4 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java @@ -24,7 +24,7 @@ import org.springframework.stereotype.Component; * zookeeper conf */ @Component -@PropertySource("classpath:zookeeper.properties") +@PropertySource("classpath:common.properties") public class ZookeeperConfig { //zk connect config diff --git a/dolphinscheduler-common/src/main/resources/common/common.properties b/dolphinscheduler-common/src/main/resources/common.properties similarity index 67% rename from dolphinscheduler-common/src/main/resources/common/common.properties rename to dolphinscheduler-common/src/main/resources/common.properties index 01e321167d..9826c56593 100644 --- a/dolphinscheduler-common/src/main/resources/common/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -18,14 +18,21 @@ #task queue implementation, default "zookeeper" dolphinscheduler.queue.impl=zookeeper -# user data directory path, self configuration, please make sure the directory exists and have read write permissions -data.basedir.path=/tmp/dolphinscheduler +#zookeeper cluster. multiple are separated by commas. eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181 +zookeeper.quorum=localhost:2181 -# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions -data.download.basedir.path=/tmp/dolphinscheduler/download +#dolphinscheduler root directory +zookeeper.dolphinscheduler.root=/dolphinscheduler -# process execute directory. self configuration, please make sure the directory exists and have read write permissions -process.exec.basepath=/tmp/dolphinscheduler/exec +#dolphinscheduler failover directory +zookeeper.session.timeout=300 +zookeeper.connection.timeout=300 +zookeeper.retry.base.sleep=100 +zookeeper.retry.max.sleep=30000 +zookeeper.retry.maxtime=5 + +# resource upload startup type : HDFS,S3,NONE +res.upload.startup.type=NONE # Users who have permission to create directories under the HDFS root path hdfs.root.user=hdfs @@ -33,8 +40,15 @@ hdfs.root.user=hdfs # data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended data.store2hdfs.basepath=/dolphinscheduler -# resource upload startup type : HDFS,S3,NONE -res.upload.startup.type=NONE +# user data directory path, self configuration, please make sure the directory exists and have read write permissions +data.basedir.path=/tmp/dolphinscheduler + +# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions +data.download.basedir.path=/tmp/dolphinscheduler/download + +# process execute directory. self configuration, please make sure the directory exists and have read write permissions +process.exec.basepath=/tmp/dolphinscheduler/exec + # whether kerberos starts hadoop.security.authentication.startup.state=false @@ -57,3 +71,24 @@ resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties # is development state? default "false" development.state=true + +# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml +# to the conf directory，support s3，for example : s3a://dolphinscheduler +fs.defaultFS=hdfs://mycluster:8020 + +# s3 need，s3 endpoint +fs.s3a.endpoint=http://192.168.199.91:9010 + +# s3 need，s3 access key +fs.s3a.access.key=A3DXS30FO22544RE + +# s3 need，s3 secret key +fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK + +#resourcemanager ha note this need ips , this empty if single +yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx + +# If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine +yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s + + diff --git a/dolphinscheduler-common/src/main/resources/common/hadoop/hadoop.properties b/dolphinscheduler-common/src/main/resources/common/hadoop/hadoop.properties deleted file mode 100644 index 2c19b4a52e..0000000000 --- a/dolphinscheduler-common/src/main/resources/common/hadoop/hadoop.properties +++ /dev/null @@ -1,35 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml -# to the conf directory，support s3，for example : s3a://dolphinscheduler -fs.defaultFS=hdfs://mycluster:8020 - -# s3 need，s3 endpoint -fs.s3a.endpoint=http://192.168.199.91:9010 - -# s3 need，s3 access key -fs.s3a.access.key=A3DXS30FO22544RE - -# s3 need，s3 secret key -fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK - -#resourcemanager ha note this need ips , this empty if single -yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx - -# If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine -yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s \ No newline at end of file diff --git a/dolphinscheduler-common/src/main/resources/quartz.properties b/dolphinscheduler-common/src/main/resources/quartz.properties index d388691c3f..684aabcd66 100644 --- a/dolphinscheduler-common/src/main/resources/quartz.properties +++ b/dolphinscheduler-common/src/main/resources/quartz.properties @@ -18,6 +18,15 @@ #============================================================================ # Configure Main Scheduler Properties #============================================================================ +#org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.StdJDBCDelegate +org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.PostgreSQLDelegate +#org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver +org.quartz.dataSource.myDs.driver = org.postgresql.Driver +#org.quartz.dataSource.myDs.URL = jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?characterEncoding=utf8 +org.quartz.dataSource.myDs.URL = jdbc:postgresql://localhost:5432/dolphinscheduler?characterEncoding=utf8 +org.quartz.dataSource.myDs.user = test +org.quartz.dataSource.myDs.password = test + org.quartz.scheduler.instanceName = DolphinScheduler org.quartz.scheduler.instanceId = AUTO org.quartz.scheduler.makeSchedulerThreadDaemon = true @@ -37,8 +46,7 @@ org.quartz.threadPool.threadPriority = 5 #============================================================================ org.quartz.jobStore.class = org.quartz.impl.jdbcjobstore.JobStoreTX -#org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.StdJDBCDelegate -org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.PostgreSQLDelegate + org.quartz.jobStore.tablePrefix = QRTZ_ org.quartz.jobStore.isClustered = true org.quartz.jobStore.misfireThreshold = 60000 @@ -49,11 +57,5 @@ org.quartz.jobStore.dataSource = myDs # Configure Datasources #============================================================================ org.quartz.dataSource.myDs.connectionProvider.class = org.apache.dolphinscheduler.server.quartz.DruidConnectionProvider -#org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver -org.quartz.dataSource.myDs.driver = org.postgresql.Driver -#org.quartz.dataSource.myDs.URL = jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?characterEncoding=utf8 -org.quartz.dataSource.myDs.URL = jdbc:postgresql://192.168.xx.xx:5432/dolphinscheduler?characterEncoding=utf8 -org.quartz.dataSource.myDs.user = xx -org.quartz.dataSource.myDs.password = xx org.quartz.dataSource.myDs.maxConnections = 10 org.quartz.dataSource.myDs.validationQuery = select 1 \ No newline at end of file diff --git a/dolphinscheduler-common/src/main/resources/zookeeper.properties b/dolphinscheduler-common/src/main/resources/zookeeper.properties deleted file mode 100644 index a560de4f23..0000000000 --- a/dolphinscheduler-common/src/main/resources/zookeeper.properties +++ /dev/null @@ -1,43 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -#zookeeper cluster. multiple are separated by commas. eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181 -zookeeper.quorum=localhost:2181 - -#dolphinscheduler root directory -zookeeper.dolphinscheduler.root=/dolphinscheduler - -#zookeeper server dirctory -#zookeeper.dolphinscheduler.dead.servers=/dolphinscheduler/dead-servers -#zookeeper.dolphinscheduler.masters=/dolphinscheduler/masters -#zookeeper.dolphinscheduler.workers=/dolphinscheduler/workers - -#zookeeper lock dirctory -#zookeeper.dolphinscheduler.lock.masters=/dolphinscheduler/lock/masters -#zookeeper.dolphinscheduler.lock.workers=/dolphinscheduler/lock/workers - -#dolphinscheduler failover directory -#zookeeper.dolphinscheduler.lock.failover.masters=/dolphinscheduler/lock/failover/masters -#zookeeper.dolphinscheduler.lock.failover.workers=/dolphinscheduler/lock/failover/workers -#zookeeper.dolphinscheduler.lock.failover.startup.masters=/dolphinscheduler/lock/failover/startup-masters - -#dolphinscheduler failover directory -zookeeper.session.timeout=300 -zookeeper.connection.timeout=300 -zookeeper.retry.base.sleep=100 -zookeeper.retry.max.sleep=30000 -zookeeper.retry.maxtime=5 \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PropertyUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PropertyUtils.java index e92491e096..cdd481a5d7 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PropertyUtils.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PropertyUtils.java @@ -49,7 +49,7 @@ public class PropertyUtils { * init */ private void init(){ - String[] propertyFiles = new String[]{Constants.DAO_PROPERTIES_PATH}; + String[] propertyFiles = new String[]{Constants.APPLICATION_PROPERTIES}; for (String fileName : propertyFiles) { InputStream fis = null; try { diff --git a/dolphinscheduler-dao/src/main/resources/application-dao.properties b/dolphinscheduler-dao/src/main/resources/application.properties similarity index 100% rename from dolphinscheduler-dao/src/main/resources/application-dao.properties rename to dolphinscheduler-dao/src/main/resources/application.properties diff --git a/dolphinscheduler-server/src/main/resources/application-master.properties b/dolphinscheduler-server/src/main/resources/application-master.properties deleted file mode 100644 index 49d28e810f..0000000000 --- a/dolphinscheduler-server/src/main/resources/application-master.properties +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -logging.config=classpath:master_logback.xml \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/resources/application-worker.properties b/dolphinscheduler-server/src/main/resources/application-worker.properties deleted file mode 100644 index be4319d204..0000000000 --- a/dolphinscheduler-server/src/main/resources/application-worker.properties +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -logging.config=classpath:worker_logback.xml diff --git a/install.sh b/install.sh index b73ca09079..85e470051f 100644 --- a/install.sh +++ b/install.sh @@ -218,30 +218,6 @@ keytabPath="$installPath/conf/hdfs.headless.keytab" # zk root directory zkRoot="/dolphinscheduler" -# used to record the zk directory of the hanging machine -zkDeadServers="$zkRoot/dead-servers" - -# masters directory -zkMasters="$zkRoot/masters" - -# workers directory -zkWorkers="$zkRoot/workers" - -# zk master distributed lock -mastersLock="$zkRoot/lock/masters" - -# zk worker distributed lock -workersLock="$zkRoot/lock/workers" - -# zk master fault-tolerant distributed lock -mastersFailover="$zkRoot/lock/failover/masters" - -# zk worker fault-tolerant distributed lock -workersFailover="$zkRoot/lock/failover/workers" - -# zk master start fault tolerant distributed lock -mastersStartupFailover="$zkRoot/lock/failover/startup-masters" - # zk session timeout zkSessionTimeout="300" @@ -321,11 +297,10 @@ apiMaxHttpPostSize="5000000" # 1,replace file echo "1,replace file" if [ $dbtype == "mysql" ];then - sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application-dao.properties - sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application-dao.properties - sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application-dao.properties - sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=com.mysql.jdbc.Driver#g" conf/application-dao.properties - + sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application.properties + sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application.properties + sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application.properties + sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=com.mysql.jdbc.Driver#g" conf/application.properties sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/quartz.properties sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${username}#g" conf/quartz.properties @@ -335,10 +310,10 @@ if [ $dbtype == "mysql" ];then fi if [ $dbtype == "postgresql" ];then - sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application-dao.properties - sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application-dao.properties - sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application-dao.properties - sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=org.postgresql.Driver#g" conf/application-dao.properties + sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application.properties + sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application.properties + sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application.properties + sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=org.postgresql.Driver#g" conf/application.properties sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/quartz.properties sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${username}#g" conf/quartz.properties @@ -349,46 +324,34 @@ fi -sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g" conf/common/hadoop/hadoop.properties -sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g" conf/common/hadoop/hadoop.properties -sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g" conf/common/hadoop/hadoop.properties -sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g" conf/common/hadoop/hadoop.properties -sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common/hadoop/hadoop.properties -sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common/hadoop/hadoop.properties - - -sed -i ${txt} "s#data.basedir.path.*#data.basedir.path=${programPath}#g" conf/common/common.properties -sed -i ${txt} "s#data.download.basedir.path.*#data.download.basedir.path=${downloadPath}#g" conf/common/common.properties -sed -i ${txt} "s#process.exec.basepath.*#process.exec.basepath=${execPath}#g" conf/common/common.properties -sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common/common.properties -sed -i ${txt} "s#data.store2hdfs.basepath.*#data.store2hdfs.basepath=${hdfsPath}#g" conf/common/common.properties -sed -i ${txt} "s#res.upload.startup.type.*#res.upload.startup.type=${resUploadStartupType}#g" conf/common/common.properties -sed -i ${txt} "s#dolphinscheduler.env.path.*#dolphinscheduler.env.path=${shellEnvPath}#g" conf/common/common.properties -sed -i ${txt} "s#resource.view.suffixs.*#resource.view.suffixs=${resSuffixs}#g" conf/common/common.properties -sed -i ${txt} "s#development.state.*#development.state=${devState}#g" conf/common/common.properties -sed -i ${txt} "s#hadoop.security.authentication.startup.state.*#hadoop.security.authentication.startup.state=${kerberosStartUp}#g" conf/common/common.properties -sed -i ${txt} "s#java.security.krb5.conf.path.*#java.security.krb5.conf.path=${krb5ConfPath}#g" conf/common/common.properties -sed -i ${txt} "s#login.user.keytab.username.*#login.user.keytab.username=${keytabUserName}#g" conf/common/common.properties -sed -i ${txt} "s#login.user.keytab.path.*#login.user.keytab.path=${keytabPath}#g" conf/common/common.properties - -sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.dead.servers.*#zookeeper.dolphinscheduler.dead.servers=${zkDeadServers}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.masters.*#zookeeper.dolphinscheduler.masters=${zkMasters}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.workers.*#zookeeper.dolphinscheduler.workers=${zkWorkers}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.masters.*#zookeeper.dolphinscheduler.lock.masters=${mastersLock}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.workers.*#zookeeper.dolphinscheduler.lock.workers=${workersLock}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.failover.masters.*#zookeeper.dolphinscheduler.lock.failover.masters=${mastersFailover}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.failover.workers.*#zookeeper.dolphinscheduler.lock.failover.workers=${workersFailover}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.failover.startup.masters.*#zookeeper.dolphinscheduler.lock.failover.startup.masters=${mastersStartupFailover}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.session.timeout.*#zookeeper.session.timeout=${zkSessionTimeout}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.connection.timeout.*#zookeeper.connection.timeout=${zkConnectionTimeout}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.retry.sleep.*#zookeeper.retry.sleep=${zkRetrySleep}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.retry.maxtime.*#zookeeper.retry.maxtime=${zkRetryMaxtime}#g" conf/zookeeper.properties - -sed -i ${txt} "s#server.port.*#server.port=${masterPort}#g" conf/application-master.properties -sed -i ${txt} "s#server.port.*#server.port=${workerPort}#g" conf/application-worker.properties - +sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g" conf/common.properties +sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g" conf/common.properties +sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g" conf/common.properties +sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g" conf/common.properties +sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common.properties +sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common.properties + + +sed -i ${txt} "s#data.basedir.path.*#data.basedir.path=${programPath}#g" conf/common.properties +sed -i ${txt} "s#data.download.basedir.path.*#data.download.basedir.path=${downloadPath}#g" conf/common.properties +sed -i ${txt} "s#process.exec.basepath.*#process.exec.basepath=${execPath}#g" conf/common.properties +sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common.properties +sed -i ${txt} "s#data.store2hdfs.basepath.*#data.store2hdfs.basepath=${hdfsPath}#g" conf/common.properties +sed -i ${txt} "s#res.upload.startup.type.*#res.upload.startup.type=${resUploadStartupType}#g" conf/common.properties +sed -i ${txt} "s#dolphinscheduler.env.path.*#dolphinscheduler.env.path=${shellEnvPath}#g" conf/common.properties +sed -i ${txt} "s#resource.view.suffixs.*#resource.view.suffixs=${resSuffixs}#g" conf/common.properties +sed -i ${txt} "s#development.state.*#development.state=${devState}#g" conf/common.properties +sed -i ${txt} "s#hadoop.security.authentication.startup.state.*#hadoop.security.authentication.startup.state=${kerberosStartUp}#g" conf/common.properties +sed -i ${txt} "s#java.security.krb5.conf.path.*#java.security.krb5.conf.path=${krb5ConfPath}#g" conf/common.properties +sed -i ${txt} "s#login.user.keytab.username.*#login.user.keytab.username=${keytabUserName}#g" conf/common.properties +sed -i ${txt} "s#login.user.keytab.path.*#login.user.keytab.path=${keytabPath}#g" conf/common.properties + +sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/common.properties +sed -i ${txt} "s#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/common.properties +sed -i ${txt} "s#zookeeper.session.timeout.*#zookeeper.session.timeout=${zkSessionTimeout}#g" conf/common.properties +sed -i ${txt} "s#zookeeper.connection.timeout.*#zookeeper.connection.timeout=${zkConnectionTimeout}#g" conf/common.properties +sed -i ${txt} "s#zookeeper.retry.sleep.*#zookeeper.retry.sleep=${zkRetrySleep}#g" conf/common.properties +sed -i ${txt} "s#zookeeper.retry.maxtime.*#zookeeper.retry.maxtime=${zkRetryMaxtime}#g" conf/common.properties sed -i ${txt} "s#server.port.*#server.port=${apiServerPort}#g" conf/application-api.properties sed -i ${txt} "s#server.servlet.session.timeout.*#server.servlet.session.timeout=${apiServerSessionTimeout}#g" conf/application-api.properties diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index 292f2eca39..b3310c384f 100644 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -57,13 +57,13 @@ pid=$DOLPHINSCHEDULER_LOG_DIR/dolphinscheduler-$command.pid cd $DOLPHINSCHEDULER_HOME if [ "$command" = "api-server" ]; then - LOG_FILE="-Dspring.profiles.active=api" + LOG_FILE="-Dlogging.config=classpath:apiserver_logback.xml -Dspring.profiles.active=api" CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer elif [ "$command" = "master-server" ]; then - LOG_FILE="-Dspring.profiles.active=master -Ddruid.mysql.usePingMethod=false" + LOG_FILE="-Dlogging.config=classpath:master_logback.xml -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.server.master.MasterServer elif [ "$command" = "worker-server" ]; then - LOG_FILE="-Dspring.profiles.active=worker -Ddruid.mysql.usePingMethod=false" + LOG_FILE="-Dlogging.config=classpath:worker_logback.xml -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer elif [ "$command" = "alert-server" ]; then LOG_FILE="-Dlogback.configurationFile=conf/alert_logback.xml" @@ -71,7 +71,7 @@ elif [ "$command" = "alert-server" ]; then elif [ "$command" = "logger-server" ]; then CLASS=org.apache.dolphinscheduler.server.rpc.LoggerServer elif [ "$command" = "combined-server" ]; then - LOG_FILE="-Dspring.profiles.active=combined" + LOG_FILE="-Dlogging.config=classpath:combined_logback.xml -Dspring.profiles.active=api -Dserver.is-combined-server=true" CLASS=org.apache.dolphinscheduler.api.CombinedApplicationServer else echo "Error: No command named \`$command' was found." From 96d49a09501874b9ed8de28638ef48231f8fdd29 Mon Sep 17 00:00:00 2001 From: lilin Date: Mon, 30 Dec 2019 18:26:10 +0800 Subject: [PATCH 53/53] remove delete check exist --- .../dolphinscheduler/api/service/UdfFuncService.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java index df439ce38f..20324928fb 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java @@ -303,12 +303,7 @@ public class UdfFuncService extends BaseService{ @Transactional(rollbackFor = Exception.class) public Result delete(int id) { Result result = new Result(); - //check exist - UdfFunc udfFunc = udfFuncMapper.selectById(id); - if (udfFunc == null) { - putMsg(result, Status.UDF_FUNCTION_NOT_EXIST); - return result; - } + udfFuncMapper.deleteById(id); udfUserMapper.deleteByUdfFuncId(id); putMsg(result, Status.SUCCESS);