From 6c90575a39721b8342aff72bb725d646654c1154 Mon Sep 17 00:00:00 2001 From: Yeleights Date: Thu, 13 Feb 2020 15:39:30 +0800 Subject: [PATCH 1/4] add process null check Remove duplicate code in processDefinitionService add processDefinitionService UT --- .../api/service/BaseDAGService.java | 40 +--- .../api/service/ProcessDefinitionService.java | 87 +++------ .../service/ProcessDefinitionServiceTest.java | 177 ++++++++++++++++-- .../dolphinscheduler/dao/utils/DagHelper.java | 35 +++- .../dao/utils/DagHelperTest.java | 19 +- 5 files changed, 239 insertions(+), 119 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseDAGService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseDAGService.java index af66591bed..de2c8d9cea 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseDAGService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseDAGService.java @@ -20,12 +20,11 @@ import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.process.ProcessDag; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.ProcessData; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.utils.DagHelper; -import java.util.ArrayList; import java.util.List; /** @@ -48,41 +47,8 @@ public class BaseDAGService extends BaseService{ List taskNodeList = processData.getTasks(); - List taskNodeRelations = new ArrayList<>(); + ProcessDag processDag = DagHelper.getProcessDag(taskNodeList); - //Traversing node information and building relationships - for (TaskNode taskNode : taskNodeList) { - String preTasks = taskNode.getPreTasks(); - List preTasksList = JSONUtils.toList(preTasks, String.class); - - //if previous tasks not empty - if (preTasksList != null) { - for (String depNode : preTasksList) { - taskNodeRelations.add(new TaskNodeRelation(depNode, taskNode.getName())); - } - } - } - - ProcessDag processDag = new ProcessDag(); - processDag.setEdges(taskNodeRelations); - processDag.setNodes(taskNodeList); - - - // generate detail Dag, to be executed - DAG dag = new DAG<>(); - - if (CollectionUtils.isNotEmpty(processDag.getNodes())) { - for (TaskNode node : processDag.getNodes()) { - dag.addNode(node.getName(), node); - } - } - - if (CollectionUtils.isNotEmpty(processDag.getEdges())) { - for (TaskNodeRelation edge : processDag.getEdges()) { - dag.addEdge(edge.getStartNode(), edge.getEndNode()); - } - } - - return dag; + return DagHelper.buildDagGraph(processDag); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index e9cfe341dd..38755d6727 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -46,6 +46,7 @@ import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.ProcessDao; import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.dao.mapper.*; +import org.apache.dolphinscheduler.dao.utils.DagHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -948,11 +949,16 @@ public class ProcessDefinitionService extends BaseDAGService { return result; } - String processDefinitionJson = processDefinition.getProcessDefinitionJson(); - ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); + //process data check + if (null == processData) { + logger.error("process data is null"); + putMsg(result,Status.DATA_IS_NOT_VALID, processDefinitionJson); + return result; + } + List taskNodeList = (processData.getTasks() == null) ? new ArrayList<>() : processData.getTasks(); result.put(Constants.DATA_LIST, taskNodeList); @@ -974,14 +980,13 @@ public class ProcessDefinitionService extends BaseDAGService { Map> taskNodeMap = new HashMap<>(); String[] idList = defineIdList.split(","); - List definitionIdList = Arrays.asList(idList); List idIntList = new ArrayList<>(); - for(String definitionId : definitionIdList) { + for(String definitionId : idList) { idIntList.add(Integer.parseInt(definitionId)); } Integer[] idArray = idIntList.toArray(new Integer[idIntList.size()]); List processDefinitionList = processDefineMapper.queryDefinitionListByIdList(idArray); - if (processDefinitionList == null || processDefinitionList.size() ==0) { + if (CollectionUtils.isEmpty(processDefinitionList)) { logger.info("process definition not exists"); putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, defineIdList); return result; @@ -1031,9 +1036,10 @@ public class ProcessDefinitionService extends BaseDAGService { Map result = new HashMap<>(); ProcessDefinition processDefinition = processDefineMapper.selectById(processId); - if (processDefinition == null) { + if (null == processDefinition) { logger.info("process define not exists"); - throw new RuntimeException("process define not exists"); + putMsg(result,Status.PROCESS_DEFINE_NOT_EXIST, processDefinition); + return result; } DAG dag = genDagGraph(processDefinition); /** @@ -1121,10 +1127,10 @@ public class ProcessDefinitionService extends BaseDAGService { pTreeViewDto.getChildren().add(treeViewDto); } postNodeList = dag.getSubsequentNodes(nodeName); - if (postNodeList != null && postNodeList.size() > 0) { + if (CollectionUtils.isNotEmpty(postNodeList)) { for (String nextNodeName : postNodeList) { List treeViewDtoList = waitingRunningNodeMap.get(nextNodeName); - if (treeViewDtoList != null && treeViewDtoList.size() > 0) { + if (CollectionUtils.isNotEmpty(treeViewDtoList)) { treeViewDtoList.add(treeViewDto); waitingRunningNodeMap.put(nextNodeName, treeViewDtoList); } else { @@ -1136,7 +1142,6 @@ public class ProcessDefinitionService extends BaseDAGService { } runningNodeMap.remove(nodeName); } - if (waitingRunningNodeMap == null || waitingRunningNodeMap.size() == 0) { break; } else { @@ -1161,68 +1166,22 @@ public class ProcessDefinitionService extends BaseDAGService { private DAG genDagGraph(ProcessDefinition processDefinition) throws Exception { String processDefinitionJson = processDefinition.getProcessDefinitionJson(); - ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); - List taskNodeList = processData.getTasks(); + //check process data + if (null != processData) { + List taskNodeList = processData.getTasks(); + processDefinition.setGlobalParamList(processData.getGlobalParams()); + ProcessDag processDag = DagHelper.getProcessDag(taskNodeList); - processDefinition.setGlobalParamList(processData.getGlobalParams()); - - - List taskNodeRelations = new ArrayList<>(); - - // Traverse node information and build relationships - for (TaskNode taskNode : taskNodeList) { - String preTasks = taskNode.getPreTasks(); - List preTasksList = JSONUtils.toList(preTasks, String.class); - - // If the dependency is not empty - if (preTasksList != null) { - for (String depNode : preTasksList) { - taskNodeRelations.add(new TaskNodeRelation(depNode, taskNode.getName())); - } - } + // Generate concrete Dag to be executed + return DagHelper.buildDagGraph(processDag); } - ProcessDag processDag = new ProcessDag(); - processDag.setEdges(taskNodeRelations); - processDag.setNodes(taskNodeList); - - - // Generate concrete Dag to be executed - return genDagGraph(processDag); - - + return new DAG<>(); } - /** - * Generate the DAG of process - * - * @return DAG - */ - private DAG genDagGraph(ProcessDag processDag) { - DAG dag = new DAG<>(); - - /** - * Add the ndoes - */ - if (CollectionUtils.isNotEmpty(processDag.getNodes())) { - for (TaskNode node : processDag.getNodes()) { - dag.addNode(node.getName(), node); - } - } - - /** - * Add the edges - */ - if (CollectionUtils.isNotEmpty(processDag.getEdges())) { - for (TaskNodeRelation edge : processDag.getEdges()) { - dag.addEdge(edge.getStartNode(), edge.getEndNode()); - } - } - return dag; - } /** diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java index 51b440b9ca..290f6ae015 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java @@ -16,10 +16,8 @@ */ package org.apache.dolphinscheduler.api.service; -import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; -import com.baomidou.mybatisplus.core.metadata.IPage; import org.apache.dolphinscheduler.api.ApiApplicationServer; import org.apache.dolphinscheduler.api.dto.ProcessMeta; import org.apache.dolphinscheduler.api.enums.Status; @@ -82,6 +80,12 @@ public class ProcessDefinitionServiceTest { @Mock private ProcessDao processDao; + @Mock + private ProcessInstanceMapper processInstanceMapper; + + @Mock + private TaskInstanceMapper taskInstanceMapper; + private String sqlDependentJson = "{\"globalParams\":[]," + "\"tasks\":[{\"type\":\"SQL\",\"id\":\"tasks-27297\",\"name\":\"sql\"," + "\"params\":{\"type\":\"MYSQL\",\"datasource\":1,\"sql\":\"select * from test\"," + @@ -99,6 +103,12 @@ public class ProcessDefinitionServiceTest { "\"timeout\":{\"strategy\":\"\",\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + "\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}"; + private String shellJson = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-9527\",\"name\":\"shell-1\"," + + "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"#!/bin/bash\\necho \\\"shell-1\\\"\"}," + + "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + + "\"timeout\":{\"strategy\":\"\",\"interval\":1,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + + "\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}"; + @Test public void testQueryProccessDefinitionList() { String projectName = "project_test1"; @@ -149,7 +159,7 @@ public class ProcessDefinitionServiceTest { } @Test - public void testQueryProccessDefinitionById() { + public void testQueryProcessDefinitionById() { String projectName = "project_test1"; Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName)); @@ -255,7 +265,7 @@ public class ProcessDefinitionServiceTest { "project_test1", 46); Assert.assertEquals(Status.DELETE_PROCESS_DEFINE_BY_ID_ERROR, deleteFail.get(Constants.STATUS)); - //delte success + //delete success Mockito.when(processDefineMapper.deleteById(46)).thenReturn(1); Map deleteSuccess = processDefinitionService.deleteProcessDefinitionById(loginUser, "project_test1", 46); @@ -304,6 +314,155 @@ public class ProcessDefinitionServiceTest { Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, failRes.get(Constants.STATUS)); } + @Test + public void testVerifyProcessDefinitionName() { + String projectName = "project_test1"; + Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName)); + + Project project = getProject(projectName); + User loginUser = new User(); + loginUser.setId(-1); + loginUser.setUserType(UserType.GENERAL_USER); + + //project check auth fail + Map result = new HashMap<>(5); + putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); + Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result); + Map map = processDefinitionService.verifyProccessDefinitionName(loginUser, + "project_test1", "test_pdf"); + Assert.assertEquals(Status.PROJECT_NOT_FOUNT, map.get(Constants.STATUS)); + + //project check auth success, process not exist + putMsg(result, Status.SUCCESS, projectName); + Mockito.when(processDefineMapper.queryByDefineName(project.getId(),"test_pdf")).thenReturn(null); + Map processNotExistRes = processDefinitionService.verifyProccessDefinitionName(loginUser, + "project_test1", "test_pdf"); + Assert.assertEquals(Status.SUCCESS, processNotExistRes.get(Constants.STATUS)); + + //process exist + Mockito.when(processDefineMapper.queryByDefineName(project.getId(),"test_pdf")).thenReturn(getProcessDefinition()); + Map processExistRes = processDefinitionService.verifyProccessDefinitionName(loginUser, + "project_test1", "test_pdf"); + Assert.assertEquals(Status.PROCESS_INSTANCE_EXIST, processExistRes.get(Constants.STATUS)); + } + + @Test + public void testCheckProcessNodeList() { + + Map dataNotValidRes = processDefinitionService.checkProcessNodeList(null, ""); + Assert.assertEquals(Status.DATA_IS_NOT_VALID, dataNotValidRes.get(Constants.STATUS)); + + //task not empty + String processDefinitionJson = shellJson; + ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); + assert processData != null; + Map taskEmptyRes = processDefinitionService.checkProcessNodeList(processData, processDefinitionJson); + Assert.assertEquals(Status.SUCCESS, taskEmptyRes.get(Constants.STATUS)); + + //task empty + processData.setTasks(null); + Map taskNotEmptyRes = processDefinitionService.checkProcessNodeList(processData, processDefinitionJson); + Assert.assertEquals(Status.DATA_IS_NULL, taskNotEmptyRes.get(Constants.STATUS)); + + //json abnormal + String abnormalJson = processDefinitionJson.replaceAll("SHELL",""); + processData = JSONUtils.parseObject(abnormalJson, ProcessData.class); + Map abnormalTaskRes = processDefinitionService.checkProcessNodeList(processData, abnormalJson); + Assert.assertEquals(Status.PROCESS_NODE_S_PARAMETER_INVALID, abnormalTaskRes.get(Constants.STATUS)); + } + + @Test + public void testGetTaskNodeListByDefinitionId() throws Exception { + //process definition not exist + Mockito.when(processDefineMapper.selectById(46)).thenReturn(null); + Map processDefinitionNullRes = processDefinitionService.getTaskNodeListByDefinitionId(46); + Assert.assertEquals(Status.PROCESS_DEFINE_NOT_EXIST, processDefinitionNullRes.get(Constants.STATUS)); + + //process data null + ProcessDefinition processDefinition = getProcessDefinition(); + Mockito.when(processDefineMapper.selectById(46)).thenReturn(processDefinition); + Map successRes = processDefinitionService.getTaskNodeListByDefinitionId(46); + Assert.assertEquals(Status.DATA_IS_NOT_VALID, successRes.get(Constants.STATUS)); + + //success + processDefinition.setProcessDefinitionJson(shellJson); + Mockito.when(processDefineMapper.selectById(46)).thenReturn(processDefinition); + Map dataNotValidRes = processDefinitionService.getTaskNodeListByDefinitionId(46); + Assert.assertEquals(Status.SUCCESS, dataNotValidRes.get(Constants.STATUS)); + } + + @Test + public void testGetTaskNodeListByDefinitionIdList() throws Exception { + //process definition not exist + String defineIdList = "46"; + Integer[] idArray = {46}; + Mockito.when(processDefineMapper.queryDefinitionListByIdList(idArray)).thenReturn(null); + Map processNotExistRes = processDefinitionService.getTaskNodeListByDefinitionIdList(defineIdList); + Assert.assertEquals(Status.PROCESS_DEFINE_NOT_EXIST, processNotExistRes.get(Constants.STATUS)); + + //process definition exist + ProcessDefinition processDefinition = getProcessDefinition(); + processDefinition.setProcessDefinitionJson(shellJson); + List processDefinitionList = new ArrayList<>(); + processDefinitionList.add(processDefinition); + Mockito.when(processDefineMapper.queryDefinitionListByIdList(idArray)).thenReturn(processDefinitionList); + Map successRes = processDefinitionService.getTaskNodeListByDefinitionIdList(defineIdList); + Assert.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS)); + } + + @Test + public void testQueryProccessDefinitionAllByProjectId() { + int projectId = 1; + ProcessDefinition processDefinition = getProcessDefinition(); + processDefinition.setProcessDefinitionJson(shellJson); + List processDefinitionList = new ArrayList<>(); + processDefinitionList.add(processDefinition); + Mockito.when(processDefineMapper.queryAllDefinitionList(projectId)).thenReturn(processDefinitionList); + Map successRes = processDefinitionService.queryProccessDefinitionAllByProjectId(projectId); + Assert.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS)); + } + + @Test + public void testViewTree() throws Exception { + //process definition not exist + ProcessDefinition processDefinition = getProcessDefinition(); + processDefinition.setProcessDefinitionJson(shellJson); + Mockito.when(processDefineMapper.selectById(46)).thenReturn(null); + Map processDefinitionNullRes = processDefinitionService.viewTree(46, 10); + Assert.assertEquals(Status.PROCESS_DEFINE_NOT_EXIST, processDefinitionNullRes.get(Constants.STATUS)); + + List processInstanceList = new ArrayList<>(); + ProcessInstance processInstance = new ProcessInstance(); + processInstance.setId(1); + processInstance.setName("test_instance"); + processInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + processInstance.setHost("192.168.xx.xx"); + processInstance.setStartTime(new Date()); + processInstance.setEndTime(new Date()); + processInstanceList.add(processInstance); + + TaskInstance taskInstance = new TaskInstance(); + taskInstance.setStartTime(new Date()); + taskInstance.setEndTime(new Date()); + taskInstance.setTaskType("SHELL"); + taskInstance.setId(1); + taskInstance.setName("test_task_instance"); + taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + taskInstance.setHost("192.168.xx.xx"); + + //task instance not exist + Mockito.when(processDefineMapper.selectById(46)).thenReturn(processDefinition); + Mockito.when(processInstanceMapper.queryByProcessDefineId(46, 10)).thenReturn(processInstanceList); + Mockito.when(taskInstanceMapper.queryByInstanceIdAndName(processInstance.getId(), "shell-1")).thenReturn(null); + Map taskNullRes = processDefinitionService.viewTree(46, 10); + Assert.assertEquals(Status.SUCCESS, taskNullRes.get(Constants.STATUS)); + + //task instance exist + Mockito.when(taskInstanceMapper.queryByInstanceIdAndName(processInstance.getId(), "shell-1")).thenReturn(taskInstance); + Map taskNotNuLLRes = processDefinitionService.viewTree(46, 10); + Assert.assertEquals(Status.SUCCESS, taskNotNuLLRes.get(Constants.STATUS)); + } + /** * add datasource param and dependent when export process * @throws JSONException @@ -334,13 +493,9 @@ public class ProcessDefinitionServiceTest { @Test public void testAddExportTaskNodeSpecialParam() throws JSONException { - String shellJson = "{\"globalParams\":[],\"tasks\":[{\"id\":\"tasks-9527\",\"name\":\"shell-1\"," + - "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"#!/bin/bash\\necho \\\"shell-1\\\"\"}," + - "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + - "\"timeout\":{\"strategy\":\"\",\"interval\":1,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + - "\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}"; + String shellData = shellJson; - String resultStr = processDefinitionService.addExportTaskNodeSpecialParam(shellJson); + String resultStr = processDefinitionService.addExportTaskNodeSpecialParam(shellData); JSONAssert.assertEquals(shellJson, resultStr, false); } @@ -610,7 +765,7 @@ public class ProcessDefinitionServiceTest { private ProcessDefinition getProcessDefinition(){ ProcessDefinition processDefinition = new ProcessDefinition(); processDefinition.setId(46); - processDefinition.setName("testProject"); + processDefinition.setName("test_pdf"); processDefinition.setProjectId(2); processDefinition.setTenantId(1); processDefinition.setDescription(""); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java index ac38ddd2e8..7a4dc655f7 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java @@ -319,18 +319,14 @@ public class DagHelper { DAG dag = new DAG<>(); - /** - * add vertex - */ + //add vertex if (CollectionUtils.isNotEmpty(processDag.getNodes())){ for (TaskNode node : processDag.getNodes()){ dag.addNode(node.getName(),node); } } - /** - * add edge - */ + //add edge if (CollectionUtils.isNotEmpty(processDag.getEdges())){ for (TaskNodeRelation edge : processDag.getEdges()){ dag.addEdge(edge.getStartNode(),edge.getEndNode()); @@ -338,4 +334,31 @@ public class DagHelper { } return dag; } + + /** + * get process dag + * @param taskNodeList task node list + * @return Process dag + */ + public static ProcessDag getProcessDag(List taskNodeList) { + List taskNodeRelations = new ArrayList<>(); + + // Traverse node information and build relationships + for (TaskNode taskNode : taskNodeList) { + String preTasks = taskNode.getPreTasks(); + List preTasksList = JSONUtils.toList(preTasks, String.class); + + // If the dependency is not empty + if (preTasksList != null) { + for (String depNode : preTasksList) { + taskNodeRelations.add(new TaskNodeRelation(depNode, taskNode.getName())); + } + } + } + + ProcessDag processDag = new ProcessDag(); + processDag.setEdges(taskNodeRelations); + processDag.setNodes(taskNodeList); + return processDag; + } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/DagHelperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/DagHelperTest.java index a1e3f819e3..95c7d2f086 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/DagHelperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/DagHelperTest.java @@ -24,6 +24,8 @@ import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.process.ProcessDag; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.dao.entity.ProcessData; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.junit.Assert; import org.junit.Test; @@ -37,7 +39,6 @@ import java.util.Map; * dag helper test */ public class DagHelperTest { - /** * test task node can submit * @throws JsonProcessingException if error throws JsonProcessingException @@ -131,4 +132,20 @@ public class DagHelperTest { return DagHelper.buildDagGraph(processDag); } + @Test + public void testBuildDagGraph() { + String shellJson = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-9527\",\"name\":\"shell-1\"," + + "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"#!/bin/bash\\necho \\\"shell-1\\\"\"}," + + "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + + "\"timeout\":{\"strategy\":\"\",\"interval\":1,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + + "\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}"; + + ProcessData processData = JSONUtils.parseObject(shellJson, ProcessData.class); + assert processData != null; + List taskNodeList = processData.getTasks(); + ProcessDag processDag = DagHelper.getProcessDag(taskNodeList); + DAG dag = DagHelper.buildDagGraph(processDag); + Assert.assertNotNull(dag); + } + } From 77023aee1b999a18afbb073484d0ed0bea8e9806 Mon Sep 17 00:00:00 2001 From: Yeleights Date: Thu, 13 Feb 2020 16:18:26 +0800 Subject: [PATCH 2/4] add DagHelperTest to pom --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 53184e7412..11facc1e5a 100644 --- a/pom.xml +++ b/pom.xml @@ -715,6 +715,7 @@ **/dao/mapper/AlertMapperTest.java **/dao/mapper/CommandMapperTest.java **/dao/cron/CronUtilsTest.java + **/dao/utils/DagHelperTest.java **/alert/template/AlertTemplateFactoryTest.java **/alert/template/impl/DefaultHTMLTemplateTest.java **/server/worker/task/datax/DataxTaskTest.java From 99ac7398518ba5f0970b192cecd40024d8006075 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Sat, 15 Feb 2020 18:55:16 +0800 Subject: [PATCH 3/4] rename vaild to valid (#1961) --- .../dolphinscheduler/api/service/ExecutorService.java | 6 +++--- .../org/apache/dolphinscheduler/api/utils/CheckUtils.java | 2 +- .../org/apache/dolphinscheduler/common/utils/JSONUtils.java | 2 +- .../apache/dolphinscheduler/common/utils/JSONUtilsTest.java | 6 +++--- .../dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index 6edd48d499..0389890691 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -117,7 +117,7 @@ public class ExecutorService extends BaseService{ } if (!checkTenantSuitable(processDefinition)){ - logger.error("there is not any vaild tenant for the process definition: id:{},name:{}, ", + logger.error("there is not any valid tenant for the process definition: id:{},name:{}, ", processDefinition.getId(), processDefinition.getName()); putMsg(result, Status.TENANT_NOT_SUITABLE); return result; @@ -206,7 +206,7 @@ public class ExecutorService extends BaseService{ return checkResult; } if (!checkTenantSuitable(processDefinition)){ - logger.error("there is not any vaild tenant for the process definition: id:{},name:{}, ", + logger.error("there is not any valid tenant for the process definition: id:{},name:{}, ", processDefinition.getId(), processDefinition.getName()); putMsg(result, Status.TENANT_NOT_SUITABLE); } @@ -539,7 +539,7 @@ public class ExecutorService extends BaseService{ } } }else{ - logger.error("there is not vaild schedule date for the process definition: id:{},date:{}", + logger.error("there is not valid schedule date for the process definition: id:{},date:{}", processDefineId, schedule); } }else{ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java index c5c702404d..a888712511 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java @@ -86,7 +86,7 @@ public class CheckUtils { * @return true if other parameters are valid, otherwise return false */ public static boolean checkOtherParams(String otherParams) { - return StringUtils.isNotEmpty(otherParams) && !JSONUtils.checkJsonVaild(otherParams); + return StringUtils.isNotEmpty(otherParams) && !JSONUtils.checkJsonValid(otherParams); } /** diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java index 9e9e4f6546..ec523b1ff2 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java @@ -126,7 +126,7 @@ public class JSONUtils { * @param json json * @return true if valid */ - public static boolean checkJsonVaild(String json) { + public static boolean checkJsonValid(String json) { if (StringUtils.isEmpty(json)) { return false; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java index 799874ad71..bd924e4852 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java @@ -117,9 +117,9 @@ public class JSONUtilsTest { } @Test - public void testCheckJsonVaild() { - Assert.assertTrue(JSONUtils.checkJsonVaild("3")); - Assert.assertFalse(JSONUtils.checkJsonVaild("")); + public void testCheckJsonValid() { + Assert.assertTrue(JSONUtils.checkJsonValid("3")); + Assert.assertFalse(JSONUtils.checkJsonValid("")); } @Test diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java index c60cc3a655..16ba4b06c4 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java @@ -121,7 +121,7 @@ public class TaskInstanceMapperTest { } /** - * test find vaild task list by process instance id + * test find valid task list by process instance id */ @Test public void testFindValidTaskListByProcessId() { From c6f252451bd68fe54f4da95eeeb0c76fbf3c6c0b Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Sat, 15 Feb 2020 19:58:45 +0800 Subject: [PATCH 4/4] =?UTF-8?q?1=EF=BC=8Cquartz.properties=20add=20conf=20?= =?UTF-8?q?category=202=EF=BC=8Cdolphinscheduler-daemon.sh=20modify=20(#19?= =?UTF-8?q?60)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 1，remove dolphinscheduler-rpc module 2，add dolphinscheduler-remote module 3，add dolphinscheduler-service module 4，refactor LoggerServer module (#1925) * 1，remove dolphinscheduler-rpc module 2，add dolphinscheduler-remote module 3，add dolphinscheduler-service module 4，refactor LoggerServer module * ProcessUtils modify * Refactor architecture (#1926) * move version to parent pom * move version properties to parent pom for easy management * remove freemarker dependency * delete CombinedApplicationServer * #1871 correct spelling * #1873 some updates for TaskQueueZkImpl * #1875 remove unused properties in pom * #1878 1. remove tomcat dependency 2. remove combined_logback.xml in api module 3. format pom.xml for not aligning * #1885 fix api server startup failure 1. add jsp-2.1 dependency 2. remove jasper-runtime dependency * add stringutils ut (#1921) * add stringutils ut * Newfeature for #1675. (#1908) Continue to finish the rest works, add the cache feature for dependence,mr,python,sub_process,procedure and shell. * Add modify user name for process definition (#1919) * class overrides equals() and should therefore also override hashCode() * #1862 add modify user in process difinition list * #1862 add pg-1.2.2 ddl.sql * modify ScriptRunnerTest * add updateProessDifinition UT * modify updateProcessDifinition UT * modify updateProcessDifinition UT * modify mysql 1.2.2 ddl.sql&dml.sql * add scope test to mysql in pom * modify pg-1.2.2 ddl.sql * refactor module * updates Co-authored-by: khadgarmage Co-authored-by: zhukai Co-authored-by: Yelli * dolphinscheduler-common remove spring (#1931) * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * SpringApplicationContext class title add license (#1932) * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * add license (#1934) * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * Refactor architecture (#1936) * move datasource classes to dao module * fix send4LetterWord bug * LoggerServiceTest remove ProcessDao (#1944) * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * dolphinscheduler-common remove spring * LoggerServiceTest remove ProcessDao * exclude jasper-compiler in case of runtime conflict (#1938) * move datasource classes to dao module * fix send4LetterWord bug * exclude jasper-compiler in case of runtime conflict * DataAnaylysisServiceTest and ProcessDefinitionService modify * remote module add comment * OSUtilsTest modify * add finally block to close channel (#1951) * move datasource classes to dao module * fix send4LetterWord bug * exclude jasper-compiler in case of runtime conflict * add finally block to close channel * 1，quartz.properties add conf category 2，dolphinscheduler-daemon.sh modify * dolphinscheduler-binary.xml modify Co-authored-by: Tboy Co-authored-by: khadgarmage Co-authored-by: zhukai Co-authored-by: Yelli --- .../src/main/assembly/dolphinscheduler-binary.xml | 15 +++++++++++++++ script/dolphinscheduler-daemon.sh | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml index b4326c6795..28bbb361cd 100644 --- a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml +++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml @@ -112,6 +112,21 @@ + + + + ${basedir}/../dolphinscheduler-service/src/main/resources + + **/*.properties + **/*.xml + **/*.json + **/*.yml + + conf + + + + ${basedir}/../dolphinscheduler-server/target/dolphinscheduler-server-${project.version} diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index d4db103fe1..d942bca7d2 100644 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -69,7 +69,7 @@ elif [ "$command" = "alert-server" ]; then LOG_FILE="-Dserver=alert-server" CLASS=org.apache.dolphinscheduler.alert.AlertServer elif [ "$command" = "logger-server" ]; then - CLASS=org.apache.dolphinscheduler.server.rpc.LoggerServer + CLASS=org.apache.dolphinscheduler.server.log.LoggerServer elif [ "$command" = "combined-server" ]; then LOG_FILE="-Dlogging.config=classpath:combined_logback.xml -Dspring.profiles.active=api -Dserver.is-combined-server=true" CLASS=org.apache.dolphinscheduler.api.CombinedApplicationServer