wangxj3
3 years ago
committed by
GitHub
18 changed files with 765 additions and 39 deletions
@ -0,0 +1,91 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.task.switchtask; |
||||
|
||||
import org.apache.dolphinscheduler.common.enums.DependentRelation; |
||||
import org.apache.dolphinscheduler.common.process.ResourceInfo; |
||||
import org.apache.dolphinscheduler.common.task.AbstractParameters; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
|
||||
public class SwitchParameters extends AbstractParameters { |
||||
|
||||
private DependentRelation dependRelation; |
||||
private String relation; |
||||
private List<String> nextNode; |
||||
|
||||
@Override |
||||
public boolean checkParameters() { |
||||
return true; |
||||
} |
||||
|
||||
@Override |
||||
public List<ResourceInfo> getResourceFilesList() { |
||||
return new ArrayList<>(); |
||||
} |
||||
|
||||
private int resultConditionLocation; |
||||
private List<SwitchResultVo> dependTaskList; |
||||
|
||||
public DependentRelation getDependRelation() { |
||||
return dependRelation; |
||||
} |
||||
|
||||
public void setDependRelation(DependentRelation dependRelation) { |
||||
this.dependRelation = dependRelation; |
||||
} |
||||
|
||||
public int getResultConditionLocation() { |
||||
return resultConditionLocation; |
||||
} |
||||
|
||||
public void setResultConditionLocation(int resultConditionLocation) { |
||||
this.resultConditionLocation = resultConditionLocation; |
||||
} |
||||
|
||||
public String getRelation() { |
||||
return relation; |
||||
} |
||||
|
||||
public void setRelation(String relation) { |
||||
this.relation = relation; |
||||
} |
||||
|
||||
public List<SwitchResultVo> getDependTaskList() { |
||||
return dependTaskList; |
||||
} |
||||
|
||||
public void setDependTaskList(List<SwitchResultVo> dependTaskList) { |
||||
this.dependTaskList = dependTaskList; |
||||
} |
||||
|
||||
public List<String> getNextNode() { |
||||
return nextNode; |
||||
} |
||||
|
||||
public void setNextNode(Object nextNode) { |
||||
if (nextNode instanceof String) { |
||||
List<String> nextNodeList = new ArrayList<>(); |
||||
nextNodeList.add(String.valueOf(nextNode)); |
||||
this.nextNode = nextNodeList; |
||||
} else { |
||||
this.nextNode = (ArrayList) nextNode; |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,49 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.task.switchtask; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
|
||||
public class SwitchResultVo { |
||||
|
||||
private String condition; |
||||
private List<String> nextNode; |
||||
|
||||
public String getCondition() { |
||||
return condition; |
||||
} |
||||
|
||||
public void setCondition(String condition) { |
||||
this.condition = condition; |
||||
} |
||||
|
||||
public List<String> getNextNode() { |
||||
return nextNode; |
||||
} |
||||
|
||||
public void setNextNode(Object nextNode) { |
||||
if (nextNode instanceof String) { |
||||
List<String> nextNodeList = new ArrayList<>(); |
||||
nextNodeList.add(String.valueOf(nextNode)); |
||||
this.nextNode = nextNodeList; |
||||
} else { |
||||
this.nextNode = (ArrayList) nextNode; |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,180 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.master.runner; |
||||
|
||||
import org.apache.dolphinscheduler.common.enums.DependResult; |
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; |
||||
import org.apache.dolphinscheduler.common.process.Property; |
||||
import org.apache.dolphinscheduler.common.task.switchtask.SwitchParameters; |
||||
import org.apache.dolphinscheduler.common.task.switchtask.SwitchResultVo; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.common.utils.NetUtils; |
||||
import org.apache.dolphinscheduler.common.utils.StringUtils; |
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
||||
import org.apache.dolphinscheduler.server.utils.LogUtils; |
||||
import org.apache.dolphinscheduler.server.utils.SwitchTaskUtils; |
||||
|
||||
import java.util.Date; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
import java.util.concurrent.ConcurrentHashMap; |
||||
import java.util.regex.Matcher; |
||||
import java.util.regex.Pattern; |
||||
import java.util.stream.Collectors; |
||||
|
||||
public class SwitchTaskExecThread extends MasterBaseTaskExecThread { |
||||
|
||||
protected final String rgex = "['\"]*\\$\\{(.*?)\\}['\"]*"; |
||||
|
||||
/** |
||||
* complete task map |
||||
*/ |
||||
private Map<String, ExecutionStatus> completeTaskList = new ConcurrentHashMap<>(); |
||||
|
||||
/** |
||||
* switch result |
||||
*/ |
||||
private DependResult conditionResult; |
||||
|
||||
/** |
||||
* constructor of MasterBaseTaskExecThread |
||||
* |
||||
* @param taskInstance task instance |
||||
*/ |
||||
public SwitchTaskExecThread(TaskInstance taskInstance) { |
||||
super(taskInstance); |
||||
taskInstance.setStartTime(new Date()); |
||||
} |
||||
|
||||
@Override |
||||
public Boolean submitWaitComplete() { |
||||
try { |
||||
this.taskInstance = submit(); |
||||
logger.info("taskInstance submit end"); |
||||
Thread.currentThread().setName(getThreadName()); |
||||
initTaskParameters(); |
||||
logger.info("switch task start"); |
||||
waitTaskQuit(); |
||||
updateTaskState(); |
||||
} catch (Exception e) { |
||||
logger.error("switch task run exception", e); |
||||
} |
||||
return true; |
||||
} |
||||
|
||||
private void waitTaskQuit() { |
||||
List<TaskInstance> taskInstances = processService.findValidTaskListByProcessId( |
||||
taskInstance.getProcessInstanceId() |
||||
); |
||||
for (TaskInstance task : taskInstances) { |
||||
completeTaskList.putIfAbsent(task.getName(), task.getState()); |
||||
} |
||||
|
||||
SwitchParameters switchParameters = taskInstance.getSwitchDependency(); |
||||
List<SwitchResultVo> switchResultVos = switchParameters.getDependTaskList(); |
||||
SwitchResultVo switchResultVo = new SwitchResultVo(); |
||||
switchResultVo.setNextNode(switchParameters.getNextNode()); |
||||
switchResultVos.add(switchResultVo); |
||||
int finalConditionLocation = switchResultVos.size() - 1; |
||||
int i = 0; |
||||
conditionResult = DependResult.SUCCESS; |
||||
for (SwitchResultVo info : switchResultVos) { |
||||
logger.info("the {} execution ", (i + 1)); |
||||
logger.info("original condition sentence：{}", info.getCondition()); |
||||
if (StringUtils.isEmpty(info.getCondition())) { |
||||
finalConditionLocation = i; |
||||
break; |
||||
} |
||||
String content = setTaskParams(info.getCondition().replaceAll("'", "\""), rgex); |
||||
logger.info("format condition sentence：：{}", content); |
||||
Boolean result = null; |
||||
try { |
||||
result = SwitchTaskUtils.evaluate(content); |
||||
} catch (Exception e) { |
||||
logger.info("error sentence : {}", content); |
||||
conditionResult = DependResult.FAILED; |
||||
//result = false;
|
||||
break; |
||||
} |
||||
logger.info("condition result : {}", result); |
||||
if (result) { |
||||
finalConditionLocation = i; |
||||
break; |
||||
} |
||||
i++; |
||||
} |
||||
switchParameters.setDependTaskList(switchResultVos); |
||||
switchParameters.setResultConditionLocation(finalConditionLocation); |
||||
taskInstance.setSwitchDependency(switchParameters); |
||||
|
||||
//conditionResult = DependResult.SUCCESS;
|
||||
logger.info("the switch task depend result : {}", conditionResult); |
||||
} |
||||
|
||||
/** |
||||
* update task state |
||||
*/ |
||||
private void updateTaskState() { |
||||
ExecutionStatus status; |
||||
if (this.cancel) { |
||||
status = ExecutionStatus.KILL; |
||||
} else { |
||||
status = (conditionResult == DependResult.SUCCESS) ? ExecutionStatus.SUCCESS : ExecutionStatus.FAILURE; |
||||
} |
||||
taskInstance.setEndTime(new Date()); |
||||
taskInstance.setState(status); |
||||
processService.updateTaskInstance(taskInstance); |
||||
} |
||||
|
||||
private void initTaskParameters() { |
||||
taskInstance.setLogPath(LogUtils.getTaskLogPath(processInstance.getProcessDefinitionCode(), |
||||
processInstance.getProcessDefinitionVersion(), |
||||
taskInstance.getProcessInstanceId(), |
||||
taskInstance.getId())); |
||||
this.taskInstance.setStartTime(new Date()); |
||||
this.taskInstance.setHost(NetUtils.getAddr(masterConfig.getListenPort())); |
||||
this.taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION); |
||||
this.processService.saveTaskInstance(taskInstance); |
||||
} |
||||
|
||||
public String setTaskParams(String content, String rgex) { |
||||
Pattern pattern = Pattern.compile(rgex); |
||||
Matcher m = pattern.matcher(content); |
||||
Map<String, Property> globalParams = JSONUtils.toList(processInstance.getGlobalParams(), Property.class).stream().collect(Collectors.toMap(Property::getProp, Property -> Property)); |
||||
Map<String, Property> varParams = JSONUtils.toList(taskInstance.getVarPool(), Property.class).stream().collect(Collectors.toMap(Property::getProp, Property -> Property)); |
||||
if (varParams.size() > 0) { |
||||
varParams.putAll(globalParams); |
||||
globalParams = varParams; |
||||
} |
||||
while (m.find()) { |
||||
String paramName = m.group(1); |
||||
Property property = globalParams.get(paramName); |
||||
if (property == null) { |
||||
return ""; |
||||
} |
||||
String value = property.getValue(); |
||||
if (!org.apache.commons.lang.math.NumberUtils.isNumber(value)) { |
||||
value = "\"" + value + "\""; |
||||
} |
||||
logger.info("paramName：{}，paramValue{}", paramName, value); |
||||
content = content.replace("${" + paramName + "}", value); |
||||
} |
||||
return content; |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,38 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.utils; |
||||
|
||||
import javax.script.ScriptEngine; |
||||
import javax.script.ScriptEngineManager; |
||||
import javax.script.ScriptException; |
||||
|
||||
public class SwitchTaskUtils { |
||||
private static ScriptEngineManager manager; |
||||
private static ScriptEngine engine; |
||||
|
||||
static { |
||||
manager = new ScriptEngineManager(); |
||||
engine = manager.getEngineByName("js"); |
||||
} |
||||
|
||||
public static boolean evaluate(String expression) throws ScriptException { |
||||
Object result = engine.eval(expression); |
||||
return (Boolean) result; |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,167 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.master; |
||||
|
||||
import org.apache.dolphinscheduler.common.Constants; |
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; |
||||
import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; |
||||
import org.apache.dolphinscheduler.common.enums.TimeoutFlag; |
||||
import org.apache.dolphinscheduler.common.task.switchtask.SwitchParameters; |
||||
import org.apache.dolphinscheduler.common.task.switchtask.SwitchResultVo; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; |
||||
import org.apache.dolphinscheduler.dao.entity.TaskDefinition; |
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
||||
import org.apache.dolphinscheduler.server.master.config.MasterConfig; |
||||
import org.apache.dolphinscheduler.server.master.runner.SwitchTaskExecThread; |
||||
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; |
||||
import org.apache.dolphinscheduler.service.process.ProcessService; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.HashMap; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
|
||||
import org.junit.Assert; |
||||
import org.junit.Before; |
||||
import org.junit.Test; |
||||
import org.junit.runner.RunWith; |
||||
import org.mockito.Mockito; |
||||
import org.mockito.junit.MockitoJUnitRunner; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
import org.springframework.context.ApplicationContext; |
||||
|
||||
@RunWith(MockitoJUnitRunner.Silent.class) |
||||
public class SwitchTaskTest { |
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(SwitchTaskTest.class); |
||||
|
||||
/** |
||||
* TaskNode.runFlag : task can be run normally |
||||
*/ |
||||
public static final String FLOWNODE_RUN_FLAG_NORMAL = "NORMAL"; |
||||
|
||||
private ProcessService processService; |
||||
|
||||
private ProcessInstance processInstance; |
||||
|
||||
@Before |
||||
public void before() { |
||||
ApplicationContext applicationContext = Mockito.mock(ApplicationContext.class); |
||||
SpringApplicationContext springApplicationContext = new SpringApplicationContext(); |
||||
springApplicationContext.setApplicationContext(applicationContext); |
||||
|
||||
MasterConfig config = new MasterConfig(); |
||||
Mockito.when(applicationContext.getBean(MasterConfig.class)).thenReturn(config); |
||||
config.setMasterTaskCommitRetryTimes(3); |
||||
config.setMasterTaskCommitInterval(1000); |
||||
|
||||
processService = Mockito.mock(ProcessService.class); |
||||
Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService); |
||||
|
||||
processInstance = getProcessInstance(); |
||||
Mockito.when(processService |
||||
.findProcessInstanceById(processInstance.getId())) |
||||
.thenReturn(processInstance); |
||||
} |
||||
|
||||
private TaskInstance testBasicInit(ExecutionStatus expectResult) { |
||||
TaskDefinition taskDefinition = new TaskDefinition(); |
||||
taskDefinition.setTimeoutFlag(TimeoutFlag.OPEN); |
||||
taskDefinition.setTimeoutNotifyStrategy(TaskTimeoutStrategy.WARN); |
||||
taskDefinition.setTimeout(0); |
||||
Mockito.when(processService.findTaskDefinition(1L, 1)) |
||||
.thenReturn(taskDefinition); |
||||
TaskInstance taskInstance = getTaskInstance(getTaskNode(), processInstance); |
||||
|
||||
// for MasterBaseTaskExecThread.submit
|
||||
Mockito.when(processService |
||||
.submitTask(taskInstance)) |
||||
.thenReturn(taskInstance); |
||||
// for MasterBaseTaskExecThread.call
|
||||
Mockito.when(processService |
||||
.findTaskInstanceById(taskInstance.getId())) |
||||
.thenReturn(taskInstance); |
||||
// for SwitchTaskExecThread.initTaskParameters
|
||||
Mockito.when(processService |
||||
.saveTaskInstance(taskInstance)) |
||||
.thenReturn(true); |
||||
// for SwitchTaskExecThread.updateTaskState
|
||||
Mockito.when(processService |
||||
.updateTaskInstance(taskInstance)) |
||||
.thenReturn(true); |
||||
|
||||
return taskInstance; |
||||
} |
||||
|
||||
@Test |
||||
public void testExe() throws Exception { |
||||
TaskInstance taskInstance = testBasicInit(ExecutionStatus.SUCCESS); |
||||
taskInstance.setState(ExecutionStatus.SUBMITTED_SUCCESS); |
||||
SwitchTaskExecThread taskExecThread = new SwitchTaskExecThread(taskInstance); |
||||
taskExecThread.call(); |
||||
Assert.assertEquals(ExecutionStatus.SUCCESS, taskExecThread.getTaskInstance().getState()); |
||||
} |
||||
|
||||
private SwitchParameters getTaskNode() { |
||||
SwitchParameters conditionsParameters = new SwitchParameters(); |
||||
|
||||
SwitchResultVo switchResultVo1 = new SwitchResultVo(); |
||||
switchResultVo1.setCondition(" 2 == 1"); |
||||
switchResultVo1.setNextNode("t1"); |
||||
SwitchResultVo switchResultVo2 = new SwitchResultVo(); |
||||
switchResultVo2.setCondition(" 2 == 2"); |
||||
switchResultVo2.setNextNode("t2"); |
||||
SwitchResultVo switchResultVo3 = new SwitchResultVo(); |
||||
switchResultVo3.setCondition(" 3 == 2"); |
||||
switchResultVo3.setNextNode("t3"); |
||||
List<SwitchResultVo> list = new ArrayList<>(); |
||||
list.add(switchResultVo1); |
||||
list.add(switchResultVo2); |
||||
list.add(switchResultVo3); |
||||
conditionsParameters.setDependTaskList(list); |
||||
conditionsParameters.setNextNode("t"); |
||||
conditionsParameters.setRelation("AND"); |
||||
|
||||
return conditionsParameters; |
||||
} |
||||
|
||||
private ProcessInstance getProcessInstance() { |
||||
ProcessInstance processInstance = new ProcessInstance(); |
||||
processInstance.setId(1000); |
||||
processInstance.setState(ExecutionStatus.RUNNING_EXECUTION); |
||||
processInstance.setProcessDefinitionCode(1L); |
||||
return processInstance; |
||||
} |
||||
|
||||
private TaskInstance getTaskInstance(SwitchParameters conditionsParameters, ProcessInstance processInstance) { |
||||
TaskInstance taskInstance = new TaskInstance(); |
||||
taskInstance.setId(1000); |
||||
Map<String, Object> taskParamsMap = new HashMap<>(); |
||||
taskParamsMap.put(Constants.SWITCH_RESULT, ""); |
||||
taskInstance.setTaskParams(JSONUtils.toJsonString(taskParamsMap)); |
||||
taskInstance.setSwitchDependency(conditionsParameters); |
||||
taskInstance.setName("C"); |
||||
taskInstance.setTaskType("SWITCH"); |
||||
taskInstance.setProcessInstanceId(processInstance.getId()); |
||||
taskInstance.setTaskCode(1L); |
||||
taskInstance.setTaskDefinitionVersion(1); |
||||
return taskInstance; |
||||
} |
||||
} |
||||
Loading…
Reference in new issue