github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

[Future-9396]Support output parameters transfer from parent workflow to child work flow (#9410) 

* [Future-9396]Support output parameters transfer from parent workflow to child work flow

* fix note
3.0.0/version-upgrade
caishunfeng 3 years ago committed by GitHub
parent
14d71d1462
commit
b285ccf930
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 209 additions and 174 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 54
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java
  2. 313
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
  3. 16
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java

54
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java
Unescape View File

@ -17,10 +17,14 @@
package org.apache.dolphinscheduler.server.master.runner; package org.apache.dolphinscheduler.server.master.runner;
import com.google.common.collect.Lists; import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE;
import org.apache.commons.collections.CollectionUtils; import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE;
import org.apache.commons.lang.StringUtils; import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVERY_START_NODE_STRING;
import org.apache.commons.lang.math.NumberUtils; import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVER_PROCESS_ID_STRING;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_NODES;
import static org.apache.dolphinscheduler.common.Constants.DEFAULT_WORKER_GROUP;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_BLOCKING;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.FailureStrategy; import org.apache.dolphinscheduler.common.enums.FailureStrategy;
@ -67,8 +71,10 @@ import org.apache.dolphinscheduler.service.alert.ProcessAlertManager;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
import org.apache.dolphinscheduler.service.queue.PeerTaskInstancePriorityQueue; import org.apache.dolphinscheduler.service.queue.PeerTaskInstancePriorityQueue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.math.NumberUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
@ -86,13 +92,10 @@ import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE; import org.slf4j.Logger;
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE; import org.slf4j.LoggerFactory;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVERY_START_NODE_STRING;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVER_PROCESS_ID_STRING; import com.google.common.collect.Lists;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_NODES;
import static org.apache.dolphinscheduler.common.Constants.DEFAULT_WORKER_GROUP;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_BLOCKING;
/** /**
* master exec thread,split dag * master exec thread,split dag
@ -447,6 +450,7 @@ public class WorkflowExecuteThread {
if (taskInstance.getState().typeIsSuccess()) { if (taskInstance.getState().typeIsSuccess()) {
completeTaskMap.put(taskInstance.getTaskCode(), taskInstance.getId()); completeTaskMap.put(taskInstance.getTaskCode(), taskInstance.getId());
processInstance.setVarPool(taskInstance.getVarPool());
processService.saveProcessInstance(processInstance); processService.saveProcessInstance(processInstance);
if (!processInstance.isBlocked()) { if (!processInstance.isBlocked()) {
submitPostNode(Long.toString(taskInstance.getTaskCode())); submitPostNode(Long.toString(taskInstance.getTaskCode()));
@ -1210,6 +1214,10 @@ public class WorkflowExecuteThread {
if (allProperty.size() > 0) { if (allProperty.size() > 0) {
taskInstance.setVarPool(JSONUtils.toJsonString(allProperty.values())); taskInstance.setVarPool(JSONUtils.toJsonString(allProperty.values()));
} }
} else {
if (StringUtils.isNotEmpty(processInstance.getVarPool())) {
taskInstance.setVarPool(processInstance.getVarPool());
}
} }
} }
@ -1279,19 +1287,19 @@ public class WorkflowExecuteThread {
taskInstances.add(task); taskInstances.add(task);
} }
//the end node of the branch of the dag //the end node of the branch of the dag
if (StringUtils.isNotEmpty(parentNodeCode) && dag.getEndNode().contains(parentNodeCode)){ if (StringUtils.isNotEmpty(parentNodeCode) && dag.getEndNode().contains(parentNodeCode)) {
TaskInstance endTaskInstance = taskInstanceMap.get(completeTaskMap.get(NumberUtils.toLong(parentNodeCode))); TaskInstance endTaskInstance = taskInstanceMap.get(completeTaskMap.get(NumberUtils.toLong(parentNodeCode)));
String taskInstanceVarPool = endTaskInstance.getVarPool(); String taskInstanceVarPool = endTaskInstance.getVarPool();
if(StringUtils.isNotEmpty(taskInstanceVarPool)) { if (StringUtils.isNotEmpty(taskInstanceVarPool)) {
Set<Property> taskProperties = JSONUtils.toList(taskInstanceVarPool, Property.class) Set<Property> taskProperties = JSONUtils.toList(taskInstanceVarPool, Property.class)
.stream().collect(Collectors.toSet()); .stream().collect(Collectors.toSet());
String processInstanceVarPool = processInstance.getVarPool(); String processInstanceVarPool = processInstance.getVarPool();
if (StringUtils.isNotEmpty(processInstanceVarPool)) { if (StringUtils.isNotEmpty(processInstanceVarPool)) {
Set<Property> properties = JSONUtils.toList(processInstanceVarPool, Property.class) Set<Property> properties = JSONUtils.toList(processInstanceVarPool, Property.class)
.stream().collect(Collectors.toSet()); .stream().collect(Collectors.toSet());
properties.addAll(taskProperties); properties.addAll(taskProperties);
processInstance.setVarPool(JSONUtils.toJsonString(properties)); processInstance.setVarPool(JSONUtils.toJsonString(properties));
}else{ } else {
processInstance.setVarPool(JSONUtils.toJsonString(taskProperties)); processInstance.setVarPool(JSONUtils.toJsonString(taskProperties));
} }
} }
@ -1637,7 +1645,7 @@ public class WorkflowExecuteThread {
stateEvent.setExecutionStatus(processInstance.getState()); stateEvent.setExecutionStatus(processInstance.getState());
stateEvent.setProcessInstanceId(this.processInstance.getId()); stateEvent.setProcessInstanceId(this.processInstance.getId());
stateEvent.setType(StateEventType.PROCESS_STATE_CHANGE); stateEvent.setType(StateEventType.PROCESS_STATE_CHANGE);
// this.processStateChangeHandler(stateEvent); // this.processStateChangeHandler(stateEvent);
// replace with `stateEvents`, make sure `WorkflowExecuteThread` can be deleted to avoid memory leaks // replace with `stateEvents`, make sure `WorkflowExecuteThread` can be deleted to avoid memory leaks
this.stateEvents.add(stateEvent); this.stateEvents.add(stateEvent);
} }
@ -1650,10 +1658,10 @@ public class WorkflowExecuteThread {
ExecutionStatus state = stateEvent.getExecutionStatus(); ExecutionStatus state = stateEvent.getExecutionStatus();
if (processInstance.getState() != state) { if (processInstance.getState() != state) {
logger.info( logger.info(
"work flow process instance [id: {}, name:{}], state change from {} to {}, cmd type: {}", "work flow process instance [id: {}, name:{}], state change from {} to {}, cmd type: {}",
processInstance.getId(), processInstance.getName(), processInstance.getId(), processInstance.getName(),
processInstance.getState(), state, processInstance.getState(), state,
processInstance.getCommandType()); processInstance.getCommandType());
processInstance.setState(state); processInstance.setState(state);
if (state.typeIsFinished()) { if (state.typeIsFinished()) {

313
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
Unescape View File

@ -17,13 +17,18 @@
package org.apache.dolphinscheduler.service.process; package org.apache.dolphinscheduler.service.process;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import static java.util.stream.Collectors.toSet;
import com.fasterxml.jackson.core.type.TypeReference; import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE;
import com.fasterxml.jackson.databind.node.ObjectNode; import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE;
import com.google.common.collect.Lists; import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_EMPTY_SUB_PROCESS;
import org.apache.commons.collections.CollectionUtils; import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_FATHER_PARAMS;
import org.apache.commons.lang.StringUtils; import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVER_PROCESS_ID_STRING;
import org.apache.commons.lang.math.NumberUtils; import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS_DEFINE_CODE;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS_PARENT_INSTANCE_ID;
import static org.apache.dolphinscheduler.common.Constants.LOCAL_PARAMS;
import static org.apache.dolphinscheduler.plugin.task.api.utils.DataQualityConstants.TASK_INSTANCE_ID;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.AuthorizationType;
import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.CommandType;
@ -124,11 +129,10 @@ import org.apache.dolphinscheduler.service.log.LogClientService;
import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
import org.apache.dolphinscheduler.service.task.TaskPluginManager; import org.apache.dolphinscheduler.service.task.TaskPluginManager;
import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired; import org.apache.commons.lang.StringUtils;
import org.springframework.stereotype.Component; import org.apache.commons.lang.math.NumberUtils;
import org.springframework.transaction.annotation.Transactional;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
@ -143,17 +147,16 @@ import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static java.util.stream.Collectors.toSet; import org.slf4j.Logger;
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE; import org.slf4j.LoggerFactory;
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE; import org.springframework.beans.factory.annotation.Autowired;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_EMPTY_SUB_PROCESS; import org.springframework.stereotype.Component;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_FATHER_PARAMS; import org.springframework.transaction.annotation.Transactional;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVER_PROCESS_ID_STRING;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS_DEFINE_CODE; import com.fasterxml.jackson.core.type.TypeReference;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS_PARENT_INSTANCE_ID; import com.fasterxml.jackson.databind.node.ObjectNode;
import static org.apache.dolphinscheduler.common.Constants.LOCAL_PARAMS; import com.google.common.collect.Lists;
import static org.apache.dolphinscheduler.plugin.task.api.utils.DataQualityConstants.TASK_INSTANCE_ID;
/** /**
* process relative dao that some mappers in this. * process relative dao that some mappers in this.
@ -163,12 +166,12 @@ public class ProcessServiceImpl implements ProcessService {
private final Logger logger = LoggerFactory.getLogger(getClass()); private final Logger logger = LoggerFactory.getLogger(getClass());
private final int[] stateArray = new int[]{ExecutionStatus.SUBMITTED_SUCCESS.ordinal(), private final int[] stateArray = new int[] {ExecutionStatus.SUBMITTED_SUCCESS.ordinal(),
ExecutionStatus.DISPATCH.ordinal(), ExecutionStatus.DISPATCH.ordinal(),
ExecutionStatus.RUNNING_EXECUTION.ordinal(), ExecutionStatus.RUNNING_EXECUTION.ordinal(),
ExecutionStatus.DELAY_EXECUTION.ordinal(), ExecutionStatus.DELAY_EXECUTION.ordinal(),
ExecutionStatus.READY_PAUSE.ordinal(), ExecutionStatus.READY_PAUSE.ordinal(),
ExecutionStatus.READY_STOP.ordinal()}; ExecutionStatus.READY_STOP.ordinal()};
@Autowired @Autowired
private UserMapper userMapper; private UserMapper userMapper;
@ -309,7 +312,7 @@ public class ProcessServiceImpl implements ProcessService {
if (processDefinition.getExecutionType().typeIsSerialWait()) { if (processDefinition.getExecutionType().typeIsSerialWait()) {
while (true) { while (true) {
List<ProcessInstance> runningProcessInstances = this.processInstanceMapper.queryByProcessDefineCodeAndStatusAndNextId(processInstance.getProcessDefinitionCode(), List<ProcessInstance> runningProcessInstances = this.processInstanceMapper.queryByProcessDefineCodeAndStatusAndNextId(processInstance.getProcessDefinitionCode(),
Constants.RUNNING_PROCESS_STATE, processInstance.getId()); Constants.RUNNING_PROCESS_STATE, processInstance.getId());
if (CollectionUtils.isEmpty(runningProcessInstances)) { if (CollectionUtils.isEmpty(runningProcessInstances)) {
processInstance.setState(ExecutionStatus.SUBMITTED_SUCCESS); processInstance.setState(ExecutionStatus.SUBMITTED_SUCCESS);
saveProcessInstance(processInstance); saveProcessInstance(processInstance);
@ -322,14 +325,14 @@ public class ProcessServiceImpl implements ProcessService {
} }
} else if (processDefinition.getExecutionType().typeIsSerialDiscard()) { } else if (processDefinition.getExecutionType().typeIsSerialDiscard()) {
List<ProcessInstance> runningProcessInstances = this.processInstanceMapper.queryByProcessDefineCodeAndStatusAndNextId(processInstance.getProcessDefinitionCode(), List<ProcessInstance> runningProcessInstances = this.processInstanceMapper.queryByProcessDefineCodeAndStatusAndNextId(processInstance.getProcessDefinitionCode(),
Constants.RUNNING_PROCESS_STATE, processInstance.getId()); Constants.RUNNING_PROCESS_STATE, processInstance.getId());
if (CollectionUtils.isEmpty(runningProcessInstances)) { if (CollectionUtils.isEmpty(runningProcessInstances)) {
processInstance.setState(ExecutionStatus.STOP); processInstance.setState(ExecutionStatus.STOP);
saveProcessInstance(processInstance); saveProcessInstance(processInstance);
} }
} else if (processDefinition.getExecutionType().typeIsSerialPriority()) { } else if (processDefinition.getExecutionType().typeIsSerialPriority()) {
List<ProcessInstance> runningProcessInstances = this.processInstanceMapper.queryByProcessDefineCodeAndStatusAndNextId(processInstance.getProcessDefinitionCode(), List<ProcessInstance> runningProcessInstances = this.processInstanceMapper.queryByProcessDefineCodeAndStatusAndNextId(processInstance.getProcessDefinitionCode(),
Constants.RUNNING_PROCESS_STATE, processInstance.getId()); Constants.RUNNING_PROCESS_STATE, processInstance.getId());
if (CollectionUtils.isNotEmpty(runningProcessInstances)) { if (CollectionUtils.isNotEmpty(runningProcessInstances)) {
for (ProcessInstance info : runningProcessInstances) { for (ProcessInstance info : runningProcessInstances) {
info.setCommandType(CommandType.STOP); info.setCommandType(CommandType.STOP);
@ -342,7 +345,7 @@ public class ProcessServiceImpl implements ProcessService {
String address = host.split(":")[0]; String address = host.split(":")[0];
int port = Integer.parseInt(host.split(":")[1]); int port = Integer.parseInt(host.split(":")[1]);
StateEventChangeCommand stateEventChangeCommand = new StateEventChangeCommand( StateEventChangeCommand stateEventChangeCommand = new StateEventChangeCommand(
info.getId(), 0, info.getState(), info.getId(), 0 info.getId(), 0, info.getState(), info.getId(), 0
); );
try { try {
stateEventCallbackService.sendResult(address, port, stateEventChangeCommand.convert2Command()); stateEventCallbackService.sendResult(address, port, stateEventChangeCommand.convert2Command());
@ -662,21 +665,21 @@ public class ProcessServiceImpl implements ProcessService {
// process instance quit by "waiting thread" state // process instance quit by "waiting thread" state
if (originCommand == null) { if (originCommand == null) {
Command command = new Command( Command command = new Command(
CommandType.RECOVER_WAITING_THREAD, CommandType.RECOVER_WAITING_THREAD,
processInstance.getTaskDependType(), processInstance.getTaskDependType(),
processInstance.getFailureStrategy(), processInstance.getFailureStrategy(),
processInstance.getExecutorId(), processInstance.getExecutorId(),
processInstance.getProcessDefinition().getCode(), processInstance.getProcessDefinition().getCode(),
JSONUtils.toJsonString(cmdParam), JSONUtils.toJsonString(cmdParam),
processInstance.getWarningType(), processInstance.getWarningType(),
processInstance.getWarningGroupId(), processInstance.getWarningGroupId(),
processInstance.getScheduleTime(), processInstance.getScheduleTime(),
processInstance.getWorkerGroup(), processInstance.getWorkerGroup(),
processInstance.getEnvironmentCode(), processInstance.getEnvironmentCode(),
processInstance.getProcessInstancePriority(), processInstance.getProcessInstancePriority(),
processInstance.getDryRun(), processInstance.getDryRun(),
processInstance.getId(), processInstance.getId(),
processInstance.getProcessDefinitionVersion() processInstance.getProcessDefinitionVersion()
); );
saveCommand(command); saveCommand(command);
return; return;
@ -708,8 +711,8 @@ public class ProcessServiceImpl implements ProcessService {
private Date getScheduleTime(Command command, Map<String, String> cmdParam) { private Date getScheduleTime(Command command, Map<String, String> cmdParam) {
Date scheduleTime = command.getScheduleTime(); Date scheduleTime = command.getScheduleTime();
if (scheduleTime == null if (scheduleTime == null
&& cmdParam != null && cmdParam != null
&& cmdParam.containsKey(CMDPARAM_COMPLEMENT_DATA_START_DATE)) { && cmdParam.containsKey(CMDPARAM_COMPLEMENT_DATA_START_DATE)) {
Date start = DateUtils.stringToDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_START_DATE)); Date start = DateUtils.stringToDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_START_DATE));
Date end = DateUtils.stringToDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_END_DATE)); Date end = DateUtils.stringToDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_END_DATE));
@ -720,7 +723,7 @@ public class ProcessServiceImpl implements ProcessService {
scheduleTime = complementDateList.get(0); scheduleTime = complementDateList.get(0);
} else { } else {
logger.error("set scheduler time error: complement date list is empty, command: {}", logger.error("set scheduler time error: complement date list is empty, command: {}",
command.toString()); command.toString());
} }
} }
return scheduleTime; return scheduleTime;
@ -769,10 +772,10 @@ public class ProcessServiceImpl implements ProcessService {
// curing global params // curing global params
processInstance.setGlobalParams(ParameterUtils.curingGlobalParams( processInstance.setGlobalParams(ParameterUtils.curingGlobalParams(
processDefinition.getGlobalParamMap(), processDefinition.getGlobalParamMap(),
processDefinition.getGlobalParamList(), processDefinition.getGlobalParamList(),
getCommandTypeIfComplement(processInstance, command), getCommandTypeIfComplement(processInstance, command),
processInstance.getScheduleTime())); processInstance.getScheduleTime()));
// set process instance priority // set process instance priority
processInstance.setProcessInstancePriority(command.getProcessInstancePriority()); processInstance.setProcessInstancePriority(command.getProcessInstancePriority());
@ -799,7 +802,7 @@ public class ProcessServiceImpl implements ProcessService {
startParamMap.putAll(fatherParamMap); startParamMap.putAll(fatherParamMap);
// set start param into global params // set start param into global params
if (startParamMap.size() > 0 if (startParamMap.size() > 0
&& processDefinition.getGlobalParamMap() != null) { && processDefinition.getGlobalParamMap() != null) {
for (Map.Entry<String, String> param : processDefinition.getGlobalParamMap().entrySet()) { for (Map.Entry<String, String> param : processDefinition.getGlobalParamMap().entrySet()) {
String val = startParamMap.get(param.getKey()); String val = startParamMap.get(param.getKey());
if (val != null) { if (val != null) {
@ -863,8 +866,8 @@ public class ProcessServiceImpl implements ProcessService {
private Boolean checkCmdParam(Command command, Map<String, String> cmdParam) { private Boolean checkCmdParam(Command command, Map<String, String> cmdParam) {
if (command.getTaskDependType() == TaskDependType.TASK_ONLY || command.getTaskDependType() == TaskDependType.TASK_PRE) { if (command.getTaskDependType() == TaskDependType.TASK_ONLY || command.getTaskDependType() == TaskDependType.TASK_PRE) {
if (cmdParam == null if (cmdParam == null
|| !cmdParam.containsKey(Constants.CMD_PARAM_START_NODES) || !cmdParam.containsKey(Constants.CMD_PARAM_START_NODES)
|| cmdParam.get(Constants.CMD_PARAM_START_NODES).isEmpty()) { || cmdParam.get(Constants.CMD_PARAM_START_NODES).isEmpty()) {
logger.error("command node depend type is {}, but start nodes is null ", command.getTaskDependType()); logger.error("command node depend type is {}, but start nodes is null ", command.getTaskDependType());
return false; return false;
} }
@ -908,10 +911,10 @@ public class ProcessServiceImpl implements ProcessService {
// Recalculate global parameters after rerun. // Recalculate global parameters after rerun.
processInstance.setGlobalParams(ParameterUtils.curingGlobalParams( processInstance.setGlobalParams(ParameterUtils.curingGlobalParams(
processDefinition.getGlobalParamMap(), processDefinition.getGlobalParamMap(),
processDefinition.getGlobalParamList(), processDefinition.getGlobalParamList(),
commandTypeIfComplement, commandTypeIfComplement,
processInstance.getScheduleTime())); processInstance.getScheduleTime()));
processInstance.setProcessDefinition(processDefinition); processInstance.setProcessDefinition(processDefinition);
} }
//reset command parameter //reset command parameter
@ -954,7 +957,7 @@ public class ProcessServiceImpl implements ProcessService {
initTaskInstance(this.findTaskInstanceById(taskId)); initTaskInstance(this.findTaskInstanceById(taskId));
} }
cmdParam.put(Constants.CMD_PARAM_RECOVERY_START_NODE_STRING, cmdParam.put(Constants.CMD_PARAM_RECOVERY_START_NODE_STRING,
String.join(Constants.COMMA, convertIntListToString(failedList))); String.join(Constants.COMMA, convertIntListToString(failedList)));
processInstance.setCommandParam(JSONUtils.toJsonString(cmdParam)); processInstance.setCommandParam(JSONUtils.toJsonString(cmdParam));
processInstance.setRunTimes(runTime + 1); processInstance.setRunTimes(runTime + 1);
break; break;
@ -967,7 +970,7 @@ public class ProcessServiceImpl implements ProcessService {
cmdParam.remove(Constants.CMD_PARAM_RECOVERY_START_NODE_STRING); cmdParam.remove(Constants.CMD_PARAM_RECOVERY_START_NODE_STRING);
List<Integer> suspendedNodeList = this.findTaskIdByInstanceState(processInstance.getId(), ExecutionStatus.PAUSE); List<Integer> suspendedNodeList = this.findTaskIdByInstanceState(processInstance.getId(), ExecutionStatus.PAUSE);
List<Integer> stopNodeList = findTaskIdByInstanceState(processInstance.getId(), List<Integer> stopNodeList = findTaskIdByInstanceState(processInstance.getId(),
ExecutionStatus.KILL); ExecutionStatus.KILL);
suspendedNodeList.addAll(stopNodeList); suspendedNodeList.addAll(stopNodeList);
for (Integer taskId : suspendedNodeList) { for (Integer taskId : suspendedNodeList) {
// initialize the pause state // initialize the pause state
@ -1044,7 +1047,7 @@ public class ProcessServiceImpl implements ProcessService {
} }
return processDefineLogMapper.queryByDefinitionCodeAndVersion( return processDefineLogMapper.queryByDefinitionCodeAndVersion(
processInstance.getProcessDefinitionCode(), processInstance.getProcessDefinitionVersion()); processInstance.getProcessDefinitionCode(), processInstance.getProcessDefinitionVersion());
} }
} }
@ -1086,13 +1089,13 @@ public class ProcessServiceImpl implements ProcessService {
List<Date> complementDate = CronUtils.getSelfFireDateList(start, end, listSchedules); List<Date> complementDate = CronUtils.getSelfFireDateList(start, end, listSchedules);
if (complementDate.size() > 0 if (complementDate.size() > 0
&& Flag.NO == processInstance.getIsSubProcess()) { && Flag.NO == processInstance.getIsSubProcess()) {
processInstance.setScheduleTime(complementDate.get(0)); processInstance.setScheduleTime(complementDate.get(0));
} }
processInstance.setGlobalParams(ParameterUtils.curingGlobalParams( processInstance.setGlobalParams(ParameterUtils.curingGlobalParams(
processDefinition.getGlobalParamMap(), processDefinition.getGlobalParamMap(),
processDefinition.getGlobalParamList(), processDefinition.getGlobalParamList(),
CommandType.COMPLEMENT_DATA, processInstance.getScheduleTime())); CommandType.COMPLEMENT_DATA, processInstance.getScheduleTime()));
} }
/** /**
@ -1111,7 +1114,7 @@ public class ProcessServiceImpl implements ProcessService {
Map<String, String> paramMap = JSONUtils.toMap(cmdParam); Map<String, String> paramMap = JSONUtils.toMap(cmdParam);
// write sub process id into cmd param. // write sub process id into cmd param.
if (paramMap.containsKey(CMD_PARAM_SUB_PROCESS) if (paramMap.containsKey(CMD_PARAM_SUB_PROCESS)
&& CMD_PARAM_EMPTY_SUB_PROCESS.equals(paramMap.get(CMD_PARAM_SUB_PROCESS))) { && CMD_PARAM_EMPTY_SUB_PROCESS.equals(paramMap.get(CMD_PARAM_SUB_PROCESS))) {
paramMap.remove(CMD_PARAM_SUB_PROCESS); paramMap.remove(CMD_PARAM_SUB_PROCESS);
paramMap.put(CMD_PARAM_SUB_PROCESS, String.valueOf(subProcessInstance.getId())); paramMap.put(CMD_PARAM_SUB_PROCESS, String.valueOf(subProcessInstance.getId()));
subProcessInstance.setCommandParam(JSONUtils.toJsonString(paramMap)); subProcessInstance.setCommandParam(JSONUtils.toJsonString(paramMap));
@ -1123,8 +1126,8 @@ public class ProcessServiceImpl implements ProcessService {
if (StringUtils.isNotEmpty(parentInstanceId)) { if (StringUtils.isNotEmpty(parentInstanceId)) {
ProcessInstance parentInstance = findProcessInstanceDetailById(Integer.parseInt(parentInstanceId)); ProcessInstance parentInstance = findProcessInstanceDetailById(Integer.parseInt(parentInstanceId));
if (parentInstance != null) { if (parentInstance != null) {
subProcessInstance.setGlobalParams( subProcessInstance.setGlobalParams(joinGlobalParams(parentInstance.getGlobalParams(), subProcessInstance.getGlobalParams()));
joinGlobalParams(parentInstance.getGlobalParams(), subProcessInstance.getGlobalParams())); subProcessInstance.setVarPool(joinVarPool(parentInstance.getVarPool(), subProcessInstance.getVarPool()));
this.saveProcessInstance(subProcessInstance); this.saveProcessInstance(subProcessInstance);
} else { } else {
logger.error("sub process command params error, cannot find parent instance: {} ", cmdParam); logger.error("sub process command params error, cannot find parent instance: {} ", cmdParam);
@ -1162,11 +1165,31 @@ public class ProcessServiceImpl implements ProcessService {
// e.g. the subProp's type is not equals with parent, or subProp's direct is not equals with parent // e.g. the subProp's type is not equals with parent, or subProp's direct is not equals with parent
// It's suggested to add node name in property, this kind of problem can be solved. // It's suggested to add node name in property, this kind of problem can be solved.
List<Property> extraSubParams = subParams.stream() List<Property> extraSubParams = subParams.stream()
.filter(subProp -> !parentParamKeys.contains(subProp.getProp())).collect(Collectors.toList()); .filter(subProp -> !parentParamKeys.contains(subProp.getProp())).collect(Collectors.toList());
parentParams.addAll(extraSubParams); parentParams.addAll(extraSubParams);
return JSONUtils.toJsonString(parentParams); return JSONUtils.toJsonString(parentParams);
} }
/**
* join parent var pool params into sub process.
* only the keys doesn't in sub process global would be joined.
*
* @param parentValPool
* @param subValPool
* @return
*/
private String joinVarPool(String parentValPool, String subValPool) {
List<Property> parentValPools = Lists.newArrayList(JSONUtils.toList(parentValPool, Property.class));
parentValPools = parentValPools.stream().filter(valPool -> valPool.getDirect() == Direct.OUT).collect(Collectors.toList());
List<Property> subValPools = Lists.newArrayList(JSONUtils.toList(subValPool, Property.class));
Set<String> parentValPoolKeys = parentValPools.stream().map(Property::getProp).collect(toSet());
List<Property> extraSubValPools = subValPools.stream().filter(sub -> !parentValPoolKeys.contains(sub.getProp())).collect(Collectors.toList());
parentValPools.addAll(extraSubValPools);
return JSONUtils.toJsonString(parentValPools);
}
/** /**
* initialize task instance * initialize task instance
* *
@ -1175,7 +1198,7 @@ public class ProcessServiceImpl implements ProcessService {
private void initTaskInstance(TaskInstance taskInstance) { private void initTaskInstance(TaskInstance taskInstance) {
if (!taskInstance.isSubProcess() if (!taskInstance.isSubProcess()
&& (taskInstance.getState().typeIsCancel() || taskInstance.getState().typeIsFailure())) { && (taskInstance.getState().typeIsCancel() || taskInstance.getState().typeIsFailure())) {
taskInstance.setFlag(Flag.NO); taskInstance.setFlag(Flag.NO);
updateTaskInstance(taskInstance); updateTaskInstance(taskInstance);
return; return;
@ -1220,12 +1243,12 @@ public class ProcessServiceImpl implements ProcessService {
@Transactional(rollbackFor = Exception.class) @Transactional(rollbackFor = Exception.class)
public TaskInstance submitTask(ProcessInstance processInstance, TaskInstance taskInstance) { public TaskInstance submitTask(ProcessInstance processInstance, TaskInstance taskInstance) {
logger.info("start submit task : {}, instance id:{}, state: {}", logger.info("start submit task : {}, instance id:{}, state: {}",
taskInstance.getName(), taskInstance.getProcessInstanceId(), processInstance.getState()); taskInstance.getName(), taskInstance.getProcessInstanceId(), processInstance.getState());
//submit to db //submit to db
TaskInstance task = submitTaskInstanceToDB(taskInstance, processInstance); TaskInstance task = submitTaskInstanceToDB(taskInstance, processInstance);
if (task == null) { if (task == null) {
logger.error("end submit task to db error, task name:{}, process id:{} state: {} ", logger.error("end submit task to db error, task name:{}, process id:{} state: {} ",
taskInstance.getName(), taskInstance.getProcessInstance(), processInstance.getState()); taskInstance.getName(), taskInstance.getProcessInstance(), processInstance.getState());
return null; return null;
} }
@ -1234,7 +1257,7 @@ public class ProcessServiceImpl implements ProcessService {
} }
logger.info("end submit task to db successfully:{} {} state:{} complete, instance id:{} state: {} ", logger.info("end submit task to db successfully:{} {} state:{} complete, instance id:{} state: {} ",
taskInstance.getId(), taskInstance.getName(), task.getState(), processInstance.getId(), processInstance.getState()); taskInstance.getId(), taskInstance.getName(), task.getState(), processInstance.getId(), processInstance.getState());
return task; return task;
} }
@ -1292,7 +1315,7 @@ public class ProcessServiceImpl implements ProcessService {
} }
} }
logger.info("sub process instance is not found,parent task:{},parent instance:{}", logger.info("sub process instance is not found,parent task:{},parent instance:{}",
parentTask.getId(), parentProcessInstance.getId()); parentTask.getId(), parentProcessInstance.getId());
return null; return null;
} }
@ -1390,21 +1413,21 @@ public class ProcessServiceImpl implements ProcessService {
String processParam = getSubWorkFlowParam(instanceMap, parentProcessInstance, fatherParams); String processParam = getSubWorkFlowParam(instanceMap, parentProcessInstance, fatherParams);
int subProcessInstanceId = childInstance == null ? 0 : childInstance.getId(); int subProcessInstanceId = childInstance == null ? 0 : childInstance.getId();
return new Command( return new Command(
commandType, commandType,
TaskDependType.TASK_POST, TaskDependType.TASK_POST,
parentProcessInstance.getFailureStrategy(), parentProcessInstance.getFailureStrategy(),
parentProcessInstance.getExecutorId(), parentProcessInstance.getExecutorId(),
subProcessDefinition.getCode(), subProcessDefinition.getCode(),
processParam, processParam,
parentProcessInstance.getWarningType(), parentProcessInstance.getWarningType(),
parentProcessInstance.getWarningGroupId(), parentProcessInstance.getWarningGroupId(),
parentProcessInstance.getScheduleTime(), parentProcessInstance.getScheduleTime(),
task.getWorkerGroup(), task.getWorkerGroup(),
task.getEnvironmentCode(), task.getEnvironmentCode(),
parentProcessInstance.getProcessInstancePriority(), parentProcessInstance.getProcessInstancePriority(),
parentProcessInstance.getDryRun(), parentProcessInstance.getDryRun(),
subProcessInstanceId, subProcessInstanceId,
subProcessDefinition.getVersion() subProcessDefinition.getVersion()
); );
} }
@ -1441,7 +1464,7 @@ public class ProcessServiceImpl implements ProcessService {
*/ */
private void updateSubProcessDefinitionByParent(ProcessInstance parentProcessInstance, long childDefinitionCode) { private void updateSubProcessDefinitionByParent(ProcessInstance parentProcessInstance, long childDefinitionCode) {
ProcessDefinition fatherDefinition = this.findProcessDefinition(parentProcessInstance.getProcessDefinitionCode(), ProcessDefinition fatherDefinition = this.findProcessDefinition(parentProcessInstance.getProcessDefinitionCode(),
parentProcessInstance.getProcessDefinitionVersion()); parentProcessInstance.getProcessDefinitionVersion());
ProcessDefinition childDefinition = this.findProcessDefinitionByCode(childDefinitionCode); ProcessDefinition childDefinition = this.findProcessDefinitionByCode(childDefinitionCode);
if (childDefinition != null && fatherDefinition != null) { if (childDefinition != null && fatherDefinition != null) {
childDefinition.setWarningGroupId(fatherDefinition.getWarningGroupId()); childDefinition.setWarningGroupId(fatherDefinition.getWarningGroupId());
@ -1460,8 +1483,8 @@ public class ProcessServiceImpl implements ProcessService {
public TaskInstance submitTaskInstanceToDB(TaskInstance taskInstance, ProcessInstance processInstance) { public TaskInstance submitTaskInstanceToDB(TaskInstance taskInstance, ProcessInstance processInstance) {
ExecutionStatus processInstanceState = processInstance.getState(); ExecutionStatus processInstanceState = processInstance.getState();
if (processInstanceState.typeIsFinished() if (processInstanceState.typeIsFinished()
|| processInstanceState == ExecutionStatus.READY_PAUSE || processInstanceState == ExecutionStatus.READY_PAUSE
|| processInstanceState == ExecutionStatus.READY_STOP) { || processInstanceState == ExecutionStatus.READY_STOP) {
logger.warn("processInstance {} was {}, skip submit task", processInstance.getProcessDefinitionCode(), processInstanceState); logger.warn("processInstance {} was {}, skip submit task", processInstance.getProcessDefinitionCode(), processInstanceState);
return null; return null;
} }
@ -1500,10 +1523,10 @@ public class ProcessServiceImpl implements ProcessService {
// the task already exists in task queue // the task already exists in task queue
// return state // return state
if ( if (
state == ExecutionStatus.RUNNING_EXECUTION state == ExecutionStatus.RUNNING_EXECUTION
|| state == ExecutionStatus.DELAY_EXECUTION || state == ExecutionStatus.DELAY_EXECUTION
|| state == ExecutionStatus.KILL || state == ExecutionStatus.KILL
|| state == ExecutionStatus.DISPATCH || state == ExecutionStatus.DISPATCH
) { ) {
return state; return state;
} }
@ -1512,7 +1535,7 @@ public class ProcessServiceImpl implements ProcessService {
if (processInstance.getState() == ExecutionStatus.READY_PAUSE) { if (processInstance.getState() == ExecutionStatus.READY_PAUSE) {
state = ExecutionStatus.PAUSE; state = ExecutionStatus.PAUSE;
} else if (processInstance.getState() == ExecutionStatus.READY_STOP } else if (processInstance.getState() == ExecutionStatus.READY_STOP
|| !checkProcessStrategy(taskInstance, processInstance)) { || !checkProcessStrategy(taskInstance, processInstance)) {
state = ExecutionStatus.KILL; state = ExecutionStatus.KILL;
} else { } else {
state = ExecutionStatus.SUBMITTED_SUCCESS; state = ExecutionStatus.SUBMITTED_SUCCESS;
@ -1535,7 +1558,7 @@ public class ProcessServiceImpl implements ProcessService {
for (TaskInstance task : taskInstances) { for (TaskInstance task : taskInstances) {
if (task.getState() == ExecutionStatus.FAILURE if (task.getState() == ExecutionStatus.FAILURE
&& task.getRetryTimes() >= task.getMaxRetryTimes()) { && task.getRetryTimes() >= task.getMaxRetryTimes()) {
return false; return false;
} }
} }
@ -1647,8 +1670,8 @@ public class ProcessServiceImpl implements ProcessService {
taskInstance.setProcessInstance(processInstance); taskInstance.setProcessInstance(processInstance);
taskInstance.setProcessDefine(processInstance.getProcessDefinition()); taskInstance.setProcessDefine(processInstance.getProcessDefinition());
TaskDefinition taskDefinition = this.findTaskDefinition( TaskDefinition taskDefinition = this.findTaskDefinition(
taskInstance.getTaskCode(), taskInstance.getTaskCode(),
taskInstance.getTaskDefinitionVersion()); taskInstance.getTaskDefinitionVersion());
this.updateTaskDefinitionResources(taskDefinition); this.updateTaskDefinitionResources(taskDefinition);
taskInstance.setTaskDefine(taskDefinition); taskInstance.setTaskDefine(taskDefinition);
} }
@ -1661,17 +1684,17 @@ public class ProcessServiceImpl implements ProcessService {
@Override @Override
public void updateTaskDefinitionResources(TaskDefinition taskDefinition) { public void updateTaskDefinitionResources(TaskDefinition taskDefinition) {
Map<String, Object> taskParameters = JSONUtils.parseObject( Map<String, Object> taskParameters = JSONUtils.parseObject(
taskDefinition.getTaskParams(), taskDefinition.getTaskParams(),
new TypeReference<Map<String, Object>>() { new TypeReference<Map<String, Object>>() {
}); });
if (taskParameters != null) { if (taskParameters != null) {
// if contains mainJar field, query resource from database // if contains mainJar field, query resource from database
// Flink, Spark, MR // Flink, Spark, MR
if (taskParameters.containsKey("mainJar")) { if (taskParameters.containsKey("mainJar")) {
Object mainJarObj = taskParameters.get("mainJar"); Object mainJarObj = taskParameters.get("mainJar");
ResourceInfo mainJar = JSONUtils.parseObject( ResourceInfo mainJar = JSONUtils.parseObject(
JSONUtils.toJsonString(mainJarObj), JSONUtils.toJsonString(mainJarObj),
ResourceInfo.class); ResourceInfo.class);
ResourceInfo resourceInfo = updateResourceInfo(mainJar); ResourceInfo resourceInfo = updateResourceInfo(mainJar);
if (resourceInfo != null) { if (resourceInfo != null) {
taskParameters.put("mainJar", resourceInfo); taskParameters.put("mainJar", resourceInfo);
@ -1682,10 +1705,10 @@ public class ProcessServiceImpl implements ProcessService {
String resourceListStr = JSONUtils.toJsonString(taskParameters.get("resourceList")); String resourceListStr = JSONUtils.toJsonString(taskParameters.get("resourceList"));
List<ResourceInfo> resourceInfos = JSONUtils.toList(resourceListStr, ResourceInfo.class); List<ResourceInfo> resourceInfos = JSONUtils.toList(resourceListStr, ResourceInfo.class);
List<ResourceInfo> updatedResourceInfos = resourceInfos List<ResourceInfo> updatedResourceInfos = resourceInfos
.stream() .stream()
.map(this::updateResourceInfo) .map(this::updateResourceInfo)
.filter(Objects::nonNull) .filter(Objects::nonNull)
.collect(Collectors.toList()); .collect(Collectors.toList());
taskParameters.put("resourceList", updatedResourceInfos); taskParameters.put("resourceList", updatedResourceInfos);
} }
// set task parameters // set task parameters
@ -1716,7 +1739,7 @@ public class ProcessServiceImpl implements ProcessService {
resourceInfo.setResourceName(resource.getFullName()); resourceInfo.setResourceName(resource.getFullName());
if (logger.isInfoEnabled()) { if (logger.isInfoEnabled()) {
logger.info("updated resource info {}", logger.info("updated resource info {}",
JSONUtils.toJsonString(resourceInfo)); JSONUtils.toJsonString(resourceInfo));
} }
} }
return resourceInfo; return resourceInfo;
@ -1938,7 +1961,7 @@ public class ProcessServiceImpl implements ProcessService {
public Map<Long, String> queryWorkerGroupByProcessDefinitionCodes(List<Long> processDefinitionCodeList) { public Map<Long, String> queryWorkerGroupByProcessDefinitionCodes(List<Long> processDefinitionCodeList) {
List<Schedule> processDefinitionScheduleList = scheduleMapper.querySchedulesByProcessDefinitionCodes(processDefinitionCodeList); List<Schedule> processDefinitionScheduleList = scheduleMapper.querySchedulesByProcessDefinitionCodes(processDefinitionCodeList);
return processDefinitionScheduleList.stream().collect(Collectors.toMap(Schedule::getProcessDefinitionCode, return processDefinitionScheduleList.stream().collect(Collectors.toMap(Schedule::getProcessDefinitionCode,
Schedule::getWorkerGroup)); Schedule::getWorkerGroup));
} }
/** /**
@ -2002,7 +2025,7 @@ public class ProcessServiceImpl implements ProcessService {
@Override @Override
public List<TaskInstance> queryNeedFailoverTaskInstances(String host) { public List<TaskInstance> queryNeedFailoverTaskInstances(String host) {
return taskInstanceMapper.queryByHostAndStatus(host, return taskInstanceMapper.queryByHostAndStatus(host,
stateArray); stateArray);
} }
/** /**
@ -2105,8 +2128,8 @@ public class ProcessServiceImpl implements ProcessService {
@Override @Override
public ProcessInstance findLastSchedulerProcessInterval(Long definitionCode, DateInterval dateInterval) { public ProcessInstance findLastSchedulerProcessInterval(Long definitionCode, DateInterval dateInterval) {
return processInstanceMapper.queryLastSchedulerProcess(definitionCode, return processInstanceMapper.queryLastSchedulerProcess(definitionCode,
dateInterval.getStartTime(), dateInterval.getStartTime(),
dateInterval.getEndTime()); dateInterval.getEndTime());
} }
/** /**
@ -2119,8 +2142,8 @@ public class ProcessServiceImpl implements ProcessService {
@Override @Override
public ProcessInstance findLastManualProcessInterval(Long definitionCode, DateInterval dateInterval) { public ProcessInstance findLastManualProcessInterval(Long definitionCode, DateInterval dateInterval) {
return processInstanceMapper.queryLastManualProcess(definitionCode, return processInstanceMapper.queryLastManualProcess(definitionCode,
dateInterval.getStartTime(), dateInterval.getStartTime(),
dateInterval.getEndTime()); dateInterval.getEndTime());
} }
/** /**
@ -2134,9 +2157,9 @@ public class ProcessServiceImpl implements ProcessService {
@Override @Override
public ProcessInstance findLastRunningProcess(Long definitionCode, Date startTime, Date endTime) { public ProcessInstance findLastRunningProcess(Long definitionCode, Date startTime, Date endTime) {
return processInstanceMapper.queryLastRunningProcess(definitionCode, return processInstanceMapper.queryLastRunningProcess(definitionCode,
startTime, startTime,
endTime, endTime,
stateArray); stateArray);
} }
/** /**
@ -2384,10 +2407,10 @@ public class ProcessServiceImpl implements ProcessService {
if (params != null && CollectionUtils.isNotEmpty(params.getResourceFilesList())) { if (params != null && CollectionUtils.isNotEmpty(params.getResourceFilesList())) {
resourceIds = params.getResourceFilesList(). resourceIds = params.getResourceFilesList().
stream() stream()
.filter(t -> t.getId() != 0) .filter(t -> t.getId() != 0)
.map(ResourceInfo::getId) .map(ResourceInfo::getId)
.collect(Collectors.toSet()); .collect(toSet());
} }
if (CollectionUtils.isEmpty(resourceIds)) { if (CollectionUtils.isEmpty(resourceIds)) {
return StringUtils.EMPTY; return StringUtils.EMPTY;
@ -2420,7 +2443,7 @@ public class ProcessServiceImpl implements ProcessService {
} }
TaskDefinitionLog definitionCodeAndVersion = taskDefinitionLogMapper TaskDefinitionLog definitionCodeAndVersion = taskDefinitionLogMapper
.queryByDefinitionCodeAndVersion(taskDefinitionLog.getCode(), taskDefinitionLog.getVersion()); .queryByDefinitionCodeAndVersion(taskDefinitionLog.getCode(), taskDefinitionLog.getVersion());
if (definitionCodeAndVersion == null) { if (definitionCodeAndVersion == null) {
taskDefinitionLog.setUserId(operator.getId()); taskDefinitionLog.setUserId(operator.getId());
taskDefinitionLog.setCreateTime(now); taskDefinitionLog.setCreateTime(now);
@ -2502,7 +2525,7 @@ public class ProcessServiceImpl implements ProcessService {
Map<Long, TaskDefinitionLog> taskDefinitionLogMap = null; Map<Long, TaskDefinitionLog> taskDefinitionLogMap = null;
if (CollectionUtils.isNotEmpty(taskDefinitionLogs)) { if (CollectionUtils.isNotEmpty(taskDefinitionLogs)) {
taskDefinitionLogMap = taskDefinitionLogs.stream() taskDefinitionLogMap = taskDefinitionLogs.stream()
.collect(Collectors.toMap(TaskDefinition::getCode, taskDefinitionLog -> taskDefinitionLog)); .collect(Collectors.toMap(TaskDefinition::getCode, taskDefinitionLog -> taskDefinitionLog));
} }
Date now = new Date(); Date now = new Date();
for (ProcessTaskRelationLog processTaskRelationLog : taskRelationList) { for (ProcessTaskRelationLog processTaskRelationLog : taskRelationList) {
@ -2547,9 +2570,9 @@ public class ProcessServiceImpl implements ProcessService {
List<ProcessTaskRelation> processTaskRelationList = processTaskRelationMapper.queryByTaskCode(taskCode); List<ProcessTaskRelation> processTaskRelationList = processTaskRelationMapper.queryByTaskCode(taskCode);
if (!processTaskRelationList.isEmpty()) { if (!processTaskRelationList.isEmpty()) {
Set<Long> processDefinitionCodes = processTaskRelationList Set<Long> processDefinitionCodes = processTaskRelationList
.stream() .stream()
.map(ProcessTaskRelation::getProcessDefinitionCode) .map(ProcessTaskRelation::getProcessDefinitionCode)
.collect(Collectors.toSet()); .collect(toSet());
List<ProcessDefinition> processDefinitionList = processDefineMapper.queryByCodes(processDefinitionCodes); List<ProcessDefinition> processDefinitionList = processDefineMapper.queryByCodes(processDefinitionCodes);
// check process definition is already online // check process definition is already online
for (ProcessDefinition processDefinition : processDefinitionList) { for (ProcessDefinition processDefinition : processDefinitionList) {
@ -2673,7 +2696,7 @@ public class ProcessServiceImpl implements ProcessService {
taskDefinitionLogs = genTaskDefineList(taskRelationList); taskDefinitionLogs = genTaskDefineList(taskRelationList);
} }
Map<Long, TaskDefinitionLog> taskDefinitionLogMap = taskDefinitionLogs.stream() Map<Long, TaskDefinitionLog> taskDefinitionLogMap = taskDefinitionLogs.stream()
.collect(Collectors.toMap(TaskDefinitionLog::getCode, taskDefinitionLog -> taskDefinitionLog)); .collect(Collectors.toMap(TaskDefinitionLog::getCode, taskDefinitionLog -> taskDefinitionLog));
List<TaskNode> taskNodeList = new ArrayList<>(); List<TaskNode> taskNodeList = new ArrayList<>();
for (Entry<Long, List<Long>> code : taskCodeMap.entrySet()) { for (Entry<Long, List<Long>> code : taskCodeMap.entrySet()) {
TaskDefinitionLog taskDefinitionLog = taskDefinitionLogMap.get(code.getKey()); TaskDefinitionLog taskDefinitionLog = taskDefinitionLogMap.get(code.getKey());
@ -2698,8 +2721,8 @@ public class ProcessServiceImpl implements ProcessService {
taskNode.setWorkerGroup(taskDefinitionLog.getWorkerGroup()); taskNode.setWorkerGroup(taskDefinitionLog.getWorkerGroup());
taskNode.setEnvironmentCode(taskDefinitionLog.getEnvironmentCode()); taskNode.setEnvironmentCode(taskDefinitionLog.getEnvironmentCode());
taskNode.setTimeout(JSONUtils.toJsonString(new TaskTimeoutParameter(taskDefinitionLog.getTimeoutFlag() == TimeoutFlag.OPEN, taskNode.setTimeout(JSONUtils.toJsonString(new TaskTimeoutParameter(taskDefinitionLog.getTimeoutFlag() == TimeoutFlag.OPEN,
taskDefinitionLog.getTimeoutNotifyStrategy(), taskDefinitionLog.getTimeoutNotifyStrategy(),
taskDefinitionLog.getTimeout()))); taskDefinitionLog.getTimeout())));
taskNode.setDelayTime(taskDefinitionLog.getDelayTime()); taskNode.setDelayTime(taskDefinitionLog.getDelayTime());
taskNode.setPreTasks(JSONUtils.toJsonString(code.getValue().stream().map(taskDefinitionLogMap::get).map(TaskDefinition::getCode).collect(Collectors.toList()))); taskNode.setPreTasks(JSONUtils.toJsonString(code.getValue().stream().map(taskDefinitionLogMap::get).map(TaskDefinition::getCode).collect(Collectors.toList())));
taskNode.setTaskGroupId(taskDefinitionLog.getTaskGroupId()); taskNode.setTaskGroupId(taskDefinitionLog.getTaskGroupId());
@ -2735,7 +2758,7 @@ public class ProcessServiceImpl implements ProcessService {
@Override @Override
public int updateDqExecuteResultUserId(int taskInstanceId) { public int updateDqExecuteResultUserId(int taskInstanceId) {
DqExecuteResult dqExecuteResult = DqExecuteResult dqExecuteResult =
dqExecuteResultMapper.selectOne(new QueryWrapper<DqExecuteResult>().eq(TASK_INSTANCE_ID, taskInstanceId)); dqExecuteResultMapper.selectOne(new QueryWrapper<DqExecuteResult>().eq(TASK_INSTANCE_ID, taskInstanceId));
if (dqExecuteResult == null) { if (dqExecuteResult == null) {
return -1; return -1;
} }
@ -2764,15 +2787,15 @@ public class ProcessServiceImpl implements ProcessService {
@Override @Override
public int deleteDqExecuteResultByTaskInstanceId(int taskInstanceId) { public int deleteDqExecuteResultByTaskInstanceId(int taskInstanceId) {
return dqExecuteResultMapper.delete( return dqExecuteResultMapper.delete(
new QueryWrapper<DqExecuteResult>() new QueryWrapper<DqExecuteResult>()
.eq(TASK_INSTANCE_ID, taskInstanceId)); .eq(TASK_INSTANCE_ID, taskInstanceId));
} }
@Override @Override
public int deleteTaskStatisticsValueByTaskInstanceId(int taskInstanceId) { public int deleteTaskStatisticsValueByTaskInstanceId(int taskInstanceId) {
return dqTaskStatisticsValueMapper.delete( return dqTaskStatisticsValueMapper.delete(
new QueryWrapper<DqTaskStatisticsValue>() new QueryWrapper<DqTaskStatisticsValue>()
.eq(TASK_INSTANCE_ID, taskInstanceId)); .eq(TASK_INSTANCE_ID, taskInstanceId));
} }
@Override @Override
@ -2845,7 +2868,7 @@ public class ProcessServiceImpl implements ProcessService {
public boolean robTaskGroupResouce(TaskGroupQueue taskGroupQueue) { public boolean robTaskGroupResouce(TaskGroupQueue taskGroupQueue) {
TaskGroup taskGroup = taskGroupMapper.selectById(taskGroupQueue.getGroupId()); TaskGroup taskGroup = taskGroupMapper.selectById(taskGroupQueue.getGroupId());
int affectedCount = taskGroupMapper.updateTaskGroupResource(taskGroup.getId(), taskGroupQueue.getId(), int affectedCount = taskGroupMapper.updateTaskGroupResource(taskGroup.getId(), taskGroupQueue.getId(),
TaskGroupQueueStatus.WAIT_QUEUE.getCode()); TaskGroupQueueStatus.WAIT_QUEUE.getCode());
if (affectedCount > 0) { if (affectedCount > 0) {
taskGroupQueue.setStatus(TaskGroupQueueStatus.ACQUIRE_SUCCESS); taskGroupQueue.setStatus(TaskGroupQueueStatus.ACQUIRE_SUCCESS);
this.taskGroupQueueMapper.updateById(taskGroupQueue); this.taskGroupQueueMapper.updateById(taskGroupQueue);
@ -2886,7 +2909,7 @@ public class ProcessServiceImpl implements ProcessService {
} }
try { try {
while (taskGroupMapper.releaseTaskGroupResource(taskGroup.getId(), taskGroup.getUseSize() while (taskGroupMapper.releaseTaskGroupResource(taskGroup.getId(), taskGroup.getUseSize()
, thisTaskGroupQueue.getId(), TaskGroupQueueStatus.ACQUIRE_SUCCESS.getCode()) != 1) { , thisTaskGroupQueue.getId(), TaskGroupQueueStatus.ACQUIRE_SUCCESS.getCode()) != 1) {
thisTaskGroupQueue = this.taskGroupQueueMapper.queryByTaskId(taskInstance.getId()); thisTaskGroupQueue = this.taskGroupQueueMapper.queryByTaskId(taskInstance.getId());
if (thisTaskGroupQueue.getStatus() == TaskGroupQueueStatus.RELEASE) { if (thisTaskGroupQueue.getStatus() == TaskGroupQueueStatus.RELEASE) {
return null; return null;
@ -2899,13 +2922,13 @@ public class ProcessServiceImpl implements ProcessService {
logger.info("updateTask:{}", taskInstance.getName()); logger.info("updateTask:{}", taskInstance.getName());
changeTaskGroupQueueStatus(taskInstance.getId(), TaskGroupQueueStatus.RELEASE); changeTaskGroupQueueStatus(taskInstance.getId(), TaskGroupQueueStatus.RELEASE);
TaskGroupQueue taskGroupQueue = this.taskGroupQueueMapper.queryTheHighestPriorityTasks(taskGroup.getId(), TaskGroupQueue taskGroupQueue = this.taskGroupQueueMapper.queryTheHighestPriorityTasks(taskGroup.getId(),
TaskGroupQueueStatus.WAIT_QUEUE.getCode(), Flag.NO.getCode(), Flag.NO.getCode()); TaskGroupQueueStatus.WAIT_QUEUE.getCode(), Flag.NO.getCode(), Flag.NO.getCode());
if (taskGroupQueue == null) { if (taskGroupQueue == null) {
return null; return null;
} }
while (this.taskGroupQueueMapper.updateInQueueCAS(Flag.NO.getCode(), Flag.YES.getCode(), taskGroupQueue.getId()) != 1) { while (this.taskGroupQueueMapper.updateInQueueCAS(Flag.NO.getCode(), Flag.YES.getCode(), taskGroupQueue.getId()) != 1) {
taskGroupQueue = this.taskGroupQueueMapper.queryTheHighestPriorityTasks(taskGroup.getId(), taskGroupQueue = this.taskGroupQueueMapper.queryTheHighestPriorityTasks(taskGroup.getId(),
TaskGroupQueueStatus.WAIT_QUEUE.getCode(), Flag.NO.getCode(), Flag.NO.getCode()); TaskGroupQueueStatus.WAIT_QUEUE.getCode(), Flag.NO.getCode(), Flag.NO.getCode());
if (taskGroupQueue == null) { if (taskGroupQueue == null) {
return null; return null;
} }
@ -2971,7 +2994,7 @@ public class ProcessServiceImpl implements ProcessService {
String address = host.split(":")[0]; String address = host.split(":")[0];
int port = Integer.parseInt(host.split(":")[1]); int port = Integer.parseInt(host.split(":")[1]);
TaskEventChangeCommand taskEventChangeCommand = new TaskEventChangeCommand( TaskEventChangeCommand taskEventChangeCommand = new TaskEventChangeCommand(
processInstance.getId(), taskId processInstance.getId(), taskId
); );
stateEventCallbackService.sendResult(address, port, taskEventChangeCommand.convert2Command(taskType)); stateEventCallbackService.sendResult(address, port, taskEventChangeCommand.convert2Command(taskType));
} }

16
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java
Unescape View File

@ -77,7 +77,6 @@ public abstract class AbstractParameters implements IParameters {
public Map<String, Property> getLocalParametersMap() { public Map<String, Property> getLocalParametersMap() {
Map<String, Property> localParametersMaps = new LinkedHashMap<>(); Map<String, Property> localParametersMaps = new LinkedHashMap<>();
if (localParams != null) { if (localParams != null) {
for (Property property : localParams) { for (Property property : localParams) {
localParametersMaps.put(property.getProp(),property); localParametersMaps.put(property.getProp(),property);
} }
@ -113,14 +112,14 @@ public abstract class AbstractParameters implements IParameters {
} }
public void dealOutParam(String result) { public void dealOutParam(String result) {
if (org.apache.commons.collections4.CollectionUtils.isEmpty(localParams)) { if (CollectionUtils.isEmpty(localParams)) {
return; return;
} }
List<Property> outProperty = getOutProperty(localParams); List<Property> outProperty = getOutProperty(localParams);
if (org.apache.commons.collections4.CollectionUtils.isEmpty(outProperty)) { if (CollectionUtils.isEmpty(outProperty)) {
return; return;
} }
if (org.apache.dolphinscheduler.spi.utils.StringUtils.isEmpty(result)) { if (StringUtils.isEmpty(result)) {
varPool.addAll(outProperty); varPool.addAll(outProperty);
return; return;
} }
@ -130,9 +129,9 @@ public abstract class AbstractParameters implements IParameters {
} }
for (Property info : outProperty) { for (Property info : outProperty) {
String propValue = taskResult.get(info.getProp()); String propValue = taskResult.get(info.getProp());
if (org.apache.dolphinscheduler.spi.utils.StringUtils.isNotEmpty(propValue)) { if (StringUtils.isNotEmpty(propValue)) {
info.setValue(propValue); info.setValue(propValue);
varPool.add(info); addPropertyToValPool(info);
} }
} }
} }
@ -180,4 +179,9 @@ public abstract class AbstractParameters implements IParameters {
public ResourceParametersHelper getResources() { public ResourceParametersHelper getResources() {
return new ResourceParametersHelper(); return new ResourceParametersHelper();
} }
private void addPropertyToValPool(Property property) {
varPool.removeIf(p -> p.getProp().equals(property.getProp()));
varPool.add(property);
}
} }

Write Preview
Loading…
Cancel
Save