|
|
|
@ -17,19 +17,13 @@
|
|
|
|
|
|
|
|
|
|
package org.apache.dolphinscheduler.service.process; |
|
|
|
|
|
|
|
|
|
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE; |
|
|
|
|
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE; |
|
|
|
|
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_EMPTY_SUB_PROCESS; |
|
|
|
|
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_FATHER_PARAMS; |
|
|
|
|
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVER_PROCESS_ID_STRING; |
|
|
|
|
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS; |
|
|
|
|
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS_DEFINE_CODE; |
|
|
|
|
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS_PARENT_INSTANCE_ID; |
|
|
|
|
import static org.apache.dolphinscheduler.common.Constants.LOCAL_PARAMS; |
|
|
|
|
import static org.apache.dolphinscheduler.plugin.task.api.utils.DataQualityConstants.TASK_INSTANCE_ID; |
|
|
|
|
|
|
|
|
|
import static java.util.stream.Collectors.toSet; |
|
|
|
|
|
|
|
|
|
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; |
|
|
|
|
import com.fasterxml.jackson.core.type.TypeReference; |
|
|
|
|
import com.fasterxml.jackson.databind.node.ObjectNode; |
|
|
|
|
import com.google.common.collect.Lists; |
|
|
|
|
import org.apache.commons.collections.CollectionUtils; |
|
|
|
|
import org.apache.commons.lang.StringUtils; |
|
|
|
|
import org.apache.commons.lang.math.NumberUtils; |
|
|
|
|
import org.apache.dolphinscheduler.common.Constants; |
|
|
|
|
import org.apache.dolphinscheduler.common.enums.AuthorizationType; |
|
|
|
|
import org.apache.dolphinscheduler.common.enums.CommandType; |
|
|
|
@ -130,9 +124,11 @@ import org.apache.dolphinscheduler.service.log.LogClientService;
|
|
|
|
|
import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; |
|
|
|
|
import org.apache.dolphinscheduler.service.task.TaskPluginManager; |
|
|
|
|
import org.apache.dolphinscheduler.spi.enums.ResourceType; |
|
|
|
|
|
|
|
|
|
import org.apache.commons.collections.CollectionUtils; |
|
|
|
|
import org.apache.commons.lang.StringUtils; |
|
|
|
|
import org.slf4j.Logger; |
|
|
|
|
import org.slf4j.LoggerFactory; |
|
|
|
|
import org.springframework.beans.factory.annotation.Autowired; |
|
|
|
|
import org.springframework.stereotype.Component; |
|
|
|
|
import org.springframework.transaction.annotation.Transactional; |
|
|
|
|
|
|
|
|
|
import java.util.ArrayList; |
|
|
|
|
import java.util.Arrays; |
|
|
|
@ -147,16 +143,17 @@ import java.util.Objects;
|
|
|
|
|
import java.util.Set; |
|
|
|
|
import java.util.stream.Collectors; |
|
|
|
|
|
|
|
|
|
import org.slf4j.Logger; |
|
|
|
|
import org.slf4j.LoggerFactory; |
|
|
|
|
import org.springframework.beans.factory.annotation.Autowired; |
|
|
|
|
import org.springframework.stereotype.Component; |
|
|
|
|
import org.springframework.transaction.annotation.Transactional; |
|
|
|
|
|
|
|
|
|
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; |
|
|
|
|
import com.fasterxml.jackson.core.type.TypeReference; |
|
|
|
|
import com.fasterxml.jackson.databind.node.ObjectNode; |
|
|
|
|
import com.google.common.collect.Lists; |
|
|
|
|
import static java.util.stream.Collectors.toSet; |
|
|
|
|
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE; |
|
|
|
|
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE; |
|
|
|
|
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_EMPTY_SUB_PROCESS; |
|
|
|
|
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_FATHER_PARAMS; |
|
|
|
|
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVER_PROCESS_ID_STRING; |
|
|
|
|
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS; |
|
|
|
|
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS_DEFINE_CODE; |
|
|
|
|
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS_PARENT_INSTANCE_ID; |
|
|
|
|
import static org.apache.dolphinscheduler.common.Constants.LOCAL_PARAMS; |
|
|
|
|
import static org.apache.dolphinscheduler.plugin.task.api.utils.DataQualityConstants.TASK_INSTANCE_ID; |
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
* process relative dao that some mappers in this. |
|
|
|
@ -584,6 +581,7 @@ public class ProcessService {
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
* recursive delete all task instance by process instance id |
|
|
|
|
* |
|
|
|
|
* @param processInstanceId |
|
|
|
|
*/ |
|
|
|
|
public void deleteWorkTaskInstanceByProcessInstanceId(int processInstanceId) { |
|
|
|
@ -1345,19 +1343,22 @@ public class ProcessService {
|
|
|
|
|
ProcessInstanceMap instanceMap, |
|
|
|
|
TaskInstance task) { |
|
|
|
|
CommandType commandType = getSubCommandType(parentProcessInstance, childInstance); |
|
|
|
|
Map<String, String> subProcessParam = JSONUtils.toMap(task.getTaskParams()); |
|
|
|
|
Map<String, Object> subProcessParam = JSONUtils.toMap(task.getTaskParams(), String.class, Object.class); |
|
|
|
|
long childDefineCode = 0L; |
|
|
|
|
if (subProcessParam.containsKey(Constants.CMD_PARAM_SUB_PROCESS_DEFINE_CODE)) { |
|
|
|
|
childDefineCode = Long.parseLong(subProcessParam.get(Constants.CMD_PARAM_SUB_PROCESS_DEFINE_CODE)); |
|
|
|
|
childDefineCode = NumberUtils.toLong(String.valueOf(subProcessParam.get(Constants.CMD_PARAM_SUB_PROCESS_DEFINE_CODE))); |
|
|
|
|
} |
|
|
|
|
ProcessDefinition subProcessDefinition = processDefineMapper.queryByCode(childDefineCode); |
|
|
|
|
|
|
|
|
|
Object localParams = subProcessParam.get(Constants.LOCAL_PARAMS); |
|
|
|
|
List<Property> allParam = JSONUtils.toList(JSONUtils.toJsonString(localParams), Property.class); |
|
|
|
|
Map<String, String> globalMap = this.getGlobalParamMap(parentProcessInstance.getGlobalParams()); |
|
|
|
|
Map<String, String> globalMap = this.getGlobalParamMap(task.getVarPool()); |
|
|
|
|
Map<String, String> fatherParams = new HashMap<>(); |
|
|
|
|
if (CollectionUtils.isNotEmpty(allParam)) { |
|
|
|
|
for (Property info : allParam) { |
|
|
|
|
if (Direct.OUT == info.getDirect()) { |
|
|
|
|
continue; |
|
|
|
|
} |
|
|
|
|
fatherParams.put(info.getProp(), globalMap.get(info.getProp())); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
@ -2573,7 +2574,7 @@ public class ProcessService {
|
|
|
|
|
taskCodeVersionMap.put(processTaskRelation.getPostTaskCode(), processTaskRelation.getPostTaskVersion()); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
taskCodeVersionMap.forEach((code,version) -> { |
|
|
|
|
taskCodeVersionMap.forEach((code, version) -> { |
|
|
|
|
taskDefinitionLogs.add((TaskDefinitionLog) this.findTaskDefinition(code, version)); |
|
|
|
|
}); |
|
|
|
|
return taskDefinitionLogs; |
|
|
|
@ -2685,7 +2686,7 @@ public class ProcessService {
|
|
|
|
|
|
|
|
|
|
public int updateDqExecuteResultUserId(int taskInstanceId) { |
|
|
|
|
DqExecuteResult dqExecuteResult = |
|
|
|
|
dqExecuteResultMapper.selectOne(new QueryWrapper<DqExecuteResult>().eq(TASK_INSTANCE_ID,taskInstanceId)); |
|
|
|
|
dqExecuteResultMapper.selectOne(new QueryWrapper<DqExecuteResult>().eq(TASK_INSTANCE_ID, taskInstanceId)); |
|
|
|
|
if (dqExecuteResult == null) { |
|
|
|
|
return -1; |
|
|
|
|
} |
|
|
|
@ -2713,13 +2714,13 @@ public class ProcessService {
|
|
|
|
|
public int deleteDqExecuteResultByTaskInstanceId(int taskInstanceId) { |
|
|
|
|
return dqExecuteResultMapper.delete( |
|
|
|
|
new QueryWrapper<DqExecuteResult>() |
|
|
|
|
.eq(TASK_INSTANCE_ID,taskInstanceId)); |
|
|
|
|
.eq(TASK_INSTANCE_ID, taskInstanceId)); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public int deleteTaskStatisticsValueByTaskInstanceId(int taskInstanceId) { |
|
|
|
|
return dqTaskStatisticsValueMapper.delete( |
|
|
|
|
new QueryWrapper<DqTaskStatisticsValue>() |
|
|
|
|
.eq(TASK_INSTANCE_ID,taskInstanceId)); |
|
|
|
|
.eq(TASK_INSTANCE_ID, taskInstanceId)); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public DqRule getDqRule(int ruleId) { |
|
|
|
@ -2740,6 +2741,7 @@ public class ProcessService {
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
* the first time (when submit the task ) get the resource of the task group |
|
|
|
|
* |
|
|
|
|
* @param taskId task id |
|
|
|
|
* @param taskName |
|
|
|
|
* @param groupId |
|
|
|
@ -2786,12 +2788,13 @@ public class ProcessService {
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
* try to get the task group resource(when other task release the resource) |
|
|
|
|
* |
|
|
|
|
* @param taskGroupQueue |
|
|
|
|
* @return |
|
|
|
|
*/ |
|
|
|
|
public boolean robTaskGroupResouce(TaskGroupQueue taskGroupQueue) { |
|
|
|
|
TaskGroup taskGroup = taskGroupMapper.selectById(taskGroupQueue.getGroupId()); |
|
|
|
|
int affectedCount = taskGroupMapper.updateTaskGroupResource(taskGroup.getId(),taskGroupQueue.getId(), |
|
|
|
|
int affectedCount = taskGroupMapper.updateTaskGroupResource(taskGroup.getId(), taskGroupQueue.getId(), |
|
|
|
|
TaskGroupQueueStatus.WAIT_QUEUE.getCode()); |
|
|
|
|
if (affectedCount > 0) { |
|
|
|
|
taskGroupQueue.setStatus(TaskGroupQueueStatus.ACQUIRE_SUCCESS); |
|
|
|
@ -2838,9 +2841,9 @@ public class ProcessService {
|
|
|
|
|
taskGroup = taskGroupMapper.selectById(taskInstance.getTaskGroupId()); |
|
|
|
|
} |
|
|
|
|
} catch (Exception e) { |
|
|
|
|
logger.error("release the task group error",e); |
|
|
|
|
logger.error("release the task group error", e); |
|
|
|
|
} |
|
|
|
|
logger.info("updateTask:{}",taskInstance.getName()); |
|
|
|
|
logger.info("updateTask:{}", taskInstance.getName()); |
|
|
|
|
changeTaskGroupQueueStatus(taskInstance.getId(), TaskGroupQueueStatus.RELEASE); |
|
|
|
|
TaskGroupQueue taskGroupQueue = this.taskGroupQueueMapper.queryTheHighestPriorityTasks(taskGroup.getId(), |
|
|
|
|
TaskGroupQueueStatus.WAIT_QUEUE.getCode(), Flag.NO.getCode(), Flag.NO.getCode()); |
|
|
|
@ -2903,7 +2906,7 @@ public class ProcessService {
|
|
|
|
|
return this.taskGroupQueueMapper.queryByTaskId(taskId); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public void sendStartTask2Master(ProcessInstance processInstance,int taskId, |
|
|
|
|
public void sendStartTask2Master(ProcessInstance processInstance, int taskId, |
|
|
|
|
org.apache.dolphinscheduler.remote.command.CommandType taskType) { |
|
|
|
|
String host = processInstance.getHost(); |
|
|
|
|
String address = host.split(":")[0]; |
|
|
|
|