|
|
|
|
@ -48,6 +48,7 @@ import org.apache.dolphinscheduler.common.enums.ApiTriggerType;
|
|
|
|
|
import org.apache.dolphinscheduler.common.enums.CommandType; |
|
|
|
|
import org.apache.dolphinscheduler.common.enums.ComplementDependentMode; |
|
|
|
|
import org.apache.dolphinscheduler.common.enums.CycleEnum; |
|
|
|
|
import org.apache.dolphinscheduler.common.enums.ExecutionOrder; |
|
|
|
|
import org.apache.dolphinscheduler.common.enums.FailureStrategy; |
|
|
|
|
import org.apache.dolphinscheduler.common.enums.Flag; |
|
|
|
|
import org.apache.dolphinscheduler.common.enums.Priority; |
|
|
|
|
@ -104,10 +105,12 @@ import org.apache.commons.lang3.StringUtils;
|
|
|
|
|
import java.time.ZonedDateTime; |
|
|
|
|
import java.util.ArrayList; |
|
|
|
|
import java.util.Arrays; |
|
|
|
|
import java.util.Collections; |
|
|
|
|
import java.util.HashMap; |
|
|
|
|
import java.util.HashSet; |
|
|
|
|
import java.util.List; |
|
|
|
|
import java.util.Map; |
|
|
|
|
import java.util.Objects; |
|
|
|
|
import java.util.Set; |
|
|
|
|
import java.util.stream.Collectors; |
|
|
|
|
|
|
|
|
|
@ -118,7 +121,7 @@ import org.springframework.context.annotation.Lazy;
|
|
|
|
|
import org.springframework.stereotype.Service; |
|
|
|
|
import org.springframework.transaction.annotation.Transactional; |
|
|
|
|
|
|
|
|
|
import com.google.common.collect.Lists; |
|
|
|
|
import com.google.common.base.Splitter; |
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
* executor service impl |
|
|
|
|
@ -207,6 +210,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
|
|
|
|
|
* @param startParams the global param values which pass to new process instance |
|
|
|
|
* @param expectedParallelismNumber the expected parallelism number when execute complement in parallel mode |
|
|
|
|
* @param testFlag testFlag |
|
|
|
|
* @param executionOrder the execution order when complementing data |
|
|
|
|
* @return execute process instance code |
|
|
|
|
*/ |
|
|
|
|
@Override |
|
|
|
|
@ -222,7 +226,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
|
|
|
|
|
Map<String, String> startParams, Integer expectedParallelismNumber, |
|
|
|
|
int dryRun, int testFlag, |
|
|
|
|
ComplementDependentMode complementDependentMode, Integer version, |
|
|
|
|
boolean allLevelDependent) { |
|
|
|
|
boolean allLevelDependent, ExecutionOrder executionOrder) { |
|
|
|
|
Project project = projectMapper.queryByCode(projectCode); |
|
|
|
|
// check user access for project
|
|
|
|
|
Map<String, Object> result = |
|
|
|
|
@ -236,6 +240,14 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
|
|
|
|
|
putMsg(result, Status.TASK_TIMEOUT_PARAMS_ERROR); |
|
|
|
|
return result; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if (Objects.nonNull(expectedParallelismNumber) && expectedParallelismNumber <= 0) { |
|
|
|
|
log.warn("Parameter expectedParallelismNumber is invalid, expectedParallelismNumber:{}.", |
|
|
|
|
expectedParallelismNumber); |
|
|
|
|
putMsg(result, Status.TASK_PARALLELISM_PARAMS_ERROR); |
|
|
|
|
return result; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
checkValidTenant(tenantCode); |
|
|
|
|
ProcessDefinition processDefinition; |
|
|
|
|
if (null != version) { |
|
|
|
|
@ -264,7 +276,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
|
|
|
|
|
cronTime, warningType, loginUser.getId(), warningGroupId, runMode, processInstancePriority, |
|
|
|
|
workerGroup, tenantCode, |
|
|
|
|
environmentCode, startParams, expectedParallelismNumber, dryRun, testFlag, |
|
|
|
|
complementDependentMode, allLevelDependent); |
|
|
|
|
complementDependentMode, allLevelDependent, executionOrder); |
|
|
|
|
|
|
|
|
|
if (create > 0) { |
|
|
|
|
processDefinition.setWarningGroupId(warningGroupId); |
|
|
|
|
@ -738,6 +750,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
|
|
|
|
|
* @param testFlag testFlag |
|
|
|
|
* @param environmentCode environmentCode |
|
|
|
|
* @param allLevelDependent allLevelDependent |
|
|
|
|
* @param executionOrder executionOrder |
|
|
|
|
* @return command id |
|
|
|
|
*/ |
|
|
|
|
private int createCommand(Long triggerCode, CommandType commandType, long processDefineCode, TaskDependType nodeDep, |
|
|
|
|
@ -747,7 +760,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
|
|
|
|
|
Long environmentCode, |
|
|
|
|
Map<String, String> startParams, Integer expectedParallelismNumber, int dryRun, |
|
|
|
|
int testFlag, ComplementDependentMode complementDependentMode, |
|
|
|
|
boolean allLevelDependent) { |
|
|
|
|
boolean allLevelDependent, ExecutionOrder executionOrder) { |
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
* instantiate command schedule instance |
|
|
|
|
@ -806,7 +819,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
|
|
|
|
|
log.info("Start to create {} command, processDefinitionCode:{}.", |
|
|
|
|
command.getCommandType().getDescp(), processDefineCode); |
|
|
|
|
return createComplementCommandList(triggerCode, schedule, runMode, command, expectedParallelismNumber, |
|
|
|
|
complementDependentMode, allLevelDependent); |
|
|
|
|
complementDependentMode, allLevelDependent, executionOrder); |
|
|
|
|
} catch (CronParseException cronParseException) { |
|
|
|
|
// We catch the exception here just to make compiler happy, since we have already validated the schedule
|
|
|
|
|
// cron expression before
|
|
|
|
|
@ -822,59 +835,20 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
|
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
* create complement command |
|
|
|
|
* close left and close right |
|
|
|
|
* |
|
|
|
|
* @param scheduleTimeParam |
|
|
|
|
* @param runMode |
|
|
|
|
* @return |
|
|
|
|
*/ |
|
|
|
|
protected int createComplementCommandList(Long triggerCode, String scheduleTimeParam, RunMode runMode, |
|
|
|
|
Command command, |
|
|
|
|
Integer expectedParallelismNumber, |
|
|
|
|
ComplementDependentMode complementDependentMode, |
|
|
|
|
boolean allLevelDependent) throws CronParseException { |
|
|
|
|
int createCount = 0; |
|
|
|
|
String startDate = null; |
|
|
|
|
String endDate = null; |
|
|
|
|
String dateList = null; |
|
|
|
|
int dependentProcessDefinitionCreateCount = 0; |
|
|
|
|
runMode = (runMode == null) ? RunMode.RUN_MODE_SERIAL : runMode; |
|
|
|
|
Map<String, String> cmdParam = JSONUtils.toMap(command.getCommandParam()); |
|
|
|
|
Map<String, String> scheduleParam = JSONUtils.toMap(scheduleTimeParam); |
|
|
|
|
if (scheduleParam.containsKey(CMD_PARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST)) { |
|
|
|
|
dateList = scheduleParam.get(CMD_PARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST); |
|
|
|
|
dateList = removeDuplicates(dateList); |
|
|
|
|
} |
|
|
|
|
if (scheduleParam.containsKey(CMD_PARAM_COMPLEMENT_DATA_START_DATE) && scheduleParam.containsKey( |
|
|
|
|
CMD_PARAM_COMPLEMENT_DATA_END_DATE)) { |
|
|
|
|
startDate = scheduleParam.get(CMD_PARAM_COMPLEMENT_DATA_START_DATE); |
|
|
|
|
endDate = scheduleParam.get(CMD_PARAM_COMPLEMENT_DATA_END_DATE); |
|
|
|
|
} |
|
|
|
|
switch (runMode) { |
|
|
|
|
case RUN_MODE_SERIAL: { |
|
|
|
|
log.info("RunMode of {} command is serial run, processDefinitionCode:{}.", |
|
|
|
|
command.getCommandType().getDescp(), command.getProcessDefinitionCode()); |
|
|
|
|
if (StringUtils.isNotEmpty(dateList)) { |
|
|
|
|
cmdParam.put(CMD_PARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST, dateList); |
|
|
|
|
command.setCommandParam(JSONUtils.toJsonString(cmdParam)); |
|
|
|
|
log.info("Creating command, commandInfo:{}.", command); |
|
|
|
|
createCount = commandService.createCommand(command); |
|
|
|
|
if (createCount > 0) { |
|
|
|
|
log.info("Create {} command complete, processDefinitionCode:{}", |
|
|
|
|
command.getCommandType().getDescp(), command.getProcessDefinitionCode()); |
|
|
|
|
} else { |
|
|
|
|
log.error("Create {} command error, processDefinitionCode:{}", |
|
|
|
|
command.getCommandType().getDescp(), command.getProcessDefinitionCode()); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
if (startDate != null && endDate != null) { |
|
|
|
|
cmdParam.put(CMD_PARAM_COMPLEMENT_DATA_START_DATE, startDate); |
|
|
|
|
cmdParam.put(CMD_PARAM_COMPLEMENT_DATA_END_DATE, endDate); |
|
|
|
|
private int createComplementCommand(Long triggerCode, Command command, Map<String, String> cmdParam, |
|
|
|
|
List<ZonedDateTime> dateTimeList, List<Schedule> schedules, |
|
|
|
|
ComplementDependentMode complementDependentMode, boolean allLevelDependent) { |
|
|
|
|
|
|
|
|
|
String dateTimeListStr = dateTimeList.stream() |
|
|
|
|
.map(item -> DateUtils.dateToString(item)) |
|
|
|
|
.collect(Collectors.joining(COMMA)); |
|
|
|
|
|
|
|
|
|
cmdParam.put(CMD_PARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST, dateTimeListStr); |
|
|
|
|
command.setCommandParam(JSONUtils.toJsonString(cmdParam)); |
|
|
|
|
|
|
|
|
|
log.info("Creating command, commandInfo:{}.", command); |
|
|
|
|
createCount = commandService.createCommand(command); |
|
|
|
|
int createCount = commandService.createCommand(command); |
|
|
|
|
|
|
|
|
|
if (createCount > 0) { |
|
|
|
|
log.info("Create {} command complete, processDefinitionCode:{}", |
|
|
|
|
command.getCommandType().getDescp(), command.getProcessDefinitionCode()); |
|
|
|
|
@ -882,9 +856,6 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
|
|
|
|
|
log.error("Create {} command error, processDefinitionCode:{}", |
|
|
|
|
command.getCommandType().getDescp(), command.getProcessDefinitionCode()); |
|
|
|
|
} |
|
|
|
|
// dependent process definition
|
|
|
|
|
List<Schedule> schedules = processService.queryReleaseSchedulerListByProcessDefinitionCode( |
|
|
|
|
command.getProcessDefinitionCode()); |
|
|
|
|
|
|
|
|
|
if (schedules.isEmpty() || complementDependentMode == ComplementDependentMode.OFF_MODE) { |
|
|
|
|
log.info( |
|
|
|
|
@ -894,102 +865,108 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
|
|
|
|
|
log.info( |
|
|
|
|
"Complement dependent mode is all dependent and Scheduler is not empty, need create complement dependent command, processDefinitionCode:{}.", |
|
|
|
|
command.getProcessDefinitionCode()); |
|
|
|
|
dependentProcessDefinitionCreateCount += |
|
|
|
|
createComplementDependentCommand(schedules, command, allLevelDependent); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if (createCount > 0) { |
|
|
|
|
triggerRelationService.saveTriggerToDb(ApiTriggerType.COMMAND, triggerCode, command.getId()); |
|
|
|
|
} |
|
|
|
|
break; |
|
|
|
|
return createCount; |
|
|
|
|
} |
|
|
|
|
case RUN_MODE_PARALLEL: { |
|
|
|
|
log.info("RunMode of {} command is parallel run, processDefinitionCode:{}.", |
|
|
|
|
command.getCommandType().getDescp(), command.getProcessDefinitionCode()); |
|
|
|
|
if (startDate != null && endDate != null) { |
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
* create complement command |
|
|
|
|
* close left and close right |
|
|
|
|
* |
|
|
|
|
* @param scheduleTimeParam |
|
|
|
|
* @param runMode |
|
|
|
|
* @param executionOrder |
|
|
|
|
* @return |
|
|
|
|
*/ |
|
|
|
|
protected int createComplementCommandList(Long triggerCode, String scheduleTimeParam, RunMode runMode, |
|
|
|
|
Command command, |
|
|
|
|
Integer expectedParallelismNumber, |
|
|
|
|
ComplementDependentMode complementDependentMode, |
|
|
|
|
boolean allLevelDependent, |
|
|
|
|
ExecutionOrder executionOrder) throws CronParseException { |
|
|
|
|
int createCount = 0; |
|
|
|
|
int dependentProcessDefinitionCreateCount = 0; |
|
|
|
|
runMode = (runMode == null) ? RunMode.RUN_MODE_SERIAL : runMode; |
|
|
|
|
Map<String, String> cmdParam = JSONUtils.toMap(command.getCommandParam()); |
|
|
|
|
Map<String, String> scheduleParam = JSONUtils.toMap(scheduleTimeParam); |
|
|
|
|
|
|
|
|
|
if (Objects.isNull(executionOrder)) { |
|
|
|
|
executionOrder = ExecutionOrder.DESC_ORDER; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
List<Schedule> schedules = processService.queryReleaseSchedulerListByProcessDefinitionCode( |
|
|
|
|
command.getProcessDefinitionCode()); |
|
|
|
|
List<ZonedDateTime> listDate = CronUtils.getSelfFireDateList( |
|
|
|
|
|
|
|
|
|
List<ZonedDateTime> listDate = new ArrayList<>(); |
|
|
|
|
if (scheduleParam.containsKey(CMD_PARAM_COMPLEMENT_DATA_START_DATE) && scheduleParam.containsKey( |
|
|
|
|
CMD_PARAM_COMPLEMENT_DATA_END_DATE)) { |
|
|
|
|
String startDate = scheduleParam.get(CMD_PARAM_COMPLEMENT_DATA_START_DATE); |
|
|
|
|
String endDate = scheduleParam.get(CMD_PARAM_COMPLEMENT_DATA_END_DATE); |
|
|
|
|
if (startDate != null && endDate != null) { |
|
|
|
|
listDate = CronUtils.getSelfFireDateList( |
|
|
|
|
DateUtils.stringToZoneDateTime(startDate), |
|
|
|
|
DateUtils.stringToZoneDateTime(endDate), |
|
|
|
|
schedules); |
|
|
|
|
int listDateSize = listDate.size(); |
|
|
|
|
createCount = listDate.size(); |
|
|
|
|
if (!CollectionUtils.isEmpty(listDate)) { |
|
|
|
|
if (expectedParallelismNumber != null && expectedParallelismNumber != 0) { |
|
|
|
|
createCount = Math.min(createCount, expectedParallelismNumber); |
|
|
|
|
} |
|
|
|
|
log.info("Complement command run in parallel mode, current expectedParallelismNumber:{}.", |
|
|
|
|
createCount); |
|
|
|
|
|
|
|
|
|
// Distribute the number of tasks equally to each command.
|
|
|
|
|
// The last command with insufficient quantity will be assigned to the remaining tasks.
|
|
|
|
|
int itemsPerCommand = (listDateSize / createCount); |
|
|
|
|
int remainingItems = (listDateSize % createCount); |
|
|
|
|
int startDateIndex = 0; |
|
|
|
|
int endDateIndex = 0; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
for (int i = 1; i <= createCount; i++) { |
|
|
|
|
int extra = (i <= remainingItems) ? 1 : 0; |
|
|
|
|
int singleCommandItems = (itemsPerCommand + extra); |
|
|
|
|
if (scheduleParam.containsKey(CMD_PARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST)) { |
|
|
|
|
String dateList = scheduleParam.get(CMD_PARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST); |
|
|
|
|
|
|
|
|
|
if (i == 1) { |
|
|
|
|
endDateIndex += singleCommandItems - 1; |
|
|
|
|
} else { |
|
|
|
|
startDateIndex = endDateIndex + 1; |
|
|
|
|
endDateIndex += singleCommandItems; |
|
|
|
|
if (StringUtils.isNotBlank(dateList)) { |
|
|
|
|
listDate = Splitter.on(COMMA).splitToStream(dateList) |
|
|
|
|
.map(item -> DateUtils.stringToZoneDateTime(item.trim())) |
|
|
|
|
.distinct() |
|
|
|
|
.collect(Collectors.toList()); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
cmdParam.put(CMD_PARAM_COMPLEMENT_DATA_START_DATE, |
|
|
|
|
DateUtils.dateToString(listDate.get(startDateIndex))); |
|
|
|
|
cmdParam.put(CMD_PARAM_COMPLEMENT_DATA_END_DATE, |
|
|
|
|
DateUtils.dateToString(listDate.get(endDateIndex))); |
|
|
|
|
command.setCommandParam(JSONUtils.toJsonString(cmdParam)); |
|
|
|
|
log.info("Creating command, commandInfo:{}.", command); |
|
|
|
|
if (commandService.createCommand(command) > 0) { |
|
|
|
|
log.info("Create {} command complete, processDefinitionCode:{}", |
|
|
|
|
command.getCommandType().getDescp(), command.getProcessDefinitionCode()); |
|
|
|
|
triggerRelationService.saveTriggerToDb(ApiTriggerType.COMMAND, triggerCode, |
|
|
|
|
command.getId()); |
|
|
|
|
} else { |
|
|
|
|
log.error("Create {} command error, processDefinitionCode:{}", |
|
|
|
|
command.getCommandType().getDescp(), command.getProcessDefinitionCode()); |
|
|
|
|
if (CollectionUtils.isEmpty(listDate)) { |
|
|
|
|
throw new ServiceException(Status.TASK_COMPLEMENT_DATA_DATE_ERROR); |
|
|
|
|
} |
|
|
|
|
if (schedules.isEmpty() || complementDependentMode == ComplementDependentMode.OFF_MODE) { |
|
|
|
|
log.info( |
|
|
|
|
"Complement dependent mode is off mode or Scheduler is empty, so skip create complement dependent command, processDefinitionCode:{}.", |
|
|
|
|
command.getProcessDefinitionCode()); |
|
|
|
|
|
|
|
|
|
if (executionOrder.equals(ExecutionOrder.DESC_ORDER)) { |
|
|
|
|
Collections.sort(listDate, Collections.reverseOrder()); |
|
|
|
|
} else { |
|
|
|
|
log.info( |
|
|
|
|
"Complement dependent mode is all dependent and Scheduler is not empty, need create complement dependent command, processDefinitionCode:{}.", |
|
|
|
|
command.getProcessDefinitionCode()); |
|
|
|
|
dependentProcessDefinitionCreateCount += |
|
|
|
|
createComplementDependentCommand(schedules, command, allLevelDependent); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
Collections.sort(listDate); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
switch (runMode) { |
|
|
|
|
case RUN_MODE_SERIAL: { |
|
|
|
|
log.info("RunMode of {} command is serial run, processDefinitionCode:{}.", |
|
|
|
|
command.getCommandType().getDescp(), command.getProcessDefinitionCode()); |
|
|
|
|
createCount = createComplementCommand(triggerCode, command, cmdParam, listDate, schedules, |
|
|
|
|
complementDependentMode, allLevelDependent); |
|
|
|
|
break; |
|
|
|
|
} |
|
|
|
|
if (StringUtils.isNotEmpty(dateList)) { |
|
|
|
|
List<String> listDate = Arrays.asList(dateList.split(COMMA)); |
|
|
|
|
createCount = listDate.size(); |
|
|
|
|
if (!CollectionUtils.isEmpty(listDate)) { |
|
|
|
|
case RUN_MODE_PARALLEL: { |
|
|
|
|
log.info("RunMode of {} command is parallel run, processDefinitionCode:{}.", |
|
|
|
|
command.getCommandType().getDescp(), command.getProcessDefinitionCode()); |
|
|
|
|
|
|
|
|
|
int queueNum = 0; |
|
|
|
|
if (CollectionUtils.isNotEmpty(listDate)) { |
|
|
|
|
queueNum = listDate.size(); |
|
|
|
|
if (expectedParallelismNumber != null && expectedParallelismNumber != 0) { |
|
|
|
|
createCount = Math.min(createCount, expectedParallelismNumber); |
|
|
|
|
queueNum = Math.min(queueNum, expectedParallelismNumber); |
|
|
|
|
} |
|
|
|
|
log.info("Complement command run in parallel mode, current expectedParallelismNumber:{}.", |
|
|
|
|
createCount); |
|
|
|
|
for (List<String> stringDate : Lists.partition(listDate, createCount)) { |
|
|
|
|
cmdParam.put(CMD_PARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST, String.join(COMMA, stringDate)); |
|
|
|
|
command.setCommandParam(JSONUtils.toJsonString(cmdParam)); |
|
|
|
|
log.info("Creating command, commandInfo:{}.", command); |
|
|
|
|
if (commandService.createCommand(command) > 0) { |
|
|
|
|
log.info("Create {} command complete, processDefinitionCode:{}", |
|
|
|
|
command.getCommandType().getDescp(), command.getProcessDefinitionCode()); |
|
|
|
|
} else { |
|
|
|
|
log.error("Create {} command error, processDefinitionCode:{}", |
|
|
|
|
command.getCommandType().getDescp(), command.getProcessDefinitionCode()); |
|
|
|
|
queueNum); |
|
|
|
|
List[] queues = new List[queueNum]; |
|
|
|
|
|
|
|
|
|
for (int i = 0; i < listDate.size(); i++) { |
|
|
|
|
if (Objects.isNull(queues[i % queueNum])) { |
|
|
|
|
queues[i % queueNum] = new ArrayList(); |
|
|
|
|
} |
|
|
|
|
queues[i % queueNum].add(listDate.get(i)); |
|
|
|
|
} |
|
|
|
|
for (List queue : queues) { |
|
|
|
|
createCount = createComplementCommand(triggerCode, command, cmdParam, queue, schedules, |
|
|
|
|
complementDependentMode, allLevelDependent); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
break; |
|
|
|
|
|
calvin
1 year ago
committed by
GitHub