github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

[Improve] Enhance complement function transformation (#10376)

3.1.0-release
hstdream 2 years ago committed by GitHub
parent
088c5e8ae2
commit
f6fea06f10
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 344 additions and 199 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      docs/docs/en/about/glossary.md
  2. 2
      docs/docs/zh/about/glossary.md
  3. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java
  4. 1
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
  5. 306
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java
  6. 14
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorServiceTest.java
  7. 11
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
  8. 152
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
  9. 53
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/corn/CronUtils.java

2
docs/docs/en/about/glossary.md
Unescape View File

@ -45,7 +45,7 @@ provided. **Continue** refers to regardless of the status of the task running in
failure. **End** means that once a failed task is found, Kill will also run the parallel task at the same time, and the failure. **End** means that once a failed task is found, Kill will also run the parallel task at the same time, and the
process fails and ends process fails and ends
**Complement**: Supplement historical data，Supports **interval parallel and serial** two complement methods **Complement**: Supplement historical data，supports **interval parallel and serial** two complement methods, and two types of date selection which include **date range** and **date enumeration**.
### 2.Module introduction ### 2.Module introduction

2
docs/docs/zh/about/glossary.md
Unescape View File

@ -30,7 +30,7 @@
**失败策略**：对于并行运行的任务，如果有任务失败，提供两种失败策略处理方式，**继续**是指不管并行运行任务的状态，直到流程失败结束。**结束**是指一旦发现失败任务，则同时Kill掉正在运行的并行任务，流程失败结束 **失败策略**：对于并行运行的任务，如果有任务失败，提供两种失败策略处理方式，**继续**是指不管并行运行任务的状态，直到流程失败结束。**结束**是指一旦发现失败任务，则同时Kill掉正在运行的并行任务，流程失败结束
**补数**：补历史数据，支持**区间并行和串行**两种补数方式 **补数**：补历史数据，支持**区间并行和串行**两种补数方式，其日期选择方式包括**日期范围**和**日期枚举**两种
### 2.模块介绍 ### 2.模块介绍

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java
Unescape View File

@ -87,7 +87,7 @@ public class ExecutorController extends BaseController {
* @param loginUser login user * @param loginUser login user
* @param projectCode project code * @param projectCode project code
* @param processDefinitionCode process definition code * @param processDefinitionCode process definition code
* @param scheduleTime schedule time * @param scheduleTime schedule time when CommandType is COMPLEMENT_DATA there are two ways to transfer parameters 1.date range, for example:{"complementStartDate":"2022-01-01 12:12:12","complementEndDate":"2022-01-6 12:12:12"} 2.manual input, for example:{"complementScheduleDateList":"2022-01-01 00:00:00,2022-01-02 12:12:12,2022-01-03 12:12:12"}
* @param failureStrategy failure strategy * @param failureStrategy failure strategy
* @param startNodeList start nodes list * @param startNodeList start nodes list
* @param taskDependType task depend type * @param taskDependType task depend type

1
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
Unescape View File

@ -407,6 +407,7 @@ public enum Status {
NO_CURRENT_OPERATING_PERMISSION(1400001, "The current user does not have this permission.", "当前用户无此权限"), NO_CURRENT_OPERATING_PERMISSION(1400001, "The current user does not have this permission.", "当前用户无此权限"),
FUNCTION_DISABLED(1400002, "The current feature is disabled.", "当前功能已被禁用"), FUNCTION_DISABLED(1400002, "The current feature is disabled.", "当前功能已被禁用"),
SCHEDULE_TIME_NUMBER(1400003, "The number of complement dates exceed 100.", "补数日期个数超过100"),
; ;
private final int code; private final int code;

306
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java
Unescape View File

@ -17,14 +17,12 @@
package org.apache.dolphinscheduler.api.service.impl; package org.apache.dolphinscheduler.api.service.impl;
import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.WORKFLOW_START; import com.fasterxml.jackson.core.type.TypeReference;
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE; import com.google.common.collect.Lists;
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE; import org.apache.commons.beanutils.BeanUtils;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVER_PROCESS_ID_STRING; import org.apache.commons.collections.CollectionUtils;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_NODES; import org.apache.commons.collections.MapUtils;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_PARAMS; import org.apache.commons.lang3.StringUtils;
import static org.apache.dolphinscheduler.common.Constants.MAX_TASK_TIMEOUT;
import org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant; import org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant;
import org.apache.dolphinscheduler.api.enums.ExecuteType; import org.apache.dolphinscheduler.api.enums.ExecuteType;
import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.enums.Status;
@ -69,13 +67,13 @@ import org.apache.dolphinscheduler.remote.command.StateEventChangeCommand;
import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService; import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService;
import org.apache.dolphinscheduler.service.corn.CronUtils; import org.apache.dolphinscheduler.service.corn.CronUtils;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
import org.slf4j.Logger;
import org.apache.commons.beanutils.BeanUtils; import org.slf4j.LoggerFactory;
import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired;
import org.apache.commons.collections.MapUtils; import org.springframework.stereotype.Service;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
@ -84,12 +82,16 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.slf4j.Logger; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.WORKFLOW_START;
import org.slf4j.LoggerFactory; import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE;
import org.springframework.beans.factory.annotation.Autowired; import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST;
import org.springframework.stereotype.Service; import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVER_PROCESS_ID_STRING;
import com.fasterxml.jackson.core.type.TypeReference; import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_NODES;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_PARAMS;
import static org.apache.dolphinscheduler.common.Constants.COMMA;
import static org.apache.dolphinscheduler.common.Constants.MAX_TASK_TIMEOUT;
import static org.apache.dolphinscheduler.common.Constants.SCHEDULE_TIME_MAX_LENGTH;
/** /**
* executor service impl * executor service impl
@ -187,11 +189,15 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
return result; return result;
} }
if(!checkScheduleTimeNum(commandType,cronTime)){
putMsg(result, Status.SCHEDULE_TIME_NUMBER);
return result;
}
// check master exists // check master exists
if (!checkMasterExists(result)) { if (!checkMasterExists(result)) {
return result; return result;
} }
/** /**
* create command * create command
*/ */
@ -228,6 +234,29 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
return true; return true;
} }
/**
*
* @param complementData
* @param cronTime
* @return CommandType is COMPLEMENT_DATA and cronTime's number is not greater than 100 return true , otherwise return false
*/
private boolean checkScheduleTimeNum(CommandType complementData,String cronTime) {
if (!CommandType.COMPLEMENT_DATA.equals(complementData)) {
return true;
}
if(cronTime == null){
return true;
}
Map<String,String> cronMap = JSONUtils.toMap(cronTime);
if (cronMap.containsKey(CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST)) {
String[] stringDates = cronMap.get(CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST).split(COMMA);
if (stringDates.length > SCHEDULE_TIME_MAX_LENGTH) {
return false;
}
}
return true;
}
/** /**
* check whether the process definition can be executed * check whether the process definition can be executed
* *
@ -655,27 +684,16 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
} }
command.setProcessInstanceId(0); command.setProcessInstanceId(0);
Date start = null;
Date end = null;
if (!StringUtils.isEmpty(schedule)) {
String[] interval = schedule.split(",");
if (interval.length == 2) {
start = DateUtils.getScheduleDate(interval[0]);
end = DateUtils.getScheduleDate(interval[1]);
if (start.after(end)) {
logger.info("complement data error, wrong date start:{} and end date:{} ",
start, end
);
return 0;
}
}
}
// determine whether to complement // determine whether to complement
if (commandType == CommandType.COMPLEMENT_DATA) { if (commandType == CommandType.COMPLEMENT_DATA) {
if (start == null || end == null) { if (schedule == null || StringUtils.isEmpty(schedule)) {
return 0; return 0;
} }
return createComplementCommandList(start, end, runMode, command, expectedParallelismNumber, complementDependentMode); int check = checkScheduleTime(schedule);
if(check == 0){
return 0;
}
return createComplementCommandList(schedule, runMode, command, expectedParallelismNumber, complementDependentMode);
} else { } else {
command.setCommandParam(JSONUtils.toJsonString(cmdParam)); command.setCommandParam(JSONUtils.toJsonString(cmdParam));
return processService.createCommand(command); return processService.createCommand(command);
@ -686,89 +704,117 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
* create complement command * create complement command
* close left and close right * close left and close right
* *
* @param start * @param scheduleTimeParam
* @param end
* @param runMode * @param runMode
* @return * @return
*/ */
protected int createComplementCommandList(Date start, Date end, RunMode runMode, Command command, protected int createComplementCommandList(String scheduleTimeParam, RunMode runMode, Command command,
Integer expectedParallelismNumber, ComplementDependentMode complementDependentMode) { Integer expectedParallelismNumber, ComplementDependentMode complementDependentMode) {
int createCount = 0; int createCount = 0;
String startDate = null;
String endDate = null;
String dateList = null;
int dependentProcessDefinitionCreateCount = 0; int dependentProcessDefinitionCreateCount = 0;
runMode = (runMode == null) ? RunMode.RUN_MODE_SERIAL : runMode; runMode = (runMode == null) ? RunMode.RUN_MODE_SERIAL : runMode;
Map<String, String> cmdParam = JSONUtils.toMap(command.getCommandParam()); Map<String, String> cmdParam = JSONUtils.toMap(command.getCommandParam());
Map<String, String> scheduleParam = JSONUtils.toMap(scheduleTimeParam);
if(scheduleParam.containsKey(CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST)){
dateList = scheduleParam.get(CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST);
dateList = removeDuplicates(dateList);
}
if(scheduleParam.containsKey(CMDPARAM_COMPLEMENT_DATA_START_DATE) && scheduleParam.containsKey(CMDPARAM_COMPLEMENT_DATA_END_DATE)){
startDate = scheduleParam.get(CMDPARAM_COMPLEMENT_DATA_START_DATE);
endDate = scheduleParam.get(CMDPARAM_COMPLEMENT_DATA_END_DATE);
}
switch (runMode) { switch (runMode) {
case RUN_MODE_SERIAL: { case RUN_MODE_SERIAL: {
if (start.after(end)) { if(StringUtils.isNotEmpty(dateList)){
logger.warn("The startDate {} is later than the endDate {}", start, end); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST, dateList);
break; command.setCommandParam(JSONUtils.toJsonString(cmdParam));
createCount = processService.createCommand(command);
} }
cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, DateUtils.dateToString(start)); if(startDate != null && endDate != null){
cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, DateUtils.dateToString(end)); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, startDate);
command.setCommandParam(JSONUtils.toJsonString(cmdParam)); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, endDate);
createCount = processService.createCommand(command); command.setCommandParam(JSONUtils.toJsonString(cmdParam));
createCount = processService.createCommand(command);
// dependent process definition
List<Schedule> schedules = processService.queryReleaseSchedulerListByProcessDefinitionCode(command.getProcessDefinitionCode()); // dependent process definition
List<Schedule> schedules = processService.queryReleaseSchedulerListByProcessDefinitionCode(command.getProcessDefinitionCode());
if (schedules.isEmpty() || complementDependentMode == ComplementDependentMode.OFF_MODE) {
logger.info("process code: {} complement dependent in off mode or schedule's size is 0, skip " if (schedules.isEmpty() || complementDependentMode == ComplementDependentMode.OFF_MODE) {
+ "dependent complement data", command.getProcessDefinitionCode()); logger.info("process code: {} complement dependent in off mode or schedule's size is 0, skip "
} else { + "dependent complement data", command.getProcessDefinitionCode());
dependentProcessDefinitionCreateCount += createComplementDependentCommand(schedules, command); } else {
dependentProcessDefinitionCreateCount += createComplementDependentCommand(schedules, command);
}
} }
break; break;
} }
case RUN_MODE_PARALLEL: { case RUN_MODE_PARALLEL: {
if (start.after(end)) { if(startDate != null && endDate != null){
logger.warn("The startDate {} is later than the endDate {}", start, end); List<Date> listDate = new ArrayList<>();
break; List<Schedule> schedules = processService.queryReleaseSchedulerListByProcessDefinitionCode(command.getProcessDefinitionCode());
} listDate.addAll(CronUtils.getSelfFireDateList(DateUtils.getScheduleDate(startDate), DateUtils.getScheduleDate(endDate), schedules));
int listDateSize = listDate.size();
List<Date> listDate = new ArrayList<>(); createCount = listDate.size();
List<Schedule> schedules = processService.queryReleaseSchedulerListByProcessDefinitionCode(command.getProcessDefinitionCode()); if (!CollectionUtils.isEmpty(listDate)) {
listDate.addAll(CronUtils.getSelfFireDateList(start, end, schedules)); if (expectedParallelismNumber != null && expectedParallelismNumber != 0) {
int listDateSize = listDate.size(); createCount = Math.min(listDate.size(), expectedParallelismNumber);
createCount = listDate.size(); if (listDateSize < createCount) {
if (!CollectionUtils.isEmpty(listDate)) { createCount = listDateSize;
if (expectedParallelismNumber != null && expectedParallelismNumber != 0) { }
createCount = Math.min(listDate.size(), expectedParallelismNumber); }
if (listDateSize < createCount) { logger.info("In parallel mode, current expectedParallelismNumber:{}", createCount);
createCount = listDateSize;
// Distribute the number of tasks equally to each command.
// The last command with insufficient quantity will be assigned to the remaining tasks.
int itemsPerCommand = (listDateSize / createCount);
int remainingItems = (listDateSize % createCount);
int startDateIndex = 0;
int endDateIndex = 0;
for (int i = 1; i <= createCount; i++) {
int extra = (i <= remainingItems) ? 1 : 0;
int singleCommandItems = (itemsPerCommand + extra);
if (i == 1) {
endDateIndex += singleCommandItems - 1;
} else {
startDateIndex = endDateIndex + 1;
endDateIndex += singleCommandItems;
}
cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, DateUtils.dateToString(listDate.get(startDateIndex)));
cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, DateUtils.dateToString(listDate.get(endDateIndex)));
command.setCommandParam(JSONUtils.toJsonString(cmdParam));
processService.createCommand(command);
if (schedules.isEmpty() || complementDependentMode == ComplementDependentMode.OFF_MODE) {
logger.info("process code: {} complement dependent in off mode or schedule's size is 0, skip "
+ "dependent complement data", command.getProcessDefinitionCode());
} else {
dependentProcessDefinitionCreateCount += createComplementDependentCommand(schedules, command);
}
} }
} }
logger.info("In parallel mode, current expectedParallelismNumber:{}", createCount); }
if(StringUtils.isNotEmpty(dateList)){
// Distribute the number of tasks equally to each command. List<String> listDate = Arrays.asList(dateList.split(COMMA));
// The last command with insufficient quantity will be assigned to the remaining tasks. int listDateSize = listDate.size();
int itemsPerCommand = (listDateSize / createCount); createCount = listDate.size();
int remainingItems = (listDateSize % createCount); if (!CollectionUtils.isEmpty(listDate)) {
int startDateIndex = 0; if (expectedParallelismNumber != null && expectedParallelismNumber != 0) {
int endDateIndex = 0; createCount = Math.min(listDate.size(), expectedParallelismNumber);
if (listDateSize < createCount) {
for (int i = 1; i <= createCount; i++) { createCount = listDateSize;
int extra = (i <= remainingItems) ? 1 : 0; }
int singleCommandItems = (itemsPerCommand + extra);
if (i == 1) {
endDateIndex += singleCommandItems - 1;
} else {
startDateIndex = endDateIndex + 1;
endDateIndex += singleCommandItems;
} }
logger.info("In parallel mode, current expectedParallelismNumber:{}", createCount);
cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, DateUtils.dateToString(listDate.get(startDateIndex))); for (List<String> stringDate : Lists.partition(listDate, createCount)) {
cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, DateUtils.dateToString(listDate.get(endDateIndex))); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST, String.join(COMMA, stringDate));
command.setCommandParam(JSONUtils.toJsonString(cmdParam)); command.setCommandParam(JSONUtils.toJsonString(cmdParam));
processService.createCommand(command); processService.createCommand(command);
if (schedules.isEmpty() || complementDependentMode == ComplementDependentMode.OFF_MODE) {
logger.info("process code: {} complement dependent in off mode or schedule's size is 0, skip "
+ "dependent complement data", command.getProcessDefinitionCode());
} else {
dependentProcessDefinitionCreateCount += createComplementDependentCommand(schedules, command);
} }
} }
} }
@ -854,4 +900,60 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
return validDependentProcessDefinitionList; return validDependentProcessDefinitionList;
} }
/**
*
* @param schedule
* @return check error return 0 otherwish 1
*/
private int checkScheduleTime(String schedule){
Date start = null;
Date end = null;
Map<String,String> scheduleResult = JSONUtils.toMap(schedule);
if(scheduleResult == null){
return 0;
}
if(scheduleResult.containsKey(CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST)){
if(scheduleResult.get(CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST) == null){
return 0;
}
}
if(scheduleResult.containsKey(CMDPARAM_COMPLEMENT_DATA_START_DATE)){
String startDate = scheduleResult.get(CMDPARAM_COMPLEMENT_DATA_START_DATE);
String endDate = scheduleResult.get(CMDPARAM_COMPLEMENT_DATA_END_DATE);
if (startDate == null || endDate == null) {
return 0;
}
start = DateUtils.getScheduleDate(startDate);
end = DateUtils.getScheduleDate(endDate);
if(start == null || end == null){
return 0;
}
if (start.after(end)) {
logger.error("complement data error, wrong date start:{} and end date:{} ",
start, end
);
return 0;
}
}
return 1;
}
/**
*
* @param scheduleTimeList
* @return remove duplicate date list
*/
private String removeDuplicates(String scheduleTimeList){
HashSet<String> removeDate = new HashSet<String>();
List<String> resultList = new ArrayList<String>();
if(StringUtils.isNotEmpty(scheduleTimeList)){
String[] dateArrays = scheduleTimeList.split(COMMA);
List<String> dateList = Arrays.asList(dateArrays);
removeDate.addAll(dateList);
resultList.addAll(removeDate);
return String.join(COMMA, resultList);
}
return null;
}
} }

14
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorServiceTest.java
Unescape View File

@ -199,7 +199,7 @@ public class ExecutorServiceTest {
Mockito.when(processService.queryReleaseSchedulerListByProcessDefinitionCode(processDefinitionCode)).thenReturn(zeroSchedulerList()); Mockito.when(processService.queryReleaseSchedulerListByProcessDefinitionCode(processDefinitionCode)).thenReturn(zeroSchedulerList());
Map<String, Object> result = executorService.execProcessInstance(loginUser, projectCode, Map<String, Object> result = executorService.execProcessInstance(loginUser, projectCode,
processDefinitionCode, cronTime, CommandType.START_PROCESS, processDefinitionCode, "{\"complementStartDate\":\"2020-01-01 00:00:00\",\"complementEndDate\":\"2020-01-31 23:00:00\"}", CommandType.START_PROCESS,
null, null, null, null,
null, null, 0, null, null, 0,
RunMode.RUN_MODE_SERIAL, RunMode.RUN_MODE_SERIAL,
@ -218,7 +218,7 @@ public class ExecutorServiceTest {
Mockito.when(processService.queryReleaseSchedulerListByProcessDefinitionCode(processDefinitionCode)).thenReturn(zeroSchedulerList()); Mockito.when(processService.queryReleaseSchedulerListByProcessDefinitionCode(processDefinitionCode)).thenReturn(zeroSchedulerList());
Map<String, Object> result = executorService.execProcessInstance(loginUser, projectCode, Map<String, Object> result = executorService.execProcessInstance(loginUser, projectCode,
processDefinitionCode, cronTime, CommandType.START_PROCESS, processDefinitionCode, "{\"complementStartDate\":\"2020-01-01 00:00:00\",\"complementEndDate\":\"2020-01-31 23:00:00\"}", CommandType.START_PROCESS,
null, "n1,n2", null, "n1,n2",
null, null, 0, null, null, 0,
RunMode.RUN_MODE_SERIAL, RunMode.RUN_MODE_SERIAL,
@ -237,7 +237,7 @@ public class ExecutorServiceTest {
Mockito.when(processService.queryReleaseSchedulerListByProcessDefinitionCode(processDefinitionCode)).thenReturn(zeroSchedulerList()); Mockito.when(processService.queryReleaseSchedulerListByProcessDefinitionCode(processDefinitionCode)).thenReturn(zeroSchedulerList());
Map<String, Object> result = executorService.execProcessInstance(loginUser, projectCode, Map<String, Object> result = executorService.execProcessInstance(loginUser, projectCode,
processDefinitionCode, "2020-01-31 23:00:00,2020-01-01 00:00:00", CommandType.COMPLEMENT_DATA, processDefinitionCode, "{\"complementStartDate\":\"2022-01-07 12:12:12\",\"complementEndDate\":\"2022-01-06 12:12:12\"}", CommandType.COMPLEMENT_DATA,
null, null, null, null,
null, null, 0, null, null, 0,
RunMode.RUN_MODE_SERIAL, RunMode.RUN_MODE_SERIAL,
@ -255,7 +255,7 @@ public class ExecutorServiceTest {
Mockito.when(processService.queryReleaseSchedulerListByProcessDefinitionCode(processDefinitionCode)).thenReturn(zeroSchedulerList()); Mockito.when(processService.queryReleaseSchedulerListByProcessDefinitionCode(processDefinitionCode)).thenReturn(zeroSchedulerList());
Map<String, Object> result = executorService.execProcessInstance(loginUser, projectCode, Map<String, Object> result = executorService.execProcessInstance(loginUser, projectCode,
processDefinitionCode, cronTime, CommandType.COMPLEMENT_DATA, processDefinitionCode, "{\"complementStartDate\":\"2020-01-01 00:00:00\",\"complementEndDate\":\"2020-01-31 23:00:00\"}", CommandType.COMPLEMENT_DATA,
null, null, null, null,
null, null, 0, null, null, 0,
RunMode.RUN_MODE_SERIAL, RunMode.RUN_MODE_SERIAL,
@ -273,7 +273,7 @@ public class ExecutorServiceTest {
Mockito.when(processService.queryReleaseSchedulerListByProcessDefinitionCode(processDefinitionCode)).thenReturn(zeroSchedulerList()); Mockito.when(processService.queryReleaseSchedulerListByProcessDefinitionCode(processDefinitionCode)).thenReturn(zeroSchedulerList());
Map<String, Object> result = executorService.execProcessInstance(loginUser, projectCode, Map<String, Object> result = executorService.execProcessInstance(loginUser, projectCode,
processDefinitionCode, cronTime, CommandType.COMPLEMENT_DATA, processDefinitionCode, "{\"complementStartDate\":\"2020-01-01 00:00:00\",\"complementEndDate\":\"2020-01-31 23:00:00\"}", CommandType.COMPLEMENT_DATA,
null, null, null, null,
null, null, 0, null, null, 0,
RunMode.RUN_MODE_PARALLEL, RunMode.RUN_MODE_PARALLEL,
@ -292,7 +292,7 @@ public class ExecutorServiceTest {
Mockito.when(processService.queryReleaseSchedulerListByProcessDefinitionCode(processDefinitionCode)).thenReturn(oneSchedulerList()); Mockito.when(processService.queryReleaseSchedulerListByProcessDefinitionCode(processDefinitionCode)).thenReturn(oneSchedulerList());
Map<String, Object> result = executorService.execProcessInstance(loginUser, projectCode, Map<String, Object> result = executorService.execProcessInstance(loginUser, projectCode,
processDefinitionCode, cronTime, CommandType.COMPLEMENT_DATA, processDefinitionCode, "{\"complementStartDate\":\"2020-01-01 00:00:00\",\"complementEndDate\":\"2020-01-31 23:00:00\"}", CommandType.COMPLEMENT_DATA,
null, null, null, null,
null, null, 0, null, null, 0,
RunMode.RUN_MODE_PARALLEL, RunMode.RUN_MODE_PARALLEL,
@ -308,7 +308,7 @@ public class ExecutorServiceTest {
Mockito.when(monitorService.getServerListFromRegistry(true)).thenReturn(new ArrayList<>()); Mockito.when(monitorService.getServerListFromRegistry(true)).thenReturn(new ArrayList<>());
Map<String, Object> result = executorService.execProcessInstance(loginUser, projectCode, Map<String, Object> result = executorService.execProcessInstance(loginUser, projectCode,
processDefinitionCode, cronTime, CommandType.COMPLEMENT_DATA, processDefinitionCode, "{\"complementStartDate\":\"2020-01-01 00:00:00\",\"complementEndDate\":\"2020-01-31 23:00:00\"}", CommandType.COMPLEMENT_DATA,
null, null, null, null,
null, null, 0, null, null, 0,
RunMode.RUN_MODE_PARALLEL, RunMode.RUN_MODE_PARALLEL,

11
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
Unescape View File

@ -361,6 +361,11 @@ public final class Constants {
*/ */
public static final String CMDPARAM_COMPLEMENT_DATA_END_DATE = "complementEndDate"; public static final String CMDPARAM_COMPLEMENT_DATA_END_DATE = "complementEndDate";
/**
* complement data Schedule date
*/
public static final String CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST = "complementScheduleDateList";
/** /**
* complement date default cron string * complement date default cron string
*/ */
@ -809,4 +814,10 @@ public final class Constants {
* tenant * tenant
*/ */
public static final int TENANT_FULL_NAME_MAX_LENGTH = 30; public static final int TENANT_FULL_NAME_MAX_LENGTH = 30;
/**
* schedule time the amount of date data is too large, affecting the memory, so set 100
*/
public static final int SCHEDULE_TIME_MAX_LENGTH = 100;
} }

152
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
Unescape View File

@ -17,16 +17,10 @@
package org.apache.dolphinscheduler.server.master.runner; package org.apache.dolphinscheduler.server.master.runner;
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE; import com.google.common.collect.Lists;
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE; import org.apache.commons.collections.CollectionUtils;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVERY_START_NODE_STRING; import org.apache.commons.lang3.StringUtils;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVER_PROCESS_ID_STRING; import org.apache.commons.lang3.math.NumberUtils;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_NODES;
import static org.apache.dolphinscheduler.common.Constants.DEFAULT_WORKER_GROUP;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_BLOCKING;
import static org.apache.dolphinscheduler.plugin.task.api.enums.DataType.VARCHAR;
import static org.apache.dolphinscheduler.plugin.task.api.enums.Direct.IN;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.FailureStrategy; import org.apache.dolphinscheduler.common.enums.FailureStrategy;
@ -75,10 +69,8 @@ import org.apache.dolphinscheduler.service.alert.ProcessAlertManager;
import org.apache.dolphinscheduler.service.corn.CronUtils; import org.apache.dolphinscheduler.service.corn.CronUtils;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.queue.PeerTaskInstancePriorityQueue; import org.apache.dolphinscheduler.service.queue.PeerTaskInstancePriorityQueue;
import org.slf4j.Logger;
import org.apache.commons.collections.CollectionUtils; import org.slf4j.LoggerFactory;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
@ -96,10 +88,18 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import org.slf4j.Logger; import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE;
import org.slf4j.LoggerFactory; import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST;
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE;
import com.google.common.collect.Lists; import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVERY_START_NODE_STRING;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVER_PROCESS_ID_STRING;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_NODES;
import static org.apache.dolphinscheduler.common.Constants.COMMA;
import static org.apache.dolphinscheduler.common.Constants.DEFAULT_WORKER_GROUP;
import static org.apache.dolphinscheduler.common.Constants.YYYY_MM_DD_HH_MM_SS;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_BLOCKING;
import static org.apache.dolphinscheduler.plugin.task.api.enums.DataType.VARCHAR;
import static org.apache.dolphinscheduler.plugin.task.api.enums.Direct.IN;
/** /**
* Workflow execute task, used to execute a workflow instance. * Workflow execute task, used to execute a workflow instance.
@ -282,14 +282,14 @@ public class WorkflowExecuteRunnable implements Runnable {
public String getKey() { public String getKey() {
if (StringUtils.isNotEmpty(key) if (StringUtils.isNotEmpty(key)
|| this.processDefinition == null) { || this.processDefinition == null) {
return key; return key;
} }
key = String.format("%d_%d_%d", key = String.format("%d_%d_%d",
this.processDefinition.getCode(), this.processDefinition.getCode(),
this.processDefinition.getVersion(), this.processDefinition.getVersion(),
this.processInstance.getId()); this.processInstance.getId());
return key; return key;
} }
@ -503,7 +503,7 @@ public class WorkflowExecuteRunnable implements Runnable {
} else { } else {
ProcessInstance processInstance = this.processService.findProcessInstanceById(nextTaskInstance.getProcessInstanceId()); ProcessInstance processInstance = this.processService.findProcessInstanceById(nextTaskInstance.getProcessInstanceId());
this.processService.sendStartTask2Master(processInstance, nextTaskInstance.getId(), this.processService.sendStartTask2Master(processInstance, nextTaskInstance.getId(),
org.apache.dolphinscheduler.remote.command.CommandType.TASK_WAKEUP_EVENT_REQUEST); org.apache.dolphinscheduler.remote.command.CommandType.TASK_WAKEUP_EVENT_REQUEST);
} }
} }
} }
@ -526,12 +526,12 @@ public class WorkflowExecuteRunnable implements Runnable {
waitToRetryTaskInstanceMap.put(newTaskInstance.getTaskCode(), newTaskInstance); waitToRetryTaskInstanceMap.put(newTaskInstance.getTaskCode(), newTaskInstance);
if (!taskInstance.retryTaskIntervalOverTime()) { if (!taskInstance.retryTaskIntervalOverTime()) {
logger.info("failure task will be submitted: process id: {}, task instance code: {} state:{} retry times:{} / {}, interval:{}", logger.info("failure task will be submitted: process id: {}, task instance code: {} state:{} retry times:{} / {}, interval:{}",
processInstance.getId(), processInstance.getId(),
newTaskInstance.getTaskCode(), newTaskInstance.getTaskCode(),
newTaskInstance.getState(), newTaskInstance.getState(),
newTaskInstance.getRetryTimes(), newTaskInstance.getRetryTimes(),
newTaskInstance.getMaxRetryTimes(), newTaskInstance.getMaxRetryTimes(),
newTaskInstance.getRetryInterval()); newTaskInstance.getRetryInterval());
stateWheelExecuteThread.addTask4TimeoutCheck(processInstance, newTaskInstance); stateWheelExecuteThread.addTask4TimeoutCheck(processInstance, newTaskInstance);
stateWheelExecuteThread.addTask4RetryCheck(processInstance, newTaskInstance); stateWheelExecuteThread.addTask4RetryCheck(processInstance, newTaskInstance);
} else { } else {
@ -562,7 +562,7 @@ public class WorkflowExecuteRunnable implements Runnable {
logger.info("process instance update: {}", processInstanceId); logger.info("process instance update: {}", processInstanceId);
processInstance = processService.findProcessInstanceById(processInstanceId); processInstance = processService.findProcessInstanceById(processInstanceId);
processDefinition = processService.findProcessDefinition(processInstance.getProcessDefinitionCode(), processDefinition = processService.findProcessDefinition(processInstance.getProcessDefinitionCode(),
processInstance.getProcessDefinitionVersion()); processInstance.getProcessDefinitionVersion());
processInstance.setProcessDefinition(processDefinition); processInstance.setProcessDefinition(processDefinition);
} }
@ -591,8 +591,8 @@ public class WorkflowExecuteRunnable implements Runnable {
public boolean checkProcessInstance(StateEvent stateEvent) { public boolean checkProcessInstance(StateEvent stateEvent) {
if (this.processInstance.getId() != stateEvent.getProcessInstanceId()) { if (this.processInstance.getId() != stateEvent.getProcessInstanceId()) {
logger.error("mismatch process instance id: {}, state event:{}", logger.error("mismatch process instance id: {}, state event:{}",
this.processInstance.getId(), this.processInstance.getId(),
stateEvent); stateEvent);
return false; return false;
} }
return true; return true;
@ -774,7 +774,12 @@ public class WorkflowExecuteRunnable implements Runnable {
if (cmdParam.containsKey(Constants.CMD_PARAM_RECOVERY_START_NODE_STRING)) { if (cmdParam.containsKey(Constants.CMD_PARAM_RECOVERY_START_NODE_STRING)) {
cmdParam.remove(Constants.CMD_PARAM_RECOVERY_START_NODE_STRING); cmdParam.remove(Constants.CMD_PARAM_RECOVERY_START_NODE_STRING);
} }
cmdParam.replace(CMDPARAM_COMPLEMENT_DATA_START_DATE, DateUtils.format(scheduleDate, "yyyy-MM-dd HH:mm:ss", null)); if(cmdParam.containsKey(CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST)){
cmdParam.replace(CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST, cmdParam.get(CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST).substring(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST).indexOf(COMMA)+1));
}
if(cmdParam.containsKey(CMDPARAM_COMPLEMENT_DATA_START_DATE)){
cmdParam.replace(CMDPARAM_COMPLEMENT_DATA_START_DATE, DateUtils.format(scheduleDate, YYYY_MM_DD_HH_MM_SS, null));
}
command.setCommandParam(JSONUtils.toJsonString(cmdParam)); command.setCommandParam(JSONUtils.toJsonString(cmdParam));
command.setTaskDependType(processInstance.getTaskDependType()); command.setTaskDependType(processInstance.getTaskDependType());
command.setFailureStrategy(processInstance.getFailureStrategy()); command.setFailureStrategy(processInstance.getFailureStrategy());
@ -794,7 +799,7 @@ public class WorkflowExecuteRunnable implements Runnable {
private boolean needComplementProcess() { private boolean needComplementProcess() {
if (processInstance.isComplementData() if (processInstance.isComplementData()
&& Flag.NO == processInstance.getIsSubProcess()) { && Flag.NO == processInstance.getIsSubProcess()) {
return true; return true;
} }
return false; return false;
@ -874,7 +879,7 @@ public class WorkflowExecuteRunnable implements Runnable {
return; return;
} }
processDefinition = processService.findProcessDefinition(processInstance.getProcessDefinitionCode(), processDefinition = processService.findProcessDefinition(processInstance.getProcessDefinitionCode(),
processInstance.getProcessDefinitionVersion()); processInstance.getProcessDefinitionVersion());
processInstance.setProcessDefinition(processDefinition); processInstance.setProcessDefinition(processDefinition);
List<TaskInstance> recoverNodeList = getStartTaskInstanceList(processInstance.getCommandParam()); List<TaskInstance> recoverNodeList = getStartTaskInstanceList(processInstance.getCommandParam());
@ -894,7 +899,7 @@ public class WorkflowExecuteRunnable implements Runnable {
List<String> recoveryNodeCodeList = getRecoveryNodeCodeList(recoverNodeList); List<String> recoveryNodeCodeList = getRecoveryNodeCodeList(recoverNodeList);
List<String> startNodeNameList = parseStartNodeName(processInstance.getCommandParam()); List<String> startNodeNameList = parseStartNodeName(processInstance.getCommandParam());
ProcessDag processDag = generateFlowDag(taskNodeList, ProcessDag processDag = generateFlowDag(taskNodeList,
startNodeNameList, recoveryNodeCodeList, processInstance.getTaskDependType()); startNodeNameList, recoveryNodeCodeList, processInstance.getTaskDependType());
if (processDag == null) { if (processDag == null) {
logger.error("processDag is null"); logger.error("processDag is null");
return; return;
@ -956,15 +961,24 @@ public class WorkflowExecuteRunnable implements Runnable {
if (processInstance.isComplementData() && complementListDate.isEmpty()) { if (processInstance.isComplementData() && complementListDate.isEmpty()) {
Map<String, String> cmdParam = JSONUtils.toMap(processInstance.getCommandParam()); Map<String, String> cmdParam = JSONUtils.toMap(processInstance.getCommandParam());
if (cmdParam != null && cmdParam.containsKey(CMDPARAM_COMPLEMENT_DATA_START_DATE)) { if (cmdParam != null) {
// reset global params while there are start parameters // reset global params while there are start parameters
setGlobalParamIfCommanded(processDefinition, cmdParam); setGlobalParamIfCommanded(processDefinition, cmdParam);
Date start = DateUtils.stringToDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_START_DATE)); Date start = null;
Date end = DateUtils.stringToDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_END_DATE)); Date end = null;
if(cmdParam.containsKey(CMDPARAM_COMPLEMENT_DATA_START_DATE) && cmdParam.containsKey(CMDPARAM_COMPLEMENT_DATA_END_DATE)){
start = DateUtils.stringToDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_START_DATE));
end = DateUtils.stringToDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_END_DATE));
}
List<Schedule> schedules = processService.queryReleaseSchedulerListByProcessDefinitionCode(processInstance.getProcessDefinitionCode()); List<Schedule> schedules = processService.queryReleaseSchedulerListByProcessDefinitionCode(processInstance.getProcessDefinitionCode());
if (complementListDate.isEmpty() && needComplementProcess()) { if (complementListDate.isEmpty() && needComplementProcess()) {
complementListDate = CronUtils.getSelfFireDateList(start, end, schedules); if(start != null && end != null){
complementListDate = CronUtils.getSelfFireDateList(start, end, schedules);
}
if(cmdParam.containsKey(CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST)){
complementListDate = CronUtils.getSelfScheduleDateList(cmdParam);
}
logger.info(" process definition code:{} complement data: {}", logger.info(" process definition code:{} complement data: {}",
processInstance.getProcessDefinitionCode(), complementListDate); processInstance.getProcessDefinitionCode(), complementListDate);
@ -996,15 +1010,15 @@ public class WorkflowExecuteRunnable implements Runnable {
taskProcessor.init(taskInstance, processInstance); taskProcessor.init(taskInstance, processInstance);
if (taskInstance.getState() == ExecutionStatus.RUNNING_EXECUTION if (taskInstance.getState() == ExecutionStatus.RUNNING_EXECUTION
&& taskProcessor.getType().equalsIgnoreCase(Constants.COMMON_TASK_TYPE)) { && taskProcessor.getType().equalsIgnoreCase(Constants.COMMON_TASK_TYPE)) {
notifyProcessHostUpdate(taskInstance); notifyProcessHostUpdate(taskInstance);
} }
boolean submit = taskProcessor.action(TaskAction.SUBMIT); boolean submit = taskProcessor.action(TaskAction.SUBMIT);
if (!submit) { if (!submit) {
logger.error("process id:{} name:{} submit standby task id:{} name:{} failed!", logger.error("process id:{} name:{} submit standby task id:{} name:{} failed!",
processInstance.getId(), processInstance.getName(), processInstance.getId(), processInstance.getName(),
taskInstance.getId(), taskInstance.getName()); taskInstance.getId(), taskInstance.getName());
return null; return null;
} }
@ -1475,10 +1489,10 @@ public class WorkflowExecuteRunnable implements Runnable {
*/ */
private ExecutionStatus runningState(ExecutionStatus state) { private ExecutionStatus runningState(ExecutionStatus state) {
if (state == ExecutionStatus.READY_STOP if (state == ExecutionStatus.READY_STOP
|| state == ExecutionStatus.READY_PAUSE || state == ExecutionStatus.READY_PAUSE
|| state == ExecutionStatus.WAITING_THREAD || state == ExecutionStatus.WAITING_THREAD
|| state == ExecutionStatus.READY_BLOCK || state == ExecutionStatus.READY_BLOCK
|| state == ExecutionStatus.DELAY_EXECUTION) { || state == ExecutionStatus.DELAY_EXECUTION) {
// if the running task is not completed, the state remains unchanged // if the running task is not completed, the state remains unchanged
return state; return state;
} else { } else {
@ -1514,8 +1528,8 @@ public class WorkflowExecuteRunnable implements Runnable {
} }
if (processInstance.getFailureStrategy() == FailureStrategy.CONTINUE) { if (processInstance.getFailureStrategy() == FailureStrategy.CONTINUE) {
return readyToSubmitTaskQueue.size() == 0 return readyToSubmitTaskQueue.size() == 0
&& activeTaskProcessorMaps.size() == 0 && activeTaskProcessorMaps.size() == 0
&& waitToRetryTaskInstanceMap.size() == 0; && waitToRetryTaskInstanceMap.size() == 0;
} }
} }
return false; return false;
@ -1546,9 +1560,9 @@ public class WorkflowExecuteRunnable implements Runnable {
List<TaskInstance> pauseList = getCompleteTaskByState(ExecutionStatus.PAUSE); List<TaskInstance> pauseList = getCompleteTaskByState(ExecutionStatus.PAUSE);
if (CollectionUtils.isNotEmpty(pauseList) if (CollectionUtils.isNotEmpty(pauseList)
|| processInstance.isBlocked() || processInstance.isBlocked()
|| !isComplementEnd() || !isComplementEnd()
|| readyToSubmitTaskQueue.size() > 0) { || readyToSubmitTaskQueue.size() > 0) {
return ExecutionStatus.PAUSE; return ExecutionStatus.PAUSE;
} else { } else {
return ExecutionStatus.SUCCESS; return ExecutionStatus.SUCCESS;
@ -1613,9 +1627,9 @@ public class WorkflowExecuteRunnable implements Runnable {
List<TaskInstance> killList = getCompleteTaskByState(ExecutionStatus.KILL); List<TaskInstance> killList = getCompleteTaskByState(ExecutionStatus.KILL);
List<TaskInstance> failList = getCompleteTaskByState(ExecutionStatus.FAILURE); List<TaskInstance> failList = getCompleteTaskByState(ExecutionStatus.FAILURE);
if (CollectionUtils.isNotEmpty(stopList) if (CollectionUtils.isNotEmpty(stopList)
|| CollectionUtils.isNotEmpty(killList) || CollectionUtils.isNotEmpty(killList)
|| CollectionUtils.isNotEmpty(failList) || CollectionUtils.isNotEmpty(failList)
|| !isComplementEnd()) { || !isComplementEnd()) {
return ExecutionStatus.STOP; return ExecutionStatus.STOP;
} else { } else {
return ExecutionStatus.SUCCESS; return ExecutionStatus.SUCCESS;
@ -1673,10 +1687,10 @@ public class WorkflowExecuteRunnable implements Runnable {
ExecutionStatus state = getProcessInstanceState(processInstance); ExecutionStatus state = getProcessInstanceState(processInstance);
if (processInstance.getState() != state) { if (processInstance.getState() != state) {
logger.info( logger.info(
"work flow process instance [id: {}, name:{}], state change from {} to {}, cmd type: {}", "work flow process instance [id: {}, name:{}], state change from {} to {}, cmd type: {}",
processInstance.getId(), processInstance.getName(), processInstance.getId(), processInstance.getName(),
processInstance.getState(), state, processInstance.getState(), state,
processInstance.getCommandType()); processInstance.getCommandType());
processInstance.setState(state); processInstance.setState(state);
if (state.typeIsFinished()) { if (state.typeIsFinished()) {
@ -1701,10 +1715,10 @@ public class WorkflowExecuteRunnable implements Runnable {
ExecutionStatus state = stateEvent.getExecutionStatus(); ExecutionStatus state = stateEvent.getExecutionStatus();
if (processInstance.getState() != state) { if (processInstance.getState() != state) {
logger.info( logger.info(
"work flow process instance [id: {}, name:{}], state change from {} to {}, cmd type: {}", "work flow process instance [id: {}, name:{}], state change from {} to {}, cmd type: {}",
processInstance.getId(), processInstance.getName(), processInstance.getId(), processInstance.getName(),
processInstance.getState(), state, processInstance.getState(), state,
processInstance.getCommandType()); processInstance.getCommandType());
processInstance.setState(state); processInstance.setState(state);
if (state.typeIsFinished()) { if (state.typeIsFinished()) {
@ -1751,14 +1765,14 @@ public class WorkflowExecuteRunnable implements Runnable {
*/ */
private void removeTaskFromStandbyList(TaskInstance taskInstance) { private void removeTaskFromStandbyList(TaskInstance taskInstance) {
logger.info("remove task from stand by list, id: {} name:{}", logger.info("remove task from stand by list, id: {} name:{}",
taskInstance.getId(), taskInstance.getId(),
taskInstance.getName()); taskInstance.getName());
try { try {
readyToSubmitTaskQueue.remove(taskInstance); readyToSubmitTaskQueue.remove(taskInstance);
} catch (Exception e) { } catch (Exception e) {
logger.error("remove task instance from readyToSubmitTaskQueue error, task id:{}, Name: {}", logger.error("remove task instance from readyToSubmitTaskQueue error, task id:{}, Name: {}",
taskInstance.getId(), taskInstance.getId(),
taskInstance.getName(), e); taskInstance.getName(), e);
} }
} }
@ -1781,7 +1795,7 @@ public class WorkflowExecuteRunnable implements Runnable {
*/ */
private void killAllTasks() { private void killAllTasks() {
logger.info("kill called on process instance id: {}, num: {}", processInstance.getId(), logger.info("kill called on process instance id: {}, num: {}", processInstance.getId(),
activeTaskProcessorMaps.size()); activeTaskProcessorMaps.size());
if (readyToSubmitTaskQueue.size() > 0) { if (readyToSubmitTaskQueue.size() > 0) {
readyToSubmitTaskQueue.clear(); readyToSubmitTaskQueue.clear();
@ -2070,4 +2084,4 @@ public class WorkflowExecuteRunnable implements Runnable {
break; break;
} }
} }
} }

53
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/corn/CronUtils.java
Unescape View File

@ -17,22 +17,19 @@
package org.apache.dolphinscheduler.service.corn; package org.apache.dolphinscheduler.service.corn;
import static org.apache.dolphinscheduler.service.corn.CycleFactory.day; import com.cronutils.model.Cron;
import static org.apache.dolphinscheduler.service.corn.CycleFactory.hour; import com.cronutils.model.definition.CronDefinitionBuilder;
import static org.apache.dolphinscheduler.service.corn.CycleFactory.min; import com.cronutils.parser.CronParser;
import static org.apache.dolphinscheduler.service.corn.CycleFactory.month; import org.apache.commons.collections.CollectionUtils;
import static org.apache.dolphinscheduler.service.corn.CycleFactory.week;
import static org.apache.dolphinscheduler.service.corn.CycleFactory.year;
import static com.cronutils.model.CronType.QUARTZ;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CycleEnum; import org.apache.dolphinscheduler.common.enums.CycleEnum;
import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.dao.entity.Schedule; import org.apache.dolphinscheduler.dao.entity.Schedule;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
import org.apache.commons.collections.CollectionUtils; import org.quartz.CronExpression;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.text.ParseException; import java.text.ParseException;
import java.util.ArrayList; import java.util.ArrayList;
@ -41,14 +38,17 @@ import java.util.Collections;
import java.util.Date; import java.util.Date;
import java.util.GregorianCalendar; import java.util.GregorianCalendar;
import java.util.List; import java.util.List;
import java.util.Map;
import org.quartz.CronExpression; import static com.cronutils.model.CronType.QUARTZ;
import org.slf4j.Logger; import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST;
import org.slf4j.LoggerFactory; import static org.apache.dolphinscheduler.common.Constants.COMMA;
import static org.apache.dolphinscheduler.service.corn.CycleFactory.day;
import com.cronutils.model.Cron; import static org.apache.dolphinscheduler.service.corn.CycleFactory.hour;
import com.cronutils.model.definition.CronDefinitionBuilder; import static org.apache.dolphinscheduler.service.corn.CycleFactory.min;
import com.cronutils.parser.CronParser; import static org.apache.dolphinscheduler.service.corn.CycleFactory.month;
import static org.apache.dolphinscheduler.service.corn.CycleFactory.week;
import static org.apache.dolphinscheduler.service.corn.CycleFactory.year;
/** /**
* // todo: this utils is heavy, it rely on quartz and corn-utils. * // todo: this utils is heavy, it rely on quartz and corn-utils.
@ -283,4 +283,21 @@ public class CronUtils {
return end.getTime(); return end.getTime();
} }
/**
* get Schedule Date
* @param param
* @return date list
*/
public static List<Date> getSelfScheduleDateList(Map<String, String> param){
List<Date> result = new ArrayList<>();
String scheduleDates = param.get(CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST);
if(StringUtils.isNotEmpty(scheduleDates)){
for (String stringDate : scheduleDates.split(COMMA)) {
result.add(DateUtils.stringToDate(stringDate));
}
return result;
}
return null;
}
} }

Write Preview
Loading…
Cancel
Save