|
|
@ -68,7 +68,7 @@ public class MasterExecThread implements Runnable { |
|
|
|
/** |
|
|
|
/** |
|
|
|
* runing TaskNode |
|
|
|
* runing TaskNode |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
private final Map<MasterBaseTaskExecThread,Future<Boolean>> activeTaskNode = new ConcurrentHashMap<MasterBaseTaskExecThread,Future<Boolean>>(); |
|
|
|
private final Map<MasterBaseTaskExecThread,Future<Boolean>> activeTaskNode = new ConcurrentHashMap<>(); |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
/** |
|
|
|
* task exec service |
|
|
|
* task exec service |
|
|
@ -78,7 +78,7 @@ public class MasterExecThread implements Runnable { |
|
|
|
/** |
|
|
|
/** |
|
|
|
* submit failure nodes |
|
|
|
* submit failure nodes |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
private Boolean taskFailedSubmit = false; |
|
|
|
private boolean taskFailedSubmit = false; |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
/** |
|
|
|
* recover node id list |
|
|
|
* recover node id list |
|
|
@ -652,7 +652,7 @@ public class MasterExecThread implements Runnable { |
|
|
|
continue; |
|
|
|
continue; |
|
|
|
} |
|
|
|
} |
|
|
|
if(task.getState().typeIsPause() || task.getState().typeIsCancel()){ |
|
|
|
if(task.getState().typeIsPause() || task.getState().typeIsCancel()){ |
|
|
|
logger.info("task {} stopped, the state is {}", task.getName(), task.getState().toString()); |
|
|
|
logger.info("task {} stopped, the state is {}", task.getName(), task.getState()); |
|
|
|
}else{ |
|
|
|
}else{ |
|
|
|
addTaskToStandByList(task); |
|
|
|
addTaskToStandByList(task); |
|
|
|
} |
|
|
|
} |
|
|
@ -685,11 +685,12 @@ public class MasterExecThread implements Runnable { |
|
|
|
} |
|
|
|
} |
|
|
|
ExecutionStatus depTaskState = completeTaskList.get(depsNode).getState(); |
|
|
|
ExecutionStatus depTaskState = completeTaskList.get(depsNode).getState(); |
|
|
|
// conditions task would not return failed.
|
|
|
|
// conditions task would not return failed.
|
|
|
|
if(depTaskState.typeIsFailure()){ |
|
|
|
if(depTaskState.typeIsFailure() |
|
|
|
if(!haveConditionsAfterNode(depsNode) && !dag.getNode(depsNode).isConditionsTask()){ |
|
|
|
&& !haveConditionsAfterNode(depsNode) |
|
|
|
return DependResult.FAILED; |
|
|
|
&& !dag.getNode(depsNode).isConditionsTask()){ |
|
|
|
} |
|
|
|
return DependResult.FAILED; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
if(depTaskState.typeIsPause() || depTaskState.typeIsCancel()){ |
|
|
|
if(depTaskState.typeIsPause() || depTaskState.typeIsCancel()){ |
|
|
|
return DependResult.WAITING; |
|
|
|
return DependResult.WAITING; |
|
|
|
} |
|
|
|
} |
|
|
@ -737,7 +738,7 @@ public class MasterExecThread implements Runnable { |
|
|
|
* |
|
|
|
* |
|
|
|
* @return Boolean whether has failed task |
|
|
|
* @return Boolean whether has failed task |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
private Boolean hasFailedTask(){ |
|
|
|
private boolean hasFailedTask(){ |
|
|
|
|
|
|
|
|
|
|
|
if(this.taskFailedSubmit){ |
|
|
|
if(this.taskFailedSubmit){ |
|
|
|
return true; |
|
|
|
return true; |
|
|
@ -753,7 +754,7 @@ public class MasterExecThread implements Runnable { |
|
|
|
* |
|
|
|
* |
|
|
|
* @return Boolean whether process instance failed |
|
|
|
* @return Boolean whether process instance failed |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
private Boolean processFailed(){ |
|
|
|
private boolean processFailed(){ |
|
|
|
if(hasFailedTask()) { |
|
|
|
if(hasFailedTask()) { |
|
|
|
if(processInstance.getFailureStrategy() == FailureStrategy.END){ |
|
|
|
if(processInstance.getFailureStrategy() == FailureStrategy.END){ |
|
|
|
return true; |
|
|
|
return true; |
|
|
@ -769,9 +770,9 @@ public class MasterExecThread implements Runnable { |
|
|
|
* whether task for waiting thread |
|
|
|
* whether task for waiting thread |
|
|
|
* @return Boolean whether has waiting thread task |
|
|
|
* @return Boolean whether has waiting thread task |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
private Boolean hasWaitingThreadTask(){ |
|
|
|
private boolean hasWaitingThreadTask(){ |
|
|
|
List<TaskInstance> waitingList = getCompleteTaskByState(ExecutionStatus.WAITTING_THREAD); |
|
|
|
List<TaskInstance> waitingList = getCompleteTaskByState(ExecutionStatus.WAITTING_THREAD); |
|
|
|
return waitingList.size() > 0; |
|
|
|
return CollectionUtils.isNotEmpty(waitingList); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
/** |
|
|
@ -787,7 +788,7 @@ public class MasterExecThread implements Runnable { |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
List<TaskInstance> pauseList = getCompleteTaskByState(ExecutionStatus.PAUSE); |
|
|
|
List<TaskInstance> pauseList = getCompleteTaskByState(ExecutionStatus.PAUSE); |
|
|
|
if(pauseList.size() > 0 |
|
|
|
if(CollectionUtils.isNotEmpty(pauseList) |
|
|
|
|| !isComplementEnd() |
|
|
|
|| !isComplementEnd() |
|
|
|
|| readyToSubmitTaskList.size() > 0){ |
|
|
|
|| readyToSubmitTaskList.size() > 0){ |
|
|
|
return ExecutionStatus.PAUSE; |
|
|
|
return ExecutionStatus.PAUSE; |
|
|
@ -827,7 +828,8 @@ public class MasterExecThread implements Runnable { |
|
|
|
if(state == ExecutionStatus.READY_STOP){ |
|
|
|
if(state == ExecutionStatus.READY_STOP){ |
|
|
|
List<TaskInstance> stopList = getCompleteTaskByState(ExecutionStatus.STOP); |
|
|
|
List<TaskInstance> stopList = getCompleteTaskByState(ExecutionStatus.STOP); |
|
|
|
List<TaskInstance> killList = getCompleteTaskByState(ExecutionStatus.KILL); |
|
|
|
List<TaskInstance> killList = getCompleteTaskByState(ExecutionStatus.KILL); |
|
|
|
if(stopList.size() > 0 || killList.size() > 0 || !isComplementEnd()){ |
|
|
|
if(CollectionUtils.isNotEmpty(stopList) |
|
|
|
|
|
|
|
|| CollectionUtils.isNotEmpty(killList) || !isComplementEnd()){ |
|
|
|
return ExecutionStatus.STOP; |
|
|
|
return ExecutionStatus.STOP; |
|
|
|
}else{ |
|
|
|
}else{ |
|
|
|
return ExecutionStatus.SUCCESS; |
|
|
|
return ExecutionStatus.SUCCESS; |
|
|
@ -852,7 +854,7 @@ public class MasterExecThread implements Runnable { |
|
|
|
* whether complement end |
|
|
|
* whether complement end |
|
|
|
* @return Boolean whether is complement end |
|
|
|
* @return Boolean whether is complement end |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
private Boolean isComplementEnd() { |
|
|
|
private boolean isComplementEnd() { |
|
|
|
if(!processInstance.isComplementData()){ |
|
|
|
if(!processInstance.isComplementData()){ |
|
|
|
return true; |
|
|
|
return true; |
|
|
|
} |
|
|
|
} |
|
|
@ -877,8 +879,8 @@ public class MasterExecThread implements Runnable { |
|
|
|
logger.info( |
|
|
|
logger.info( |
|
|
|
"work flow process instance [id: {}, name:{}], state change from {} to {}, cmd type: {}", |
|
|
|
"work flow process instance [id: {}, name:{}], state change from {} to {}, cmd type: {}", |
|
|
|
processInstance.getId(), processInstance.getName(), |
|
|
|
processInstance.getId(), processInstance.getName(), |
|
|
|
processInstance.getState().toString(), state.toString(), |
|
|
|
processInstance.getState(), state, |
|
|
|
processInstance.getCommandType().toString()); |
|
|
|
processInstance.getCommandType()); |
|
|
|
processInstance.setState(state); |
|
|
|
processInstance.setState(state); |
|
|
|
ProcessInstance instance = processService.findProcessInstanceById(processInstance.getId()); |
|
|
|
ProcessInstance instance = processService.findProcessInstanceById(processInstance.getId()); |
|
|
|
instance.setState(state); |
|
|
|
instance.setState(state); |
|
|
@ -894,8 +896,7 @@ public class MasterExecThread implements Runnable { |
|
|
|
* @return DependResult |
|
|
|
* @return DependResult |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
private DependResult getDependResultForTask(TaskInstance taskInstance){ |
|
|
|
private DependResult getDependResultForTask(TaskInstance taskInstance){ |
|
|
|
DependResult inner = isTaskDepsComplete(taskInstance.getName()); |
|
|
|
return isTaskDepsComplete(taskInstance.getName()); |
|
|
|
return inner; |
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
/** |
|
|
@ -920,7 +921,7 @@ public class MasterExecThread implements Runnable { |
|
|
|
* has retry task in standby |
|
|
|
* has retry task in standby |
|
|
|
* @return Boolean whether has retry task in standby |
|
|
|
* @return Boolean whether has retry task in standby |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
private Boolean hasRetryTaskInStandBy(){ |
|
|
|
private boolean hasRetryTaskInStandBy(){ |
|
|
|
for (Map.Entry<String, TaskInstance> entry: readyToSubmitTaskList.entrySet()) { |
|
|
|
for (Map.Entry<String, TaskInstance> entry: readyToSubmitTaskList.entrySet()) { |
|
|
|
if(entry.getValue().getState().typeIsFailure()){ |
|
|
|
if(entry.getValue().getState().typeIsFailure()){ |
|
|
|
return true; |
|
|
|
return true; |
|
|
@ -958,7 +959,7 @@ public class MasterExecThread implements Runnable { |
|
|
|
continue; |
|
|
|
continue; |
|
|
|
} |
|
|
|
} |
|
|
|
logger.info("task :{}, id:{} complete, state is {} ", |
|
|
|
logger.info("task :{}, id:{} complete, state is {} ", |
|
|
|
task.getName(), task.getId(), task.getState().toString()); |
|
|
|
task.getName(), task.getId(), task.getState()); |
|
|
|
// node success , post node submit
|
|
|
|
// node success , post node submit
|
|
|
|
if(task.getState() == ExecutionStatus.SUCCESS){ |
|
|
|
if(task.getState() == ExecutionStatus.SUCCESS){ |
|
|
|
completeTaskList.put(task.getName(), task); |
|
|
|
completeTaskList.put(task.getName(), task); |
|
|
@ -990,7 +991,7 @@ public class MasterExecThread implements Runnable { |
|
|
|
completeTaskList.put(task.getName(), task); |
|
|
|
completeTaskList.put(task.getName(), task); |
|
|
|
} |
|
|
|
} |
|
|
|
// send alert
|
|
|
|
// send alert
|
|
|
|
if(this.recoverToleranceFaultTaskList.size() > 0){ |
|
|
|
if(CollectionUtils.isNotEmpty(this.recoverToleranceFaultTaskList)){ |
|
|
|
alertManager.sendAlertWorkerToleranceFault(processInstance, recoverToleranceFaultTaskList); |
|
|
|
alertManager.sendAlertWorkerToleranceFault(processInstance, recoverToleranceFaultTaskList); |
|
|
|
this.recoverToleranceFaultTaskList.clear(); |
|
|
|
this.recoverToleranceFaultTaskList.clear(); |
|
|
|
} |
|
|
|
} |
|
|
@ -1034,10 +1035,7 @@ public class MasterExecThread implements Runnable { |
|
|
|
Date now = new Date(); |
|
|
|
Date now = new Date(); |
|
|
|
long runningTime = DateUtils.diffMin(now, processInstance.getStartTime()); |
|
|
|
long runningTime = DateUtils.diffMin(now, processInstance.getStartTime()); |
|
|
|
|
|
|
|
|
|
|
|
if(runningTime > processInstance.getTimeout()){ |
|
|
|
return runningTime > processInstance.getTimeout(); |
|
|
|
return true; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
return false; |
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
/** |
|
|
@ -1081,22 +1079,19 @@ public class MasterExecThread implements Runnable { |
|
|
|
* @param taskInstance task instance |
|
|
|
* @param taskInstance task instance |
|
|
|
* @return Boolean |
|
|
|
* @return Boolean |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
private Boolean retryTaskIntervalOverTime(TaskInstance taskInstance){ |
|
|
|
private boolean retryTaskIntervalOverTime(TaskInstance taskInstance){ |
|
|
|
if(taskInstance.getState() != ExecutionStatus.FAILURE){ |
|
|
|
if(taskInstance.getState() != ExecutionStatus.FAILURE){ |
|
|
|
return Boolean.TRUE; |
|
|
|
return true; |
|
|
|
} |
|
|
|
} |
|
|
|
if(taskInstance.getId() == 0 || |
|
|
|
if(taskInstance.getId() == 0 || |
|
|
|
taskInstance.getMaxRetryTimes() ==0 || |
|
|
|
taskInstance.getMaxRetryTimes() ==0 || |
|
|
|
taskInstance.getRetryInterval() == 0 ){ |
|
|
|
taskInstance.getRetryInterval() == 0 ){ |
|
|
|
return Boolean.TRUE; |
|
|
|
return true; |
|
|
|
} |
|
|
|
} |
|
|
|
Date now = new Date(); |
|
|
|
Date now = new Date(); |
|
|
|
long failedTimeInterval = DateUtils.differSec(now, taskInstance.getEndTime()); |
|
|
|
long failedTimeInterval = DateUtils.differSec(now, taskInstance.getEndTime()); |
|
|
|
// task retry does not over time, return false
|
|
|
|
// task retry does not over time, return false
|
|
|
|
if(taskInstance.getRetryInterval() * SEC_2_MINUTES_TIME_UNIT >= failedTimeInterval){ |
|
|
|
return taskInstance.getRetryInterval() * SEC_2_MINUTES_TIME_UNIT < failedTimeInterval; |
|
|
|
return Boolean.FALSE; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
return Boolean.TRUE; |
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
/** |
|
|
@ -1189,7 +1184,7 @@ public class MasterExecThread implements Runnable { |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
private List<String> getRecoveryNodeNameList(){ |
|
|
|
private List<String> getRecoveryNodeNameList(){ |
|
|
|
List<String> recoveryNodeNameList = new ArrayList<>(); |
|
|
|
List<String> recoveryNodeNameList = new ArrayList<>(); |
|
|
|
if(recoverNodeIdList.size() > 0) { |
|
|
|
if(CollectionUtils.isNotEmpty(recoverNodeIdList)) { |
|
|
|
for (TaskInstance task : recoverNodeIdList) { |
|
|
|
for (TaskInstance task : recoverNodeIdList) { |
|
|
|
recoveryNodeNameList.add(task.getName()); |
|
|
|
recoveryNodeNameList.add(task.getName()); |
|
|
|
} |
|
|
|
} |
|
|
|