Browse Source

update process time out

pull/2/head
baoliang 6 years ago
parent
commit
a7a4a31643
  1. 35
      escheduler-dao/src/main/java/cn/escheduler/dao/AlertDao.java
  2. 32
      escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java
  3. 56
      escheduler-server/src/main/java/cn/escheduler/server/utils/AlertManager.java
  4. 4
      escheduler-server/src/test/java/cn/escheduler/server/master/AlertManagerTest.java

35
escheduler-dao/src/main/java/cn/escheduler/dao/AlertDao.java

@ -23,6 +23,8 @@ import cn.escheduler.dao.datasource.ConnectionFactory;
import cn.escheduler.dao.mapper.AlertMapper; import cn.escheduler.dao.mapper.AlertMapper;
import cn.escheduler.dao.mapper.UserAlertGroupMapper; import cn.escheduler.dao.mapper.UserAlertGroupMapper;
import cn.escheduler.dao.model.Alert; import cn.escheduler.dao.model.Alert;
import cn.escheduler.dao.model.ProcessDefinition;
import cn.escheduler.dao.model.ProcessInstance;
import cn.escheduler.dao.model.User; import cn.escheduler.dao.model.User;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -83,8 +85,9 @@ public class AlertDao extends AbstractBaseDao {
*/ */
public void sendServerStopedAlert(int alertgroupId,String host,String serverType){ public void sendServerStopedAlert(int alertgroupId,String host,String serverType){
Alert alert = new Alert(); Alert alert = new Alert();
String content = String.format("[{'type':'%s','host':'%s','event':'服务挂掉','警告级别':'严重'}]",serverType,host); String content = String.format("[{'type':'%s','host':'%s','event':'server down','warning level':'serious'}]",
alert.setTitle("容错告警"); serverType, host);
alert.setTitle("Fault tolerance warning");
alert.setShowType(ShowType.TABLE); alert.setShowType(ShowType.TABLE);
alert.setContent(content); alert.setContent(content);
alert.setAlertType(AlertType.EMAIL); alert.setAlertType(AlertType.EMAIL);
@ -94,6 +97,34 @@ public class AlertDao extends AbstractBaseDao {
alertMapper.insert(alert); alertMapper.insert(alert);
} }
/**
* process time out alert
* @param processInstance
* @param processDefinition
*/
public void sendProcessTimeoutAlert(ProcessInstance processInstance, ProcessDefinition processDefinition){
int alertgroupId = processInstance.getWarningGroupId();
String receivers = processDefinition.getReceivers();
String receiversCc = processDefinition.getReceiversCc();
Alert alert = new Alert();
String content = String.format("[{'id':'%d','name':'%s','event':'timeout','warnLevel':'middle'}]",
processInstance.getId(), processInstance.getName());
alert.setTitle("Process Timeout Warn");
alert.setShowType(ShowType.TABLE);
alert.setContent(content);
alert.setAlertType(AlertType.EMAIL);
alert.setAlertGroupId(alertgroupId);
if (StringUtils.isNotEmpty(receivers)) {
alert.setReceivers(receivers);
}
if (StringUtils.isNotEmpty(receiversCc)) {
alert.setReceiversCc(receiversCc);
}
alert.setCreateTime(new Date());
alert.setUpdateTime(new Date());
alertMapper.insert(alert);
}
/** /**
* task timeout warn * task timeout warn
*/ */

32
escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java

@ -258,7 +258,7 @@ public class MasterExecThread implements Runnable {
processDao.createRecoveryWaitingThreadCommand(null, processInstance); processDao.createRecoveryWaitingThreadCommand(null, processInstance);
} }
List<TaskInstance> taskInstances = processDao.findValidTaskListByProcessId(processInstance.getId()); List<TaskInstance> taskInstances = processDao.findValidTaskListByProcessId(processInstance.getId());
alertManager.sendWarnningOfProcessInstance(processInstance, taskInstances); alertManager.sendAlertProcessInstance(processInstance, taskInstances);
} }
@ -775,8 +775,15 @@ public class MasterExecThread implements Runnable {
private void runProcess(){ private void runProcess(){
// submit start node // submit start node
submitPostNode(null); submitPostNode(null);
// submitStandByTask(); boolean sendTimeWarning = false;
while(!processInstance.IsProcessInstanceStop()){ while(!processInstance.IsProcessInstanceStop()){
// send warning email if process time out.
if( !sendTimeWarning && checkProcessTimeOut(processInstance) ){
alertManager.sendProcessTimeoutAlert(processInstance,
processDao.findProcessDefineById(processInstance.getProcessDefinitionId()));
sendTimeWarning = true;
}
Set<MasterBaseTaskExecThread> keys = activeTaskNode.keySet(); Set<MasterBaseTaskExecThread> keys = activeTaskNode.keySet();
for (MasterBaseTaskExecThread taskExecThread : keys) { for (MasterBaseTaskExecThread taskExecThread : keys) {
Future<Boolean> future = activeTaskNode.get(taskExecThread); Future<Boolean> future = activeTaskNode.get(taskExecThread);
@ -821,7 +828,7 @@ public class MasterExecThread implements Runnable {
} }
// send alert // send alert
if(this.recoverToleranceFaultTaskList.size() > 0){ if(this.recoverToleranceFaultTaskList.size() > 0){
alertManager.sendWarnningWorkerleranceFault(processInstance, recoverToleranceFaultTaskList); alertManager.sendAlertWorkerToleranceFault(processInstance, recoverToleranceFaultTaskList);
this.recoverToleranceFaultTaskList.clear(); this.recoverToleranceFaultTaskList.clear();
} }
// updateProcessInstance completed task status // updateProcessInstance completed task status
@ -851,6 +858,25 @@ public class MasterExecThread implements Runnable {
logger.info("process:{} end, state :{}", processInstance.getId(), processInstance.getState()); logger.info("process:{} end, state :{}", processInstance.getId(), processInstance.getState());
} }
/**
* check process time out
* @param processInstance
* @return
*/
private boolean checkProcessTimeOut(ProcessInstance processInstance) {
if(processInstance.getTimeout() == 0 ){
return false;
}
Date now = new Date();
long runningTime = DateUtils.differMs(now, processInstance.getStartTime());
if(runningTime > processInstance.getTimeout()){
return true;
}
return false;
}
private boolean canSubmitTaskToQueue() { private boolean canSubmitTaskToQueue() {
return OSUtils.checkResource(conf, true); return OSUtils.checkResource(conf, true);
} }

56
escheduler-server/src/main/java/cn/escheduler/server/utils/AlertManager.java

@ -26,6 +26,7 @@ import cn.escheduler.common.utils.JSONUtils;
import cn.escheduler.dao.AlertDao; import cn.escheduler.dao.AlertDao;
import cn.escheduler.dao.DaoFactory; import cn.escheduler.dao.DaoFactory;
import cn.escheduler.dao.model.Alert; import cn.escheduler.dao.model.Alert;
import cn.escheduler.dao.model.ProcessDefinition;
import cn.escheduler.dao.model.ProcessInstance; import cn.escheduler.dao.model.ProcessInstance;
import cn.escheduler.dao.model.TaskInstance; import cn.escheduler.dao.model.TaskInstance;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -54,27 +55,27 @@ public class AlertManager {
private String getCommandCnName(CommandType commandType) { private String getCommandCnName(CommandType commandType) {
switch (commandType) { switch (commandType) {
case RECOVER_TOLERANCE_FAULT_PROCESS: case RECOVER_TOLERANCE_FAULT_PROCESS:
return "恢复容错"; return "recover tolerance fault process";
case RECOVER_SUSPENDED_PROCESS: case RECOVER_SUSPENDED_PROCESS:
return "恢复暂停流程"; return "recover suspended process";
case START_CURRENT_TASK_PROCESS: case START_CURRENT_TASK_PROCESS:
return "从当前节点开始执行"; return "start current task process";
case START_FAILURE_TASK_PROCESS: case START_FAILURE_TASK_PROCESS:
return "从失败节点开始执行"; return "start failure task process";
case START_PROCESS: case START_PROCESS:
return "启动工作流"; return "start process";
case REPEAT_RUNNING: case REPEAT_RUNNING:
return "重跑"; return "repeat running";
case SCHEDULER: case SCHEDULER:
return "定时执行"; return "scheduler";
case COMPLEMENT_DATA: case COMPLEMENT_DATA:
return "补数"; return "complement data";
case PAUSE: case PAUSE:
return "暂停工作流"; return "pause";
case STOP: case STOP:
return "停止工作流"; return "stop";
default: default:
return "未知的命令类型"; return "unknown type";
} }
} }
@ -124,14 +125,14 @@ public class AlertManager {
continue; continue;
} }
LinkedHashMap<String, String> failedTaskMap = new LinkedHashMap(); LinkedHashMap<String, String> failedTaskMap = new LinkedHashMap();
failedTaskMap.put("任务id", String.valueOf(task.getId())); failedTaskMap.put("task id", String.valueOf(task.getId()));
failedTaskMap.put("任务名称", task.getName()); failedTaskMap.put("task name", task.getName());
failedTaskMap.put("任务类型", task.getTaskType()); failedTaskMap.put("task type", task.getTaskType());
failedTaskMap.put("任务状态", task.getState().toString()); failedTaskMap.put("task state", task.getState().toString());
failedTaskMap.put("任务开始时间", DateUtils.dateToString(task.getStartTime())); failedTaskMap.put("task start time", DateUtils.dateToString(task.getStartTime()));
failedTaskMap.put("任务结束时间", DateUtils.dateToString(task.getEndTime())); failedTaskMap.put("task end time", DateUtils.dateToString(task.getEndTime()));
failedTaskMap.put("host", task.getHost()); failedTaskMap.put("host", task.getHost());
failedTaskMap.put("日志路径", task.getLogPath()); failedTaskMap.put("log path", task.getLogPath());
failedTaskList.add(failedTaskMap); failedTaskList.add(failedTaskMap);
} }
res = JSONUtils.toJson(failedTaskList); res = JSONUtils.toJson(failedTaskList);
@ -152,10 +153,10 @@ public class AlertManager {
for(TaskInstance taskInstance: toleranceTaskList){ for(TaskInstance taskInstance: toleranceTaskList){
LinkedHashMap<String, String> toleranceWorkerContentMap = new LinkedHashMap(); LinkedHashMap<String, String> toleranceWorkerContentMap = new LinkedHashMap();
toleranceWorkerContentMap.put("工作流程名称", processInstance.getName()); toleranceWorkerContentMap.put("process name", processInstance.getName());
toleranceWorkerContentMap.put("容错任务名称", taskInstance.getName()); toleranceWorkerContentMap.put("task name", taskInstance.getName());
toleranceWorkerContentMap.put("容错机器IP", taskInstance.getHost()); toleranceWorkerContentMap.put("host", taskInstance.getHost());
toleranceWorkerContentMap.put("任务失败次数", String.valueOf(taskInstance.getRetryTimes())); toleranceWorkerContentMap.put("task retry times", String.valueOf(taskInstance.getRetryTimes()));
toleranceTaskInstanceList.add(toleranceWorkerContentMap); toleranceTaskInstanceList.add(toleranceWorkerContentMap);
} }
return JSONUtils.toJson(toleranceTaskInstanceList); return JSONUtils.toJson(toleranceTaskInstanceList);
@ -166,9 +167,9 @@ public class AlertManager {
* @param processInstance * @param processInstance
* @param toleranceTaskList * @param toleranceTaskList
*/ */
public void sendWarnningWorkerleranceFault(ProcessInstance processInstance, List<TaskInstance> toleranceTaskList){ public void sendAlertWorkerToleranceFault(ProcessInstance processInstance, List<TaskInstance> toleranceTaskList){
Alert alert = new Alert(); Alert alert = new Alert();
alert.setTitle("worker容错报警"); alert.setTitle("worker fault tolerance");
alert.setShowType(ShowType.TABLE); alert.setShowType(ShowType.TABLE);
String content = getWorkerToleranceContent(processInstance, toleranceTaskList); String content = getWorkerToleranceContent(processInstance, toleranceTaskList);
alert.setContent(content); alert.setContent(content);
@ -187,7 +188,7 @@ public class AlertManager {
* send process instance alert * send process instance alert
* @param processInstance * @param processInstance
*/ */
public void sendWarnningOfProcessInstance(ProcessInstance processInstance, public void sendAlertProcessInstance(ProcessInstance processInstance,
List<TaskInstance> taskInstances){ List<TaskInstance> taskInstances){
boolean sendWarnning = false; boolean sendWarnning = false;
@ -217,7 +218,7 @@ public class AlertManager {
String cmdName = getCommandCnName(processInstance.getCommandType()); String cmdName = getCommandCnName(processInstance.getCommandType());
String success = processInstance.getState().typeIsSuccess() ? "成功" :"失败"; String success = processInstance.getState().typeIsSuccess() ? "success" :"failed";
alert.setTitle(cmdName + success); alert.setTitle(cmdName + success);
ShowType showType = processInstance.getState().typeIsSuccess() ? ShowType.TEXT : ShowType.TABLE; ShowType showType = processInstance.getState().typeIsSuccess() ? ShowType.TEXT : ShowType.TABLE;
alert.setShowType(showType); alert.setShowType(showType);
@ -233,4 +234,7 @@ public class AlertManager {
logger.info("add alert to db , alert: {}", alert.toString()); logger.info("add alert to db , alert: {}", alert.toString());
} }
public void sendProcessTimeoutAlert(ProcessInstance processInstance, ProcessDefinition processDefinition) {
alertDao.sendProcessTimeoutAlert(processInstance, processDefinition);
}
} }

4
escheduler-server/src/test/java/cn/escheduler/server/master/AlertManagerTest.java

@ -76,7 +76,7 @@ public class AlertManagerTest {
toleranceTaskList.add(toleranceTask1); toleranceTaskList.add(toleranceTask1);
toleranceTaskList.add(toleranceTask2); toleranceTaskList.add(toleranceTask2);
alertManager.sendWarnningWorkerleranceFault(processInstance, toleranceTaskList); alertManager.sendAlertWorkerToleranceFault(processInstance, toleranceTaskList);
} }
@ -103,7 +103,7 @@ public class AlertManagerTest {
toleranceTaskList.add(toleranceTask1); toleranceTaskList.add(toleranceTask1);
toleranceTaskList.add(toleranceTask2); toleranceTaskList.add(toleranceTask2);
alertManager.sendWarnningOfProcessInstance(processInstance, toleranceTaskList); alertManager.sendAlertProcessInstance(processInstance, toleranceTaskList);
} }
} }

Loading…
Cancel
Save