|
|
|
@ -28,8 +28,9 @@ import cn.escheduler.dao.model.ProcessInstance; |
|
|
|
import cn.escheduler.dao.model.TaskInstance; |
|
|
|
import cn.escheduler.dao.model.TaskInstance; |
|
|
|
import cn.escheduler.dao.model.WorkerGroup; |
|
|
|
import cn.escheduler.dao.model.WorkerGroup; |
|
|
|
import cn.escheduler.server.zk.ZKWorkerClient; |
|
|
|
import cn.escheduler.server.zk.ZKWorkerClient; |
|
|
|
import com.cronutils.utils.StringUtils; |
|
|
|
|
|
|
|
import org.apache.commons.configuration.Configuration; |
|
|
|
import org.apache.commons.configuration.Configuration; |
|
|
|
|
|
|
|
import org.apache.commons.lang3.StringUtils; |
|
|
|
|
|
|
|
import org.apache.curator.framework.CuratorFramework; |
|
|
|
import org.apache.curator.framework.recipes.locks.InterProcessMutex; |
|
|
|
import org.apache.curator.framework.recipes.locks.InterProcessMutex; |
|
|
|
import org.slf4j.Logger; |
|
|
|
import org.slf4j.Logger; |
|
|
|
import org.slf4j.LoggerFactory; |
|
|
|
import org.slf4j.LoggerFactory; |
|
|
|
@ -101,15 +102,7 @@ public class FetchTaskThread implements Runnable{ |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
private boolean checkWorkerGroup(TaskInstance taskInstance, String host){ |
|
|
|
private boolean checkWorkerGroup(TaskInstance taskInstance, String host){ |
|
|
|
|
|
|
|
|
|
|
|
int taskWorkerGroupId = taskInstance.getWorkerGroupId(); |
|
|
|
int taskWorkerGroupId = processDao.getTaskWorkerGroupId(taskInstance); |
|
|
|
ProcessInstance processInstance = processDao.findProcessInstanceByTaskId(taskInstance.getId()); |
|
|
|
|
|
|
|
if(processInstance == null){ |
|
|
|
|
|
|
|
logger.error("cannot find the task:{} process instance", taskInstance.getId()); |
|
|
|
|
|
|
|
return false; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
int processWorkerGroupId = processInstance.getWorkerGroupId(); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
taskWorkerGroupId = (taskWorkerGroupId <= 0 ? processWorkerGroupId : taskWorkerGroupId); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if(taskWorkerGroupId <= 0){ |
|
|
|
if(taskWorkerGroupId <= 0){ |
|
|
|
return true; |
|
|
|
return true; |
|
|
|
@ -120,118 +113,124 @@ public class FetchTaskThread implements Runnable{ |
|
|
|
return true; |
|
|
|
return true; |
|
|
|
} |
|
|
|
} |
|
|
|
String ips = workerGroup.getIpList(); |
|
|
|
String ips = workerGroup.getIpList(); |
|
|
|
if(ips == null){ |
|
|
|
if(StringUtils.isBlank(ips)){ |
|
|
|
logger.error("task:{} worker group:{} parameters(ip_list) is null, this task would be running on all workers", |
|
|
|
logger.error("task:{} worker group:{} parameters(ip_list) is null, this task would be running on all workers", |
|
|
|
taskInstance.getId(), workerGroup.getId()); |
|
|
|
taskInstance.getId(), workerGroup.getId()); |
|
|
|
} |
|
|
|
} |
|
|
|
String[] ipArray = ips.split(","); |
|
|
|
String[] ipArray = ips.split(Constants.COMMA); |
|
|
|
List<String> ipList = Arrays.asList(ipArray); |
|
|
|
List<String> ipList = Arrays.asList(ipArray); |
|
|
|
return ipList.contains(host); |
|
|
|
return ipList.contains(host); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@Override |
|
|
|
@Override |
|
|
|
public void run() { |
|
|
|
public void run() { |
|
|
|
|
|
|
|
|
|
|
|
while (Stopper.isRunning()){ |
|
|
|
while (Stopper.isRunning()){ |
|
|
|
InterProcessMutex mutex = null; |
|
|
|
InterProcessMutex mutex = null; |
|
|
|
try { |
|
|
|
try { |
|
|
|
if(OSUtils.checkResource(this.conf, false)) { |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// creating distributed locks, lock path /escheduler/lock/worker
|
|
|
|
ThreadPoolExecutor poolExecutor = (ThreadPoolExecutor) workerExecService; |
|
|
|
String zNodeLockPath = zkWorkerClient.getWorkerLockPath(); |
|
|
|
|
|
|
|
mutex = new InterProcessMutex(zkWorkerClient.getZkClient(), zNodeLockPath); |
|
|
|
|
|
|
|
mutex.acquire(); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
ThreadPoolExecutor poolExecutor = (ThreadPoolExecutor) workerExecService; |
|
|
|
//check memory and cpu usage and threads
|
|
|
|
|
|
|
|
if(OSUtils.checkResource(this.conf, false) && checkThreadCount(poolExecutor)) { |
|
|
|
|
|
|
|
|
|
|
|
for (int i = 0; i < taskNum; i++) { |
|
|
|
//whether have tasks, if no tasks , no need lock //get all tasks
|
|
|
|
|
|
|
|
List<String> tasksQueueList = taskQueue.getAllTasks(Constants.SCHEDULER_TASKS_QUEUE); |
|
|
|
int activeCount = poolExecutor.getActiveCount(); |
|
|
|
if(tasksQueueList.size() > 0){ |
|
|
|
if (activeCount >= workerExecNums) { |
|
|
|
// creating distributed locks, lock path /escheduler/lock/worker
|
|
|
|
logger.info("thread insufficient , activeCount : {} , workerExecNums : {}",activeCount,workerExecNums); |
|
|
|
String zNodeLockPath = zkWorkerClient.getWorkerLockPath(); |
|
|
|
continue; |
|
|
|
mutex = new InterProcessMutex(zkWorkerClient.getZkClient(), zNodeLockPath); |
|
|
|
} |
|
|
|
mutex.acquire(); |
|
|
|
|
|
|
|
|
|
|
|
// task instance id str
|
|
|
|
// task instance id str
|
|
|
|
String taskQueueStr = taskQueue.poll(Constants.SCHEDULER_TASKS_QUEUE, false); |
|
|
|
List<String> taskQueueStrArr = taskQueue.poll(Constants.SCHEDULER_TASKS_QUEUE, taskNum); |
|
|
|
|
|
|
|
|
|
|
|
if (!StringUtils.isEmpty(taskQueueStr )) { |
|
|
|
for(String taskQueueStr : taskQueueStrArr){ |
|
|
|
|
|
|
|
if (StringUtils.isNotBlank(taskQueueStr )) { |
|
|
|
|
|
|
|
|
|
|
|
String[] taskStringArray = taskQueueStr.split(Constants.UNDERLINE); |
|
|
|
if (!checkThreadCount(poolExecutor)) { |
|
|
|
String taskInstIdStr = taskStringArray[taskStringArray.length - 1]; |
|
|
|
break; |
|
|
|
Date now = new Date(); |
|
|
|
} |
|
|
|
Integer taskId = Integer.parseInt(taskInstIdStr); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// find task instance by task id
|
|
|
|
String[] taskStringArray = taskQueueStr.split(Constants.UNDERLINE); |
|
|
|
TaskInstance taskInstance = processDao.findTaskInstanceById(taskId); |
|
|
|
String taskInstIdStr = taskStringArray[taskStringArray.length - 1]; |
|
|
|
|
|
|
|
Date now = new Date(); |
|
|
|
|
|
|
|
Integer taskId = Integer.parseInt(taskInstIdStr); |
|
|
|
|
|
|
|
|
|
|
|
logger.info("worker fetch taskId : {} from queue ", taskId); |
|
|
|
// find task instance by task id
|
|
|
|
|
|
|
|
TaskInstance taskInstance = processDao.findTaskInstanceById(taskId); |
|
|
|
|
|
|
|
|
|
|
|
int retryTimes = 30; |
|
|
|
logger.info("worker fetch taskId : {} from queue ", taskId); |
|
|
|
// mainly to wait for the master insert task to succeed
|
|
|
|
|
|
|
|
while (taskInstance == null && retryTimes > 0) { |
|
|
|
|
|
|
|
Thread.sleep(Constants.SLEEP_TIME_MILLIS); |
|
|
|
|
|
|
|
taskInstance = processDao.findTaskInstanceById(taskId); |
|
|
|
|
|
|
|
retryTimes--; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if (taskInstance == null ) { |
|
|
|
int retryTimes = 30; |
|
|
|
logger.error("task instance is null. task id : {} ", taskId); |
|
|
|
// mainly to wait for the master insert task to succeed
|
|
|
|
continue; |
|
|
|
while (taskInstance == null && retryTimes > 0) { |
|
|
|
} |
|
|
|
Thread.sleep(Constants.SLEEP_TIME_MILLIS); |
|
|
|
if(!checkWorkerGroup(taskInstance, OSUtils.getHost())){ |
|
|
|
taskInstance = processDao.findTaskInstanceById(taskId); |
|
|
|
continue; |
|
|
|
retryTimes--; |
|
|
|
} |
|
|
|
} |
|
|
|
taskQueue.removeNode(Constants.SCHEDULER_TASKS_QUEUE, taskQueueStr); |
|
|
|
|
|
|
|
logger.info("remove task:{} from queue", taskQueueStr); |
|
|
|
if (taskInstance == null ) { |
|
|
|
|
|
|
|
logger.error("task instance is null. task id : {} ", taskId); |
|
|
|
|
|
|
|
continue; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
// set execute task worker host
|
|
|
|
if(!checkWorkerGroup(taskInstance, OSUtils.getHost())){ |
|
|
|
taskInstance.setHost(OSUtils.getHost()); |
|
|
|
continue; |
|
|
|
taskInstance.setStartTime(now); |
|
|
|
} |
|
|
|
|
|
|
|
taskQueue.removeNode(Constants.SCHEDULER_TASKS_QUEUE, taskQueueStr); |
|
|
|
|
|
|
|
logger.info("remove task:{} from queue", taskQueueStr); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// set execute task worker host
|
|
|
|
|
|
|
|
taskInstance.setHost(OSUtils.getHost()); |
|
|
|
|
|
|
|
taskInstance.setStartTime(now); |
|
|
|
|
|
|
|
|
|
|
|
// get process instance
|
|
|
|
|
|
|
|
ProcessInstance processInstance = processDao.findProcessInstanceDetailById(taskInstance.getProcessInstanceId()); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// get process define
|
|
|
|
// get process instance
|
|
|
|
ProcessDefinition processDefine = processDao.findProcessDefineById(taskInstance.getProcessDefinitionId()); |
|
|
|
ProcessInstance processInstance = processDao.findProcessInstanceDetailById(taskInstance.getProcessInstanceId()); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// get process define
|
|
|
|
|
|
|
|
ProcessDefinition processDefine = processDao.findProcessDefineById(taskInstance.getProcessDefinitionId()); |
|
|
|
|
|
|
|
|
|
|
|
taskInstance.setProcessInstance(processInstance); |
|
|
|
|
|
|
|
taskInstance.setProcessDefine(processDefine); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
taskInstance.setProcessInstance(processInstance); |
|
|
|
|
|
|
|
taskInstance.setProcessDefine(processDefine); |
|
|
|
|
|
|
|
|
|
|
|
// get local execute path
|
|
|
|
|
|
|
|
String execLocalPath = FileUtils.getProcessExecDir(processDefine.getProjectId(), |
|
|
|
|
|
|
|
processDefine.getId(), |
|
|
|
|
|
|
|
processInstance.getId(), |
|
|
|
|
|
|
|
taskInstance.getId()); |
|
|
|
|
|
|
|
logger.info("task instance local execute path : {} ", execLocalPath); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// get local execute path
|
|
|
|
|
|
|
|
String execLocalPath = FileUtils.getProcessExecDir(processDefine.getProjectId(), |
|
|
|
|
|
|
|
processDefine.getId(), |
|
|
|
|
|
|
|
processInstance.getId(), |
|
|
|
|
|
|
|
taskInstance.getId()); |
|
|
|
|
|
|
|
logger.info("task instance local execute path : {} ", execLocalPath); |
|
|
|
|
|
|
|
|
|
|
|
// set task execute path
|
|
|
|
|
|
|
|
taskInstance.setExecutePath(execLocalPath); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// check and create Linux users
|
|
|
|
// set task execute path
|
|
|
|
FileUtils.createWorkDirAndUserIfAbsent(execLocalPath, |
|
|
|
taskInstance.setExecutePath(execLocalPath); |
|
|
|
processInstance.getTenantCode(), logger); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
logger.info("task : {} ready to submit to task scheduler thread",taskId); |
|
|
|
// check and create Linux users
|
|
|
|
// submit task
|
|
|
|
FileUtils.createWorkDirAndUserIfAbsent(execLocalPath, |
|
|
|
workerExecService.submit(new TaskScheduleThread(taskInstance, processDao)); |
|
|
|
processInstance.getTenantCode(), logger); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
logger.info("task : {} ready to submit to task scheduler thread",taskId); |
|
|
|
|
|
|
|
// submit task
|
|
|
|
|
|
|
|
workerExecService.submit(new TaskScheduleThread(taskInstance, processDao)); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
Thread.sleep(Constants.SLEEP_TIME_MILLIS); |
|
|
|
Thread.sleep(Constants.SLEEP_TIME_MILLIS); |
|
|
|
|
|
|
|
|
|
|
|
}catch (Exception e){ |
|
|
|
}catch (Exception e){ |
|
|
|
logger.error("fetch task thread exception : " + e.getMessage(),e); |
|
|
|
logger.error("fetch task thread exception : " + e.getMessage(),e); |
|
|
|
} |
|
|
|
}finally { |
|
|
|
finally { |
|
|
|
|
|
|
|
if (mutex != null){ |
|
|
|
if (mutex != null){ |
|
|
|
try { |
|
|
|
try { |
|
|
|
mutex.release(); |
|
|
|
mutex.release(); |
|
|
|
@ -246,4 +245,18 @@ public class FetchTaskThread implements Runnable{ |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
|
|
* |
|
|
|
|
|
|
|
* @param poolExecutor |
|
|
|
|
|
|
|
* @return |
|
|
|
|
|
|
|
*/ |
|
|
|
|
|
|
|
private boolean checkThreadCount(ThreadPoolExecutor poolExecutor) { |
|
|
|
|
|
|
|
int activeCount = poolExecutor.getActiveCount(); |
|
|
|
|
|
|
|
if (activeCount >= workerExecNums) { |
|
|
|
|
|
|
|
logger.info("thread insufficient , activeCount : {} , workerExecNums : {}, will sleep : {} millis for thread resource", activeCount, workerExecNums, Constants.SLEEP_TIME_MILLIS); |
|
|
|
|
|
|
|
return false; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
return true; |
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
} |
easyscheduler
5 years ago
committed by
GitHub