|
|
@ -17,7 +17,6 @@ |
|
|
|
package cn.escheduler.server.worker; |
|
|
|
package cn.escheduler.server.worker; |
|
|
|
|
|
|
|
|
|
|
|
import cn.escheduler.common.Constants; |
|
|
|
import cn.escheduler.common.Constants; |
|
|
|
import cn.escheduler.common.IStoppable; |
|
|
|
|
|
|
|
import cn.escheduler.common.enums.ExecutionStatus; |
|
|
|
import cn.escheduler.common.enums.ExecutionStatus; |
|
|
|
import cn.escheduler.common.enums.TaskType; |
|
|
|
import cn.escheduler.common.enums.TaskType; |
|
|
|
import cn.escheduler.common.queue.ITaskQueue; |
|
|
|
import cn.escheduler.common.queue.ITaskQueue; |
|
|
@ -28,20 +27,21 @@ import cn.escheduler.common.thread.ThreadUtils; |
|
|
|
import cn.escheduler.common.utils.CollectionUtils; |
|
|
|
import cn.escheduler.common.utils.CollectionUtils; |
|
|
|
import cn.escheduler.common.utils.OSUtils; |
|
|
|
import cn.escheduler.common.utils.OSUtils; |
|
|
|
import cn.escheduler.dao.AlertDao; |
|
|
|
import cn.escheduler.dao.AlertDao; |
|
|
|
import cn.escheduler.dao.DaoFactory; |
|
|
|
|
|
|
|
import cn.escheduler.dao.ProcessDao; |
|
|
|
import cn.escheduler.dao.ProcessDao; |
|
|
|
import cn.escheduler.dao.ServerDao; |
|
|
|
|
|
|
|
import cn.escheduler.dao.model.ProcessInstance; |
|
|
|
|
|
|
|
import cn.escheduler.dao.model.TaskInstance; |
|
|
|
import cn.escheduler.dao.model.TaskInstance; |
|
|
|
|
|
|
|
import cn.escheduler.server.master.AbstractServer; |
|
|
|
import cn.escheduler.server.utils.ProcessUtils; |
|
|
|
import cn.escheduler.server.utils.ProcessUtils; |
|
|
|
import cn.escheduler.server.worker.runner.FetchTaskThread; |
|
|
|
import cn.escheduler.server.worker.runner.FetchTaskThread; |
|
|
|
import cn.escheduler.server.zk.ZKWorkerClient; |
|
|
|
import cn.escheduler.server.zk.ZKWorkerClient; |
|
|
|
import org.apache.commons.configuration.Configuration; |
|
|
|
|
|
|
|
import org.apache.commons.configuration.ConfigurationException; |
|
|
|
import org.apache.commons.configuration.ConfigurationException; |
|
|
|
import org.apache.commons.configuration.PropertiesConfiguration; |
|
|
|
import org.apache.commons.configuration.PropertiesConfiguration; |
|
|
|
import org.apache.commons.lang.StringUtils; |
|
|
|
import org.apache.commons.lang.StringUtils; |
|
|
|
import org.slf4j.Logger; |
|
|
|
import org.slf4j.Logger; |
|
|
|
import org.slf4j.LoggerFactory; |
|
|
|
import org.slf4j.LoggerFactory; |
|
|
|
|
|
|
|
import org.springframework.beans.factory.annotation.Autowired; |
|
|
|
|
|
|
|
import org.springframework.boot.WebApplicationType; |
|
|
|
|
|
|
|
import org.springframework.boot.builder.SpringApplicationBuilder; |
|
|
|
|
|
|
|
import org.springframework.context.annotation.ComponentScan; |
|
|
|
|
|
|
|
|
|
|
|
import java.util.Set; |
|
|
|
import java.util.Set; |
|
|
|
import java.util.concurrent.ExecutorService; |
|
|
|
import java.util.concurrent.ExecutorService; |
|
|
@ -51,55 +51,34 @@ import java.util.concurrent.TimeUnit; |
|
|
|
/** |
|
|
|
/** |
|
|
|
* worker server |
|
|
|
* worker server |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
public class WorkerServer implements IStoppable { |
|
|
|
@ComponentScan("cn.escheduler") |
|
|
|
|
|
|
|
public class WorkerServer extends AbstractServer { |
|
|
|
|
|
|
|
|
|
|
|
private static final Logger logger = LoggerFactory.getLogger(WorkerServer.class); |
|
|
|
private static final Logger logger = LoggerFactory.getLogger(WorkerServer.class); |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
|
|
* conf |
|
|
|
|
|
|
|
*/ |
|
|
|
|
|
|
|
private static Configuration conf; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
|
|
* object lock |
|
|
|
|
|
|
|
*/ |
|
|
|
|
|
|
|
private final Object lock = new Object(); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
|
|
* whether or not to close the state |
|
|
|
|
|
|
|
*/ |
|
|
|
|
|
|
|
private boolean terminated = false; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
/** |
|
|
|
* zk worker client |
|
|
|
* zk worker client |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
private static ZKWorkerClient zkWorkerClient = null; |
|
|
|
private static ZKWorkerClient zkWorkerClient = null; |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
|
|
* worker dao database access |
|
|
|
|
|
|
|
*/ |
|
|
|
|
|
|
|
private ServerDao serverDao = null; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
/** |
|
|
|
* process database access |
|
|
|
* process database access |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
private final ProcessDao processDao; |
|
|
|
@Autowired |
|
|
|
|
|
|
|
private ProcessDao processDao; |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
/** |
|
|
|
* alert database access |
|
|
|
* alert database access |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
private final AlertDao alertDao; |
|
|
|
@Autowired |
|
|
|
|
|
|
|
private AlertDao alertDao; |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
/** |
|
|
|
* heartbeat thread pool |
|
|
|
* heartbeat thread pool |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
private ScheduledExecutorService heartbeatWorerService; |
|
|
|
private ScheduledExecutorService heartbeatWorerService; |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
|
|
* heartbeat interval, unit second |
|
|
|
|
|
|
|
*/ |
|
|
|
|
|
|
|
private int heartBeatInterval; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
/** |
|
|
|
* task queue impl |
|
|
|
* task queue impl |
|
|
|
*/ |
|
|
|
*/ |
|
|
@ -115,29 +94,57 @@ public class WorkerServer implements IStoppable { |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
private ExecutorService fetchTaskExecutorService; |
|
|
|
private ExecutorService fetchTaskExecutorService; |
|
|
|
|
|
|
|
|
|
|
|
static { |
|
|
|
public WorkerServer(){} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
public WorkerServer(ProcessDao processDao, AlertDao alertDao){ |
|
|
|
try { |
|
|
|
try { |
|
|
|
conf = new PropertiesConfiguration(Constants.WORKER_PROPERTIES_PATH); |
|
|
|
conf = new PropertiesConfiguration(Constants.WORKER_PROPERTIES_PATH); |
|
|
|
}catch (ConfigurationException e){ |
|
|
|
}catch (ConfigurationException e){ |
|
|
|
logger.error("load configuration failed",e); |
|
|
|
logger.error("load configuration failed",e); |
|
|
|
System.exit(1); |
|
|
|
System.exit(1); |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
public WorkerServer(){ |
|
|
|
|
|
|
|
zkWorkerClient = ZKWorkerClient.getZKWorkerClient(); |
|
|
|
zkWorkerClient = ZKWorkerClient.getZKWorkerClient(); |
|
|
|
this.serverDao = zkWorkerClient.getServerDao(); |
|
|
|
|
|
|
|
this.alertDao = DaoFactory.getDaoInstance(AlertDao.class); |
|
|
|
|
|
|
|
this.processDao = DaoFactory.getDaoInstance(ProcessDao.class); |
|
|
|
|
|
|
|
taskQueue = TaskQueueFactory.getTaskQueueInstance(); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
killExecutorService = ThreadUtils.newDaemonSingleThreadExecutor("Worker-Kill-Thread-Executor"); |
|
|
|
this.taskQueue = TaskQueueFactory.getTaskQueueInstance(); |
|
|
|
|
|
|
|
|
|
|
|
fetchTaskExecutorService = ThreadUtils.newDaemonSingleThreadExecutor("Worker-Fetch-Thread-Executor"); |
|
|
|
this.killExecutorService = ThreadUtils.newDaemonSingleThreadExecutor("Worker-Kill-Thread-Executor"); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
this.fetchTaskExecutorService = ThreadUtils.newDaemonSingleThreadExecutor("Worker-Fetch-Thread-Executor"); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
public void run(){ |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
|
|
* master server startup |
|
|
|
|
|
|
|
* |
|
|
|
|
|
|
|
* master server not use web service |
|
|
|
|
|
|
|
*/ |
|
|
|
|
|
|
|
public static void main(String[] args) { |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
SpringApplicationBuilder app = new SpringApplicationBuilder(WorkerServer.class); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app.web(WebApplicationType.NONE) |
|
|
|
|
|
|
|
.run(args); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@Override |
|
|
|
|
|
|
|
public void run(String... args) throws Exception { |
|
|
|
|
|
|
|
// set the name of the current thread
|
|
|
|
|
|
|
|
Thread.currentThread().setName("Worker-Main-Thread"); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
WorkerServer workerServer = new WorkerServer(processDao,alertDao); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
workerServer.run(processDao,alertDao); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
logger.info("worker server started"); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// blocking
|
|
|
|
|
|
|
|
workerServer.awaitTermination(); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
public void run(ProcessDao processDao, AlertDao alertDao){ |
|
|
|
|
|
|
|
|
|
|
|
// heartbeat interval
|
|
|
|
// heartbeat interval
|
|
|
|
heartBeatInterval = conf.getInt(Constants.WORKER_HEARTBEAT_INTERVAL, |
|
|
|
heartBeatInterval = conf.getInt(Constants.WORKER_HEARTBEAT_INTERVAL, |
|
|
@ -187,45 +194,82 @@ public class WorkerServer implements IStoppable { |
|
|
|
|
|
|
|
|
|
|
|
// submit fetch task thread
|
|
|
|
// submit fetch task thread
|
|
|
|
fetchTaskExecutorService.execute(fetchTaskThread); |
|
|
|
fetchTaskExecutorService.execute(fetchTaskThread); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@Override |
|
|
|
|
|
|
|
public synchronized void stop(String cause) { |
|
|
|
|
|
|
|
|
|
|
|
} |
|
|
|
try { |
|
|
|
|
|
|
|
//execute only once
|
|
|
|
|
|
|
|
if(Stopper.isStoped()){ |
|
|
|
|
|
|
|
return; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
public static void main(String[] args)throws Exception{ |
|
|
|
logger.info("worker server is stopping ..., cause : {}", cause); |
|
|
|
|
|
|
|
|
|
|
|
// set the name of the current thread
|
|
|
|
// set stop signal is true
|
|
|
|
Thread.currentThread().setName("Worker-Main-Thread"); |
|
|
|
Stopper.stop(); |
|
|
|
|
|
|
|
|
|
|
|
WorkerServer workerServer = new WorkerServer(); |
|
|
|
try { |
|
|
|
|
|
|
|
//thread sleep 3 seconds for thread quitely stop
|
|
|
|
|
|
|
|
Thread.sleep(3000L); |
|
|
|
|
|
|
|
}catch (Exception e){ |
|
|
|
|
|
|
|
logger.warn("thread sleep exception:" + e.getMessage(), e); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
workerServer.run(); |
|
|
|
try { |
|
|
|
|
|
|
|
heartbeatWorerService.shutdownNow(); |
|
|
|
|
|
|
|
}catch (Exception e){ |
|
|
|
|
|
|
|
logger.warn("heartbeat service stopped exception"); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
logger.info("heartbeat service stopped"); |
|
|
|
|
|
|
|
|
|
|
|
logger.info("worker server started"); |
|
|
|
try { |
|
|
|
|
|
|
|
ThreadPoolExecutors.getInstance().shutdown(); |
|
|
|
|
|
|
|
}catch (Exception e){ |
|
|
|
|
|
|
|
logger.warn("threadpool service stopped exception:{}",e.getMessage()); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
// blocking
|
|
|
|
logger.info("threadpool service stopped"); |
|
|
|
workerServer.awaitTermination(); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
try { |
|
|
|
|
|
|
|
killExecutorService.shutdownNow(); |
|
|
|
|
|
|
|
}catch (Exception e){ |
|
|
|
|
|
|
|
logger.warn("worker kill executor service stopped exception:{}",e.getMessage()); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
logger.info("worker kill executor service stopped"); |
|
|
|
|
|
|
|
|
|
|
|
} |
|
|
|
try { |
|
|
|
|
|
|
|
fetchTaskExecutorService.shutdownNow(); |
|
|
|
|
|
|
|
}catch (Exception e){ |
|
|
|
|
|
|
|
logger.warn("worker fetch task service stopped exception:{}",e.getMessage()); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
logger.info("worker fetch task service stopped"); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
try{ |
|
|
|
|
|
|
|
zkWorkerClient.close(); |
|
|
|
|
|
|
|
}catch (Exception e){ |
|
|
|
|
|
|
|
logger.warn("zookeeper service stopped exception:{}",e.getMessage()); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
logger.info("zookeeper service stopped"); |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
//notify
|
|
|
|
* blocking implement |
|
|
|
synchronized (lock) { |
|
|
|
* @throws InterruptedException |
|
|
|
terminated = true; |
|
|
|
*/ |
|
|
|
lock.notifyAll(); |
|
|
|
public void awaitTermination() throws InterruptedException { |
|
|
|
|
|
|
|
synchronized (lock) { |
|
|
|
|
|
|
|
while (!terminated) { |
|
|
|
|
|
|
|
lock.wait(); |
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
} catch (Exception e) { |
|
|
|
|
|
|
|
logger.error("worker server stop exception : " + e.getMessage(), e); |
|
|
|
|
|
|
|
System.exit(-1); |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
/** |
|
|
|
* heartbeat thread implement |
|
|
|
* heartbeat thread implement |
|
|
|
* @return |
|
|
|
* @return |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
public Runnable heartBeatThread(){ |
|
|
|
private Runnable heartBeatThread(){ |
|
|
|
Runnable heartBeatThread = new Runnable() { |
|
|
|
Runnable heartBeatThread = new Runnable() { |
|
|
|
@Override |
|
|
|
@Override |
|
|
|
public void run() { |
|
|
|
public void run() { |
|
|
@ -240,11 +284,12 @@ public class WorkerServer implements IStoppable { |
|
|
|
return heartBeatThread; |
|
|
|
return heartBeatThread; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
/** |
|
|
|
* kill process thread implement |
|
|
|
* kill process thread implement |
|
|
|
* @return |
|
|
|
* @return |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
public Runnable getKillProcessThread(){ |
|
|
|
private Runnable getKillProcessThread(){ |
|
|
|
Runnable killProcessThread = new Runnable() { |
|
|
|
Runnable killProcessThread = new Runnable() { |
|
|
|
@Override |
|
|
|
@Override |
|
|
|
public void run() { |
|
|
|
public void run() { |
|
|
@ -286,74 +331,5 @@ public class WorkerServer implements IStoppable { |
|
|
|
return killProcessThread; |
|
|
|
return killProcessThread; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@Override |
|
|
|
|
|
|
|
public synchronized void stop(String cause) { |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
try { |
|
|
|
|
|
|
|
//execute only once
|
|
|
|
|
|
|
|
if(Stopper.isStoped()){ |
|
|
|
|
|
|
|
return; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
logger.info("worker server is stopping ..., cause : {}", cause); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// set stop signal is true
|
|
|
|
|
|
|
|
Stopper.stop(); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
try { |
|
|
|
|
|
|
|
//thread sleep 3 seconds for thread quitely stop
|
|
|
|
|
|
|
|
Thread.sleep(3000L); |
|
|
|
|
|
|
|
}catch (Exception e){ |
|
|
|
|
|
|
|
logger.warn("thread sleep exception:" + e.getMessage(), e); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
try { |
|
|
|
|
|
|
|
heartbeatWorerService.shutdownNow(); |
|
|
|
|
|
|
|
}catch (Exception e){ |
|
|
|
|
|
|
|
logger.warn("heartbeat service stopped exception"); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
logger.info("heartbeat service stopped"); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
try { |
|
|
|
|
|
|
|
ThreadPoolExecutors.getInstance().shutdown(); |
|
|
|
|
|
|
|
}catch (Exception e){ |
|
|
|
|
|
|
|
logger.warn("threadpool service stopped exception:{}",e.getMessage()); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
logger.info("threadpool service stopped"); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
try { |
|
|
|
|
|
|
|
killExecutorService.shutdownNow(); |
|
|
|
|
|
|
|
}catch (Exception e){ |
|
|
|
|
|
|
|
logger.warn("worker kill executor service stopped exception:{}",e.getMessage()); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
logger.info("worker kill executor service stopped"); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
try { |
|
|
|
|
|
|
|
fetchTaskExecutorService.shutdownNow(); |
|
|
|
|
|
|
|
}catch (Exception e){ |
|
|
|
|
|
|
|
logger.warn("worker fetch task service stopped exception:{}",e.getMessage()); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
logger.info("worker fetch task service stopped"); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
try{ |
|
|
|
|
|
|
|
zkWorkerClient.close(); |
|
|
|
|
|
|
|
}catch (Exception e){ |
|
|
|
|
|
|
|
logger.warn("zookeeper service stopped exception:{}",e.getMessage()); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
logger.info("zookeeper service stopped"); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
//notify
|
|
|
|
|
|
|
|
synchronized (lock) { |
|
|
|
|
|
|
|
terminated = true; |
|
|
|
|
|
|
|
lock.notifyAll(); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} catch (Exception e) { |
|
|
|
|
|
|
|
logger.error("worker server stop exception : " + e.getMessage(), e); |
|
|
|
|
|
|
|
System.exit(-1); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|