From 758da8b9a4dc11de50521463ca4b4c6517a92f79 Mon Sep 17 00:00:00 2001 From: lenboo Date: Tue, 9 Jul 2019 14:28:01 +0800 Subject: [PATCH 01/15] update --- .../api/controller/MonitorController.java | 4 +- .../api/service/MonitorService.java | 41 ++++++++- ...onitorUtils.java => ZookeeperMonitor.java} | 40 ++++++++- .../api/utils/ZookeeperMonitorUtilsTest.java | 34 ++++++++ .../common/queue/TaskQueueZkImpl.java | 10 --- .../common/zk/AbstractZKClient.java | 87 +++++++++++++++++-- .../common/queue/TaskQueueImplTest.java | 2 - .../java/cn/escheduler/server/ResInfo.java | 25 ++++++ .../escheduler/server/zk/ZKMasterClient.java | 39 --------- .../escheduler/server/zk/ZKWorkerClient.java | 10 --- .../server/zk/ZKWorkerClientTest.java | 5 +- 11 files changed, 223 insertions(+), 74 deletions(-) rename escheduler-api/src/main/java/cn/escheduler/api/utils/{ZookeeperMonitorUtils.java => ZookeeperMonitor.java} (64%) create mode 100644 escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java diff --git a/escheduler-api/src/main/java/cn/escheduler/api/controller/MonitorController.java b/escheduler-api/src/main/java/cn/escheduler/api/controller/MonitorController.java index 666126c0c9..cba39d5403 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/controller/MonitorController.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/controller/MonitorController.java @@ -66,7 +66,7 @@ public class MonitorController extends BaseController{ logger.info("login user: {}, query all master", loginUser.getUserName()); try{ logger.info("list master, user:{}", loginUser.getUserName()); - Map result = serverService.queryMaster(loginUser); + Map result = monitorService.queryMaster(loginUser); return returnDataList(result); }catch (Exception e){ logger.error(LIST_MASTERS_ERROR.getMsg(),e); @@ -86,7 +86,7 @@ public class MonitorController extends BaseController{ public Result listWorker(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser) { logger.info("login user: {}, query all workers", loginUser.getUserName()); try{ - Map result = serverService.queryWorker(loginUser); + Map result = monitorService.queryWorker(loginUser); return returnDataList(result); }catch (Exception e){ logger.error(LIST_WORKERS_ERROR.getMsg(),e); diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/MonitorService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/MonitorService.java index 4708b00175..08e8bf576e 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/MonitorService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/MonitorService.java @@ -18,13 +18,16 @@ package cn.escheduler.api.service; import cn.escheduler.api.enums.Status; import cn.escheduler.api.utils.Constants; -import cn.escheduler.api.utils.ZookeeperMonitorUtils; +import cn.escheduler.api.utils.ZookeeperMonitor; import cn.escheduler.dao.MonitorDBDao; +import cn.escheduler.dao.model.MasterServer; import cn.escheduler.dao.model.MonitorRecord; import cn.escheduler.dao.model.User; import cn.escheduler.dao.model.ZookeeperRecord; +import org.apache.hadoop.mapred.Master; import org.springframework.stereotype.Service; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -52,6 +55,22 @@ public class MonitorService extends BaseService{ } + /** + * query master list + * + * @param loginUser + * @return + */ + public Map queryMaster(User loginUser) { + + Map result = new HashMap<>(5); + + List masterServers = new ZookeeperMonitor().getMasterServers(); + result.put(Constants.DATA_LIST, masterServers); + putMsg(result,Status.SUCCESS); + + return result; + } /** * query zookeeper state @@ -61,7 +80,7 @@ public class MonitorService extends BaseService{ public Map queryZookeeperState(User loginUser) { Map result = new HashMap<>(5); - List zookeeperRecordList = ZookeeperMonitorUtils.zookeeperInfoList(); + List zookeeperRecordList = ZookeeperMonitor.zookeeperInfoList(); result.put(Constants.DATA_LIST, zookeeperRecordList); putMsg(result, Status.SUCCESS); @@ -69,4 +88,22 @@ public class MonitorService extends BaseService{ return result; } + + + /** + * query master list + * + * @param loginUser + * @return + */ + public Map queryWorker(User loginUser) { + + Map result = new HashMap<>(5); + + List workerServers = new ZookeeperMonitor().getWorkerServers(); + result.put(Constants.DATA_LIST, workerServers); + putMsg(result,Status.SUCCESS); + + return result; + } } diff --git a/escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitorUtils.java b/escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitor.java similarity index 64% rename from escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitorUtils.java rename to escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitor.java index 6612a355fd..a70e9dc4a8 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitorUtils.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitor.java @@ -1,22 +1,26 @@ package cn.escheduler.api.utils; import cn.escheduler.common.zk.AbstractZKClient; +import cn.escheduler.dao.model.MasterServer; import cn.escheduler.dao.model.ZookeeperRecord; +import cn.escheduler.server.ResInfo; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import sun.jvm.hotspot.opto.MachSafePointNode; import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.Map; /** * monitor zookeeper info */ -public class ZookeeperMonitorUtils { +public class ZookeeperMonitor extends AbstractZKClient{ - private static final Logger LOG = LoggerFactory.getLogger(ZookeeperMonitorUtils.class); + private static final Logger LOG = LoggerFactory.getLogger(ZookeeperMonitor.class); private static final String zookeeperList = AbstractZKClient.getZookeeperQuorum(); /** @@ -33,6 +37,38 @@ public class ZookeeperMonitorUtils { return null; } + /** + * get server list. + * @param isMaster + * @return + */ + public List getServers(boolean isMaster){ + List masterServers = new ArrayList<>(); + Map masterMap = getServerList(isMaster); + String parentPath = isMaster ? getMasterZNodeParentPath() : getWorkerZNodeParentPath(); + for(String path : masterMap.keySet()){ + MasterServer masterServer = ResInfo.parseHeartbeatForZKInfo(masterMap.get(path)); + masterServer.setZkDirectory( parentPath + "/"+ path); + masterServers.add(masterServer); + } + return masterServers; + } + + /** + * get master servers + * @return + */ + public List getMasterServers(){ + return getServers(true); + } + + /** + * master construct is the same with worker, use the master instead + * @return + */ + public List getWorkerServers(){ + return getServers(false); + } private static List zookeeperInfoList(String zookeeperServers) { diff --git a/escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java b/escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java new file mode 100644 index 0000000000..05c2ba9fe0 --- /dev/null +++ b/escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java @@ -0,0 +1,34 @@ +package cn.escheduler.api.utils; + +import cn.escheduler.dao.model.MasterServer; +import org.junit.Assert; +import org.junit.Test; + +import javax.crypto.MacSpi; + +import java.util.List; + +import static org.junit.Assert.*; + +public class ZookeeperMonitorUtilsTest { + + + @Test + public void testGetMasterLsit(){ + + ZookeeperMonitor zookeeperMonitor = new ZookeeperMonitor(); + + + List masterServerList = zookeeperMonitor.getMasterServers(); + + List workerServerList = zookeeperMonitor.getWorkerServers(); + + System.out.println("master:" + masterServerList); + System.out.println("worker:" + workerServerList); + Assert.assertEquals(masterServerList.size(), 1); + Assert.assertEquals(workerServerList.size(), 1); + + + } + +} \ No newline at end of file diff --git a/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java b/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java index eed7fb1ac5..73f7499fa7 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java @@ -417,16 +417,6 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue { } } - - /** - * get zookeeper client of CuratorFramework - * @return - */ - public CuratorFramework getZkClient() { - return zkClient; - } - - /** * Get the task queue path * @param key task queue name diff --git a/escheduler-common/src/main/java/cn/escheduler/common/zk/AbstractZKClient.java b/escheduler-common/src/main/java/cn/escheduler/common/zk/AbstractZKClient.java index e2f064be13..f7134f5e7a 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/zk/AbstractZKClient.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/zk/AbstractZKClient.java @@ -30,13 +30,12 @@ import org.apache.curator.framework.imps.CuratorFrameworkState; import org.apache.curator.framework.state.ConnectionState; import org.apache.curator.framework.state.ConnectionStateListener; import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos; import org.apache.zookeeper.CreateMode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; +import java.util.*; import static cn.escheduler.common.Constants.*; @@ -213,9 +212,9 @@ public abstract class AbstractZKClient { protected void initSystemZNode(){ try { // read master node parent path from conf - masterZNodeParentPath = conf.getString(Constants.ZOOKEEPER_ESCHEDULER_MASTERS); + masterZNodeParentPath = getMasterZNodeParentPath(); // read worker node parent path from conf - workerZNodeParentPath = conf.getString(Constants.ZOOKEEPER_ESCHEDULER_WORKERS); + workerZNodeParentPath = getWorkerZNodeParentPath(); // read server node parent path from conf deadServerZNodeParentPath = conf.getString(ZOOKEEPER_ESCHEDULER_DEAD_SERVERS); @@ -243,6 +242,7 @@ public abstract class AbstractZKClient { } } + public void removeDeadServerByHost(String host, String serverType) throws Exception { List deadServers = zkClient.getChildren().forPath(deadServerZNodeParentPath); for(String serverPath : deadServers){ @@ -291,6 +291,8 @@ public abstract class AbstractZKClient { } + + /** * for stop server * @param serverStoppable @@ -340,6 +342,81 @@ public abstract class AbstractZKClient { return sb.toString(); } + /** + * get master server list map. + * result : {host : resource info} + * @return + */ + public Map getServerList(boolean isMaster ){ + + Map masterMap = new HashMap<>(); + try { + String path = isMaster ? getMasterZNodeParentPath() : getWorkerZNodeParentPath(); + List serverList = getZkClient().getChildren().forPath(path); + for(String server : serverList){ + byte[] bytes = getZkClient().getData().forPath(path + "/" + server); + masterMap.putIfAbsent(server, new String(bytes)); + } + } catch (Exception e) { + e.printStackTrace(); + } + + return masterMap; + } + + /** + * get zkclient + * @return + */ + public CuratorFramework getZkClient() { + return zkClient; + } + + /** + * get worker node parent path + * @return + */ + protected String getWorkerZNodeParentPath(){return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_WORKERS);}; + + /** + * get master node parent path + * @return + */ + protected String getMasterZNodeParentPath(){return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_MASTERS);} + + /** + * get master lock path + * @return + */ + public String getMasterLockPath(){ + return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_MASTERS); + } + + /** + * get master start up lock path + * @return + */ + public String getMasterStartUpLockPath(){ + return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_STARTUP_MASTERS); + } + + /** + * get master failover lock path + * @return + */ + public String getMasterFailoverLockPath(){ + return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_MASTERS); + } + + /** + * get worker failover lock path + * @return + */ + public String getWorkerFailoverLockPath(){ + return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_WORKERS); + } + + @Override public String toString() { return "AbstractZKClient{" + diff --git a/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java b/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java index 21d2f5858e..16562d54aa 100644 --- a/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java +++ b/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java @@ -84,8 +84,6 @@ public class TaskQueueImplTest { return; } - String node2 = tasks.get(0); - } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/ResInfo.java b/escheduler-server/src/main/java/cn/escheduler/server/ResInfo.java index 81f126998c..581a0af482 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/ResInfo.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/ResInfo.java @@ -17,8 +17,12 @@ package cn.escheduler.server; import cn.escheduler.common.Constants; +import cn.escheduler.common.utils.DateUtils; import cn.escheduler.common.utils.JSONUtils; import cn.escheduler.common.utils.OSUtils; +import cn.escheduler.dao.model.MasterServer; + +import java.util.Date; /** * heartbeat for ZK reigster res info @@ -119,4 +123,25 @@ public class ResInfo { + lastHeartbeatTime; } + /** + * parse heartbeat info for zk + * @param heartBeatInfo + * @return + */ + public static MasterServer parseHeartbeatForZKInfo(String heartBeatInfo){ + MasterServer masterServer = null; + String[] masterArray = heartBeatInfo.split(Constants.COMMA); + if(masterArray.length != 6){ + return masterServer; + + } + masterServer = new MasterServer(); + masterServer.setHost(masterArray[0]); + masterServer.setPort(Integer.parseInt(masterArray[1])); + masterServer.setResInfo(getResInfoJson(Double.parseDouble(masterArray[2]), Double.parseDouble(masterArray[3]))); + masterServer.setCreateTime(DateUtils.stringToDate(masterArray[4])); + masterServer.setLastHeartbeatTime(DateUtils.stringToDate(masterArray[5])); + return masterServer; + } + } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java index 85c805e2fc..0bc3247bf4 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java @@ -448,45 +448,6 @@ public class ZKMasterClient extends AbstractZKClient { } - /** - * get master lock path - * @return - */ - public String getMasterLockPath(){ - return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_MASTERS); - } - - /** - * get master start up lock path - * @return - */ - public String getMasterStartUpLockPath(){ - return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_STARTUP_MASTERS); - } - - /** - * get master failover lock path - * @return - */ - public String getMasterFailoverLockPath(){ - return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_MASTERS); - } - - /** - * get worker failover lock path - * @return - */ - public String getWorkerFailoverLockPath(){ - return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_WORKERS); - } - - /** - * get zkclient - * @return - */ - public CuratorFramework getZkClient() { - return zkClient; - } /** diff --git a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKWorkerClient.java b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKWorkerClient.java index 7cf899d832..4f03d0a41c 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKWorkerClient.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKWorkerClient.java @@ -264,16 +264,6 @@ public class ZKWorkerClient extends AbstractZKClient { return workerZNode; } - - /** - * get zkclient - * @return - */ - public CuratorFramework getZkClient() { - return zkClient; - } - - /** * get worker lock path * @return diff --git a/escheduler-server/src/test/java/cn/escheduler/server/zk/ZKWorkerClientTest.java b/escheduler-server/src/test/java/cn/escheduler/server/zk/ZKWorkerClientTest.java index c8aa0930a2..c8e4b93027 100644 --- a/escheduler-server/src/test/java/cn/escheduler/server/zk/ZKWorkerClientTest.java +++ b/escheduler-server/src/test/java/cn/escheduler/server/zk/ZKWorkerClientTest.java @@ -1,6 +1,7 @@ package cn.escheduler.server.zk; import cn.escheduler.common.Constants; +import cn.escheduler.common.zk.AbstractZKClient; import org.junit.Test; import java.util.Arrays; @@ -17,8 +18,8 @@ public class ZKWorkerClientTest { public void getZKWorkerClient() throws Exception { - ZKWorkerClient zkWorkerClient = ZKWorkerClient.getZKWorkerClient(); - zkWorkerClient.removeDeadServerByHost("127.0.0.1", Constants.WORKER_PREFIX); +// ZKWorkerClient zkWorkerClient = ZKWorkerClient.getZKWorkerClient(); +// zkWorkerClient.removeDeadServerByHost("127.0.0.1", Constants.WORKER_PREFIX); } From beaa6c6aa076b4414a1dd3fc6ab7e81907a80f79 Mon Sep 17 00:00:00 2001 From: lenboo Date: Tue, 9 Jul 2019 14:37:42 +0800 Subject: [PATCH 02/15] update --- .../main/java/cn/escheduler/api/utils/ZookeeperMonitor.java | 1 - .../cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java | 3 --- 2 files changed, 4 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitor.java b/escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitor.java index a70e9dc4a8..0f44b5f7db 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitor.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitor.java @@ -7,7 +7,6 @@ import cn.escheduler.server.ResInfo; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import sun.jvm.hotspot.opto.MachSafePointNode; import java.util.ArrayList; import java.util.Date; diff --git a/escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java b/escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java index 05c2ba9fe0..adaa78f7ba 100644 --- a/escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java +++ b/escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java @@ -4,12 +4,9 @@ import cn.escheduler.dao.model.MasterServer; import org.junit.Assert; import org.junit.Test; -import javax.crypto.MacSpi; import java.util.List; -import static org.junit.Assert.*; - public class ZookeeperMonitorUtilsTest { From fcd9b661b9af02eae58cd101141ac7a2061b8ba7 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 9 Jul 2019 15:28:36 +0800 Subject: [PATCH 03/15] The start time must not be the same as the end --- .../src/main/java/cn/escheduler/api/enums/Status.java | 1 + .../cn/escheduler/api/service/SchedulerService.java | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java b/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java index bd1d5e91b3..4867f3ca98 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java @@ -163,6 +163,7 @@ public enum Status { BATCH_DELETE_PROCESS_INSTANCE_BY_IDS_ERROR(10117,"batch delete process instance by ids {0} error"), PREVIEW_SCHEDULE_ERROR(10139,"preview schedule error"), PARSE_TO_CRON_EXPRESSION_ERROR(10140,"parse cron to cron expression error"), + SCHEDULE_START_TIME_END_TIME_SAME(10141,"The start time must not be the same as the end"), UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found"), diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java index ab2ef9018a..2766a101e1 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java @@ -119,6 +119,11 @@ public class SchedulerService extends BaseService { scheduleObj.setProcessDefinitionName(processDefinition.getName()); ScheduleParam scheduleParam = JSONUtils.parseObject(schedule, ScheduleParam.class); + if (DateUtils.differSec(scheduleParam.getStartTime(),scheduleParam.getEndTime()) == 0) { + logger.warn("The start time must not be the same as the end"); + putMsg(result,Status.SCHEDULE_START_TIME_END_TIME_SAME); + return result; + } scheduleObj.setStartTime(scheduleParam.getStartTime()); scheduleObj.setEndTime(scheduleParam.getEndTime()); if (!org.quartz.CronExpression.isValidExpression(scheduleParam.getCrontab())) { @@ -205,6 +210,11 @@ public class SchedulerService extends BaseService { // updateProcessInstance param if (StringUtils.isNotEmpty(scheduleExpression)) { ScheduleParam scheduleParam = JSONUtils.parseObject(scheduleExpression, ScheduleParam.class); + if (DateUtils.differSec(scheduleParam.getStartTime(),scheduleParam.getEndTime()) == 0) { + logger.warn("The start time must not be the same as the end"); + putMsg(result,Status.SCHEDULE_START_TIME_END_TIME_SAME); + return result; + } schedule.setStartTime(scheduleParam.getStartTime()); schedule.setEndTime(scheduleParam.getEndTime()); if (!org.quartz.CronExpression.isValidExpression(scheduleParam.getCrontab())) { From 5869a87aca92e23dc0365809e6134a70cc906192 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 9 Jul 2019 15:57:55 +0800 Subject: [PATCH 04/15] The start time must not be the same as the end --- .../java/cn/escheduler/api/service/SchedulerService.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java index 2766a101e1..500a831710 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java @@ -567,6 +567,11 @@ public class SchedulerService extends BaseService { Date startTime = now.after(scheduleParam.getStartTime()) ? now : scheduleParam.getStartTime(); Date endTime = scheduleParam.getEndTime(); + if (DateUtils.differSec(scheduleParam.getStartTime(),scheduleParam.getEndTime()) == 0) { + logger.warn("The start time must not be the same as the end"); + putMsg(result,Status.SCHEDULE_START_TIME_END_TIME_SAME); + return result; + } try { cronExpression = CronUtils.parse2CronExpression(scheduleParam.getCrontab()); } catch (ParseException e) { From 9dbf0265453095e474054954834f0dfdc972c891 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 9 Jul 2019 16:06:53 +0800 Subject: [PATCH 05/15] update preview --- .../java/cn/escheduler/api/service/SchedulerService.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java index 500a831710..2766a101e1 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java @@ -567,11 +567,6 @@ public class SchedulerService extends BaseService { Date startTime = now.after(scheduleParam.getStartTime()) ? now : scheduleParam.getStartTime(); Date endTime = scheduleParam.getEndTime(); - if (DateUtils.differSec(scheduleParam.getStartTime(),scheduleParam.getEndTime()) == 0) { - logger.warn("The start time must not be the same as the end"); - putMsg(result,Status.SCHEDULE_START_TIME_END_TIME_SAME); - return result; - } try { cronExpression = CronUtils.parse2CronExpression(scheduleParam.getCrontab()); } catch (ParseException e) { From b6eb2157797a475a043762377c517440ed124e33 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Tue, 9 Jul 2019 16:07:50 +0800 Subject: [PATCH 06/15] only kerberos startup and sql task type is hive or spark --- .../java/cn/escheduler/api/service/DataSourceService.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java index 5331e03418..3615ca65ad 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java @@ -522,7 +522,10 @@ public class DataSourceService extends BaseService{ parameterMap.put(Constants.JDBC_URL, jdbcUrl); parameterMap.put(Constants.USER, userName); parameterMap.put(Constants.PASSWORD, password); - parameterMap.put(Constants.PRINCIPAL,principal); + if (CommonUtils.getKerberosStartupState() && + (type == DbType.HIVE || type == DbType.SPARK)){ + parameterMap.put(Constants.PRINCIPAL,principal); + } if (other != null && !"".equals(other)) { Map map = JSONObject.parseObject(other, new TypeReference>() { }); From 603c478d2afacf3e1f2a720432b06233af64829f Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Tue, 9 Jul 2019 16:17:15 +0800 Subject: [PATCH 07/15] Constants XXXXXX update --- .../java/cn/escheduler/api/service/DataSourceService.java | 4 ---- .../src/main/java/cn/escheduler/common/Constants.java | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java index 3615ca65ad..2acaef0e84 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java @@ -67,14 +67,10 @@ public class DataSourceService extends BaseService{ public static final String PASSWORD = cn.escheduler.common.Constants.PASSWORD; public static final String OTHER = "other"; - @Autowired - private ProjectMapper projectMapper; @Autowired private DataSourceMapper dataSourceMapper; - @Autowired - private ProjectService projectService; @Autowired private DatasourceUserMapper datasourceUserMapper; diff --git a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java index d2ee3a34c0..d90ea032f4 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java @@ -490,7 +490,7 @@ public final class Constants { public static final String DEFAULT = "Default"; public static final String PASSWORD = "password"; - public static final String XXXXXX = "xxxxxx"; + public static final String XXXXXX = "******"; public static String TASK_RECORD_TABLE_HIVE_LOG = "eamp_hive_log_hd"; From fd89dff3ab2e860dbb4d386c537a5e99786b57b2 Mon Sep 17 00:00:00 2001 From: lenboo Date: Tue, 9 Jul 2019 18:26:54 +0800 Subject: [PATCH 08/15] refactor ResInfo --- .../api/utils/ZookeeperMonitorUtilsTest.java | 2 - .../cn/escheduler/common/utils/IpUtils.java | 6 --- .../common/queue/TaskQueueImplTest.java | 2 - .../escheduler/common/utils/IpUtilsTest.java | 41 +++++++++++++++++++ .../java/cn/escheduler/server/ResInfo.java | 10 +++++ .../escheduler/server/zk/ZKMasterClient.java | 28 +++---------- .../escheduler/server/zk/ZKWorkerClient.java | 21 +--------- 7 files changed, 59 insertions(+), 51 deletions(-) create mode 100644 escheduler-common/src/test/java/cn/escheduler/common/utils/IpUtilsTest.java diff --git a/escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java b/escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java index adaa78f7ba..87a26ba449 100644 --- a/escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java +++ b/escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java @@ -20,8 +20,6 @@ public class ZookeeperMonitorUtilsTest { List workerServerList = zookeeperMonitor.getWorkerServers(); - System.out.println("master:" + masterServerList); - System.out.println("worker:" + workerServerList); Assert.assertEquals(masterServerList.size(), 1); Assert.assertEquals(workerServerList.size(), 1); diff --git a/escheduler-common/src/main/java/cn/escheduler/common/utils/IpUtils.java b/escheduler-common/src/main/java/cn/escheduler/common/utils/IpUtils.java index ddc520a876..972135ae89 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/utils/IpUtils.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/utils/IpUtils.java @@ -61,10 +61,4 @@ public class IpUtils { return sb.toString(); } - - - public static void main(String[] args){ - long ipLong = ipToLong("11.3.4.5"); - logger.info(longToIp(ipLong)); - } } diff --git a/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java b/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java index 16562d54aa..b0f9776b5f 100644 --- a/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java +++ b/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java @@ -20,13 +20,11 @@ import cn.escheduler.common.Constants; import cn.escheduler.common.utils.IpUtils; import cn.escheduler.common.utils.OSUtils; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Arrays; import java.util.List; import java.util.Random; diff --git a/escheduler-common/src/test/java/cn/escheduler/common/utils/IpUtilsTest.java b/escheduler-common/src/test/java/cn/escheduler/common/utils/IpUtilsTest.java new file mode 100644 index 0000000000..11a03a2334 --- /dev/null +++ b/escheduler-common/src/test/java/cn/escheduler/common/utils/IpUtilsTest.java @@ -0,0 +1,41 @@ +package cn.escheduler.common.utils; + +import org.junit.Assert; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class IpUtilsTest { + + @Test + public void ipToLong() { + + String ip = "192.168.110.1"; + String ip2 = "0.0.0.0"; + long longNumber = IpUtils.ipToLong(ip); + long longNumber2 = IpUtils.ipToLong(ip2); + System.out.println(longNumber); + Assert.assertEquals(longNumber, 3232263681L); + Assert.assertEquals(longNumber2, 0L); + + String ip3 = "255.255.255.255"; + long longNumber3 = IpUtils.ipToLong(ip3); + System.out.println(longNumber3); + Assert.assertEquals(longNumber3, 4294967295L); + + } + + @Test + public void longToIp() { + + String ip = "192.168.110.1"; + String ip2 = "0.0.0.0"; + long longNum = 3232263681L; + String i1 = IpUtils.longToIp(longNum); + + String i2 = IpUtils.longToIp(0); + + Assert.assertEquals(ip, i1); + Assert.assertEquals(ip2, i2); + } +} \ No newline at end of file diff --git a/escheduler-server/src/main/java/cn/escheduler/server/ResInfo.java b/escheduler-server/src/main/java/cn/escheduler/server/ResInfo.java index 581a0af482..844c7be8b0 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/ResInfo.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/ResInfo.java @@ -102,6 +102,16 @@ public class ResInfo { } + public static String getHeartBeatInfo(Date now){ + return buildHeartbeatForZKInfo(OSUtils.getHost(), + OSUtils.getProcessID(), + OSUtils.cpuUsage(), + OSUtils.memoryUsage(), + DateUtils.dateToString(now), + DateUtils.dateToString(now)); + + } + /** * build heartbeat info for zk * @param host diff --git a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java index 0bc3247bf4..3596155dd3 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java @@ -204,7 +204,7 @@ public class ZKMasterClient extends AbstractZKClient { } // specify the format of stored data in ZK nodes - String heartbeatZKInfo = getOsInfo(now); + String heartbeatZKInfo = ResInfo.getHeartBeatInfo(now); // create temporary sequence nodes for master znode masterZNode = zkClient.create().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath( masterZNodeParentPath + "/" + OSUtils.getHost() + "_", heartbeatZKInfo.getBytes()); @@ -259,10 +259,10 @@ public class ZKMasterClient extends AbstractZKClient { return false; } - List masterZNodeList = null; - masterZNodeList = zkClient.getChildren().forPath(path); - if (CollectionUtils.isNotEmpty(masterZNodeList)){ - for (String masterZNode : masterZNodeList){ + List serverList = null; + serverList = zkClient.getChildren().forPath(path); + if (CollectionUtils.isNotEmpty(serverList)){ + for (String masterZNode : serverList){ if (masterZNode.startsWith(host)){ return true; } @@ -423,22 +423,6 @@ public class ZKMasterClient extends AbstractZKClient { } - - /** - * get os info - * @param now - * @return - */ - private String getOsInfo(Date now) { - return ResInfo.buildHeartbeatForZKInfo(OSUtils.getHost(), - OSUtils.getProcessID(), - OSUtils.cpuUsage(), - OSUtils.memoryUsage(), - DateUtils.dateToString(now), - DateUtils.dateToString(now)); - } - - /** * get master znode * @return @@ -541,7 +525,7 @@ public class ZKMasterClient extends AbstractZKClient { } /** - * get host ip + * get host ip, string format: masterParentPath/ip_000001/value * @param path * @return */ diff --git a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKWorkerClient.java b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKWorkerClient.java index 4f03d0a41c..e00d72da24 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKWorkerClient.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKWorkerClient.java @@ -116,11 +116,10 @@ public class ZKWorkerClient extends AbstractZKClient { public String initWorkZNode() throws Exception { - Date now = new Date(); - String heartbeatZKInfo = getOsInfo(now); - + String heartbeatZKInfo = ResInfo.getHeartBeatInfo(new Date()); workerZNode = workerZNodeParentPath + "/" + OSUtils.getHost() + "_"; + workerZNode = zkClient.create().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(workerZNode, heartbeatZKInfo.getBytes()); logger.info("register worker node {} success", workerZNode); @@ -141,7 +140,6 @@ public class ZKWorkerClient extends AbstractZKClient { workerZNode = workerZNodeParentPath + "/" + OSUtils.getHost() + "_"; List workerZNodeList = zkClient.getChildren().forPath(workerZNodeParentPath); - if (CollectionUtils.isNotEmpty(workerZNodeList)){ boolean flag = false; for (String workerZNode : workerZNodeList){ @@ -241,21 +239,6 @@ public class ZKWorkerClient extends AbstractZKClient { } - /** - * get os info - * @param now - * @return - */ - private String getOsInfo(Date now) { - return ResInfo.buildHeartbeatForZKInfo(OSUtils.getHost(), - OSUtils.getProcessID(), - OSUtils.cpuUsage(), - OSUtils.memoryUsage(), - DateUtils.dateToString(now), - DateUtils.dateToString(now)); - } - - /** * get worker znode * @return From 5aef28ae66ecd1b07de6f3bd7328d33cd5fcb3f6 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 9 Jul 2019 18:48:01 +0800 Subject: [PATCH 09/15] update email regex --- .../src/main/java/cn/escheduler/common/Constants.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java index d90ea032f4..6c39e0a264 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java @@ -337,7 +337,7 @@ public final class Constants { /** * email regex */ - public static final Pattern REGEX_MAIL_NAME = Pattern.compile("^[a-zA-Z0-9_-]+@[a-zA-Z0-9_-]+(\\.[a-zA-Z0-9_-]+)+$"); + public static final Pattern REGEX_MAIL_NAME = Pattern.compile("^([a-z0-9A-Z]+[-|\\.]?)+[a-z0-9A-Z]@([a-z0-9A-Z]+(-[a-z0-9A-Z]+)?\\.)+[a-zA-Z]{2,}$"); /** * read permission From ffad909fae60b1cdc6589118b4dd28fc0435980c Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 9 Jul 2019 19:56:32 +0800 Subject: [PATCH 10/15] update upgrade and create log tips --- .../cn/escheduler/dao/upgrade/shell/CreateEscheduler.java | 8 +++++--- .../escheduler/dao/upgrade/shell/UpgradeEscheduler.java | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/shell/CreateEscheduler.java b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/shell/CreateEscheduler.java index 3726a70809..2f1e070e7b 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/shell/CreateEscheduler.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/shell/CreateEscheduler.java @@ -30,13 +30,15 @@ public class CreateEscheduler { public static void main(String[] args) { EschedulerManager eschedulerManager = new EschedulerManager(); - eschedulerManager.initEscheduler(); - logger.info("init escheduler finished"); + try { + eschedulerManager.initEscheduler(); + logger.info("init escheduler finished"); eschedulerManager.upgradeEscheduler(); logger.info("upgrade escheduler finished"); + logger.info("create escheduler success"); } catch (Exception e) { - logger.error("upgrade escheduler failed",e); + logger.error("create escheduler failed",e); } } diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/shell/UpgradeEscheduler.java b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/shell/UpgradeEscheduler.java index 822e020cad..7608d8ce6f 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/shell/UpgradeEscheduler.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/shell/UpgradeEscheduler.java @@ -30,7 +30,7 @@ public class UpgradeEscheduler { EschedulerManager eschedulerManager = new EschedulerManager(); try { eschedulerManager.upgradeEscheduler(); - logger.info("upgrade escheduler finished"); + logger.info("upgrade escheduler success"); } catch (Exception e) { logger.error(e.getMessage(),e); logger.info("Upgrade escheduler failed"); From 28c67fa13edc609a8aeaa1ea05b0f36c97986f33 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 9 Jul 2019 20:01:27 +0800 Subject: [PATCH 11/15] remove escheduler.sql and quartz.sql --- sql/escheduler.sql | 436 --------------------------------------------- sql/quartz.sql | 179 ------------------- 2 files changed, 615 deletions(-) delete mode 100644 sql/escheduler.sql delete mode 100644 sql/quartz.sql diff --git a/sql/escheduler.sql b/sql/escheduler.sql deleted file mode 100644 index 774de10e42..0000000000 --- a/sql/escheduler.sql +++ /dev/null @@ -1,436 +0,0 @@ -/* -Navicat MySQL Data Transfer - -Source Server : xx.xx -Source Server Version : 50725 -Source Host : 192.168.xx.xx:3306 -Source Database : escheduler - -Target Server Type : MYSQL -Target Server Version : 50725 -File Encoding : 65001 - -Date: 2019-03-23 11:47:30 -*/ - -SET FOREIGN_KEY_CHECKS=0; - --- ---------------------------- --- Table structure for t_escheduler_alert --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_alert`; -CREATE TABLE `t_escheduler_alert` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `title` varchar(64) DEFAULT NULL COMMENT '消息标题', - `show_type` tinyint(4) DEFAULT NULL COMMENT '发送格式，0是TABLE,1是TEXT', - `content` text COMMENT '消息内容（可以是邮件，可以是短信。邮件是JSON Map存放，短信是字符串）', - `alert_type` tinyint(4) DEFAULT NULL COMMENT '0是邮件，1是短信', - `alert_status` tinyint(4) DEFAULT '0' COMMENT '0是待执行，1是执行成功，2执行失败', - `log` text COMMENT '执行日志', - `alertgroup_id` int(11) DEFAULT NULL COMMENT '发送组', - `receivers` text COMMENT '收件人', - `receivers_cc` text COMMENT '抄送人', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_alertgroup --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_alertgroup`; -CREATE TABLE `t_escheduler_alertgroup` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `group_name` varchar(255) DEFAULT NULL COMMENT '组名称', - `group_type` tinyint(4) DEFAULT NULL COMMENT '组类型(邮件0，短信1...)', - `desc` varchar(255) DEFAULT NULL COMMENT '备注', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_command --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_command`; -CREATE TABLE `t_escheduler_command` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `command_type` tinyint(4) DEFAULT NULL COMMENT '命令类型：0 启动工作流,1 从当前节点开始执行,2 恢复被容错的工作流,3 恢复暂停流程,4 从失败节点开始执行,5 补数,6 调度,7 重跑,8 暂停,9 停止,10 恢复等待线程', - `process_definition_id` int(11) DEFAULT NULL COMMENT '流程定义id', - `command_param` text COMMENT '命令的参数（json格式）', - `task_depend_type` tinyint(4) DEFAULT NULL COMMENT '节点依赖类型：0 当前节点,1 向前执行,2 向后执行', - `failure_strategy` tinyint(4) DEFAULT '0' COMMENT '失败策略：0结束，1继续', - `warning_type` tinyint(4) DEFAULT '0' COMMENT '告警类型：0 不发,1 流程成功发,2 流程失败发,3 成功失败都发', - `warning_group_id` int(11) DEFAULT NULL COMMENT '告警组', - `schedule_time` datetime DEFAULT NULL COMMENT '预期运行时间', - `start_time` datetime DEFAULT NULL COMMENT '开始时间', - `executor_id` int(11) DEFAULT NULL COMMENT '执行用户id', - `dependence` varchar(255) DEFAULT NULL COMMENT '依赖字段', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - `process_instance_priority` int(11) DEFAULT NULL COMMENT '流程实例优先级：0 Highest,1 High,2 Medium,3 Low,4 Lowest', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_datasource --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_datasource`; -CREATE TABLE `t_escheduler_datasource` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `name` varchar(64) NOT NULL COMMENT '数据源名称', - `note` varchar(256) DEFAULT NULL COMMENT '描述', - `type` tinyint(4) NOT NULL COMMENT '数据源类型：0 mysql,1 postgresql,2 hive,3 spark', - `user_id` int(11) NOT NULL COMMENT '创建用户id', - `connection_params` text NOT NULL COMMENT '连接参数(json格式)', - `create_time` datetime NOT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_master_server --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_master_server`; -CREATE TABLE `t_escheduler_master_server` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `host` varchar(45) DEFAULT NULL COMMENT 'ip', - `port` int(11) DEFAULT NULL COMMENT '进程号', - `zk_directory` varchar(64) DEFAULT NULL COMMENT 'zk注册目录', - `res_info` varchar(256) DEFAULT NULL COMMENT '集群资源信息：json格式{"cpu":xxx,"memroy":xxx}', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `last_heartbeat_time` datetime DEFAULT NULL COMMENT '最后心跳时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_process_definition --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_process_definition`; -CREATE TABLE `t_escheduler_process_definition` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `name` varchar(255) DEFAULT NULL COMMENT '流程定义名称', - `version` int(11) DEFAULT NULL COMMENT '流程定义版本', - `release_state` tinyint(4) DEFAULT NULL COMMENT '流程定义的发布状态：0 未上线 1已上线', - `project_id` int(11) DEFAULT NULL COMMENT '项目id', - `user_id` int(11) DEFAULT NULL COMMENT '流程定义所属用户id', - `process_definition_json` longtext COMMENT '流程定义json串', - `desc` text COMMENT '流程定义描述', - `global_params` text COMMENT '全局参数', - `flag` tinyint(4) DEFAULT NULL COMMENT '流程是否可用\r\n：0 不可用\r\n，1 可用', - `locations` text COMMENT '节点坐标信息', - `connects` text COMMENT '节点连线信息', - `receivers` text COMMENT '收件人', - `receivers_cc` text COMMENT '抄送人', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`), - KEY `process_definition_index` (`project_id`,`id`) USING BTREE -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_process_instance --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_process_instance`; -CREATE TABLE `t_escheduler_process_instance` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `name` varchar(255) DEFAULT NULL COMMENT '流程实例名称', - `process_definition_id` int(11) DEFAULT NULL COMMENT '流程定义id', - `state` tinyint(4) DEFAULT NULL COMMENT '流程实例状态：0 提交成功,1 正在运行,2 准备暂停,3 暂停,4 准备停止,5 停止,6 失败,7 成功,8 需要容错,9 kill,10 等待线程,11 等待依赖完成', - `recovery` tinyint(4) DEFAULT NULL COMMENT '流程实例容错标识：0 正常,1 需要被容错重启', - `start_time` datetime DEFAULT NULL COMMENT '流程实例开始时间', - `end_time` datetime DEFAULT NULL COMMENT '流程实例结束时间', - `run_times` int(11) DEFAULT NULL COMMENT '流程实例运行次数', - `host` varchar(45) DEFAULT NULL COMMENT '流程实例所在的机器', - `command_type` tinyint(4) DEFAULT NULL COMMENT '命令类型：0 启动工作流,1 从当前节点开始执行,2 恢复被容错的工作流,3 恢复暂停流程,4 从失败节点开始执行,5 补数,6 调度,7 重跑,8 暂停,9 停止,10 恢复等待线程', - `command_param` text COMMENT '命令的参数（json格式）', - `task_depend_type` tinyint(4) DEFAULT NULL COMMENT '节点依赖类型：0 当前节点,1 向前执行,2 向后执行', - `max_try_times` tinyint(4) DEFAULT '0' COMMENT '最大重试次数', - `failure_strategy` tinyint(4) DEFAULT '0' COMMENT '失败策略 0 失败后结束，1 失败后继续', - `warning_type` tinyint(4) DEFAULT '0' COMMENT '告警类型：0 不发,1 流程成功发,2 流程失败发,3 成功失败都发', - `warning_group_id` int(11) DEFAULT NULL COMMENT '告警组id', - `schedule_time` datetime DEFAULT NULL COMMENT '预期运行时间', - `command_start_time` datetime DEFAULT NULL COMMENT '开始命令时间', - `global_params` text COMMENT '全局参数（固化流程定义的参数）', - `process_instance_json` longtext COMMENT '流程实例json(copy的流程定义的json)', - `flag` tinyint(4) DEFAULT '1' COMMENT '是否可用，1 可用，0不可用', - `update_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `is_sub_process` int(11) DEFAULT '0' COMMENT '是否是子工作流 1 是，0 不是', - `executor_id` int(11) NOT NULL COMMENT '命令执行者', - `locations` text COMMENT '节点坐标信息', - `connects` text COMMENT '节点连线信息', - `history_cmd` text COMMENT '历史命令，记录所有对流程实例的操作', - `dependence_schedule_times` text COMMENT '依赖节点的预估时间', - `process_instance_priority` int(11) DEFAULT NULL COMMENT '流程实例优先级：0 Highest,1 High,2 Medium,3 Low,4 Lowest', - PRIMARY KEY (`id`), - KEY `process_instance_index` (`process_definition_id`,`id`) USING BTREE, - KEY `start_time_index` (`start_time`) USING BTREE -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_project --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_project`; -CREATE TABLE `t_escheduler_project` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `name` varchar(100) DEFAULT NULL COMMENT '项目名称', - `desc` varchar(200) DEFAULT NULL COMMENT '项目描述', - `user_id` int(11) DEFAULT NULL COMMENT '所属用户', - `flag` tinyint(4) DEFAULT '1' COMMENT '是否可用 1 可用,0 不可用', - `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间', - PRIMARY KEY (`id`), - KEY `user_id_index` (`user_id`) USING BTREE -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_queue --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_queue`; -CREATE TABLE `t_escheduler_queue` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `queue_name` varchar(64) DEFAULT NULL COMMENT '队列名称', - `queue` varchar(64) DEFAULT NULL COMMENT 'yarn队列名称', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_relation_datasource_user --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_relation_datasource_user`; -CREATE TABLE `t_escheduler_relation_datasource_user` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `user_id` int(11) NOT NULL COMMENT '用户id', - `datasource_id` int(11) DEFAULT NULL COMMENT '数据源id', - `perm` int(11) DEFAULT '1' COMMENT '权限', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_relation_process_instance --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_relation_process_instance`; -CREATE TABLE `t_escheduler_relation_process_instance` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `parent_process_instance_id` int(11) DEFAULT NULL COMMENT '父流程实例id', - `parent_task_instance_id` int(11) DEFAULT NULL COMMENT '父任务实例id', - `process_instance_id` int(11) DEFAULT NULL COMMENT '子流程实例id', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_relation_project_user --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_relation_project_user`; -CREATE TABLE `t_escheduler_relation_project_user` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `user_id` int(11) NOT NULL COMMENT '用户id', - `project_id` int(11) DEFAULT NULL COMMENT '项目id', - `perm` int(11) DEFAULT '1' COMMENT '权限', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`), - KEY `user_id_index` (`user_id`) USING BTREE -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_relation_resources_user --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_relation_resources_user`; -CREATE TABLE `t_escheduler_relation_resources_user` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `user_id` int(11) NOT NULL COMMENT '用户id', - `resources_id` int(11) DEFAULT NULL COMMENT '资源id', - `perm` int(11) DEFAULT '1' COMMENT '权限', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_relation_udfs_user --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_relation_udfs_user`; -CREATE TABLE `t_escheduler_relation_udfs_user` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `user_id` int(11) NOT NULL COMMENT '用户id', - `udf_id` int(11) DEFAULT NULL COMMENT 'udf id', - `perm` int(11) DEFAULT '1' COMMENT '权限', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_relation_user_alertgroup --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_relation_user_alertgroup`; -CREATE TABLE `t_escheduler_relation_user_alertgroup` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `alertgroup_id` int(11) DEFAULT NULL COMMENT '组消息id', - `user_id` int(11) DEFAULT NULL COMMENT '用户id', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_resources --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_resources`; -CREATE TABLE `t_escheduler_resources` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `alias` varchar(64) DEFAULT NULL COMMENT '别名', - `file_name` varchar(64) DEFAULT NULL COMMENT '文件名', - `desc` varchar(256) DEFAULT NULL COMMENT '描述', - `user_id` int(11) DEFAULT NULL COMMENT '用户id', - `type` tinyint(4) DEFAULT NULL COMMENT '资源类型，0 FILE，1 UDF', - `size` bigint(20) DEFAULT NULL COMMENT '资源大小', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_schedules --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_schedules`; -CREATE TABLE `t_escheduler_schedules` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `process_definition_id` int(11) NOT NULL COMMENT '流程定义id', - `start_time` datetime NOT NULL COMMENT '调度开始时间', - `end_time` datetime NOT NULL COMMENT '调度结束时间', - `crontab` varchar(256) NOT NULL COMMENT 'crontab 表达式', - `failure_strategy` tinyint(4) NOT NULL COMMENT '失败策略： 0 结束，1 继续', - `user_id` int(11) NOT NULL COMMENT '用户id', - `release_state` tinyint(4) NOT NULL COMMENT '状态：0 未上线，1 上线', - `warning_type` tinyint(4) NOT NULL COMMENT '告警类型：0 不发,1 流程成功发,2 流程失败发,3 成功失败都发', - `warning_group_id` int(11) DEFAULT NULL COMMENT '告警组id', - `process_instance_priority` int(11) DEFAULT NULL COMMENT '流程实例优先级：0 Highest,1 High,2 Medium,3 Low,4 Lowest', - `create_time` datetime NOT NULL COMMENT '创建时间', - `update_time` datetime NOT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_session --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_session`; -CREATE TABLE `t_escheduler_session` ( - `id` varchar(64) NOT NULL COMMENT '主键', - `user_id` int(11) DEFAULT NULL COMMENT '用户id', - `ip` varchar(45) DEFAULT NULL COMMENT '登录ip', - `last_login_time` datetime DEFAULT NULL COMMENT '最后登录时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_task_instance --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_task_instance`; -CREATE TABLE `t_escheduler_task_instance` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `name` varchar(255) DEFAULT NULL COMMENT '任务名称', - `task_type` varchar(64) DEFAULT NULL COMMENT '任务类型', - `process_definition_id` int(11) DEFAULT NULL COMMENT '流程定义id', - `process_instance_id` int(11) DEFAULT NULL COMMENT '流程实例id', - `task_json` longtext COMMENT '任务节点json', - `state` tinyint(4) DEFAULT NULL COMMENT '任务实例状态：0 提交成功,1 正在运行,2 准备暂停,3 暂停,4 准备停止,5 停止,6 失败,7 成功,8 需要容错,9 kill,10 等待线程,11 等待依赖完成', - `submit_time` datetime DEFAULT NULL COMMENT '任务提交时间', - `start_time` datetime DEFAULT NULL COMMENT '任务开始时间', - `end_time` datetime DEFAULT NULL COMMENT '任务结束时间', - `host` varchar(45) DEFAULT NULL COMMENT '执行任务的机器', - `execute_path` varchar(200) DEFAULT NULL COMMENT '任务执行路径', - `log_path` varchar(200) DEFAULT NULL COMMENT '任务日志路径', - `alert_flag` tinyint(4) DEFAULT NULL COMMENT '是否告警', - `retry_times` int(4) DEFAULT '0' COMMENT '重试次数', - `pid` int(4) DEFAULT NULL COMMENT '进程pid', - `app_link` varchar(255) DEFAULT NULL COMMENT 'yarn app id', - `flag` tinyint(4) DEFAULT '1' COMMENT '是否可用：0 不可用，1 可用', - `retry_interval` int(4) DEFAULT NULL COMMENT '重试间隔', - `max_retry_times` int(2) DEFAULT NULL COMMENT '最大重试次数', - `task_instance_priority` int(11) DEFAULT NULL COMMENT '任务实例优先级：0 Highest,1 High,2 Medium,3 Low,4 Lowest', - PRIMARY KEY (`id`), - KEY `process_instance_id` (`process_instance_id`) USING BTREE, - KEY `task_instance_index` (`process_definition_id`,`process_instance_id`) USING BTREE, - CONSTRAINT `foreign_key_instance_id` FOREIGN KEY (`process_instance_id`) REFERENCES `t_escheduler_process_instance` (`id`) ON DELETE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_tenant --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_tenant`; -CREATE TABLE `t_escheduler_tenant` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `tenant_code` varchar(64) DEFAULT NULL COMMENT '租户编码', - `tenant_name` varchar(64) DEFAULT NULL COMMENT '租户名称', - `desc` varchar(256) DEFAULT NULL COMMENT '描述', - `queue_id` int(11) DEFAULT NULL COMMENT '队列id', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_udfs --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_udfs`; -CREATE TABLE `t_escheduler_udfs` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `user_id` int(11) NOT NULL COMMENT '用户id', - `func_name` varchar(100) NOT NULL COMMENT 'UDF函数名', - `class_name` varchar(255) NOT NULL COMMENT '类名', - `type` tinyint(4) NOT NULL COMMENT 'Udf函数类型', - `arg_types` varchar(255) DEFAULT NULL COMMENT '参数', - `database` varchar(255) DEFAULT NULL COMMENT '库名', - `desc` varchar(255) DEFAULT NULL COMMENT '描述', - `resource_id` int(11) NOT NULL COMMENT '资源id', - `resource_name` varchar(255) NOT NULL COMMENT '资源名称', - `create_time` datetime NOT NULL COMMENT '创建时间', - `update_time` datetime NOT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_user --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_user`; -CREATE TABLE `t_escheduler_user` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '用户id', - `user_name` varchar(64) DEFAULT NULL COMMENT '用户名', - `user_password` varchar(64) DEFAULT NULL COMMENT '用户密码', - `user_type` tinyint(4) DEFAULT NULL COMMENT '用户类型：0 管理员，1 普通用户', - `email` varchar(64) DEFAULT NULL COMMENT '邮箱', - `phone` varchar(11) DEFAULT NULL COMMENT '手机', - `tenant_id` int(11) DEFAULT NULL COMMENT '管理员0,普通用户所属租户id', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`), - UNIQUE KEY `user_name_unique` (`user_name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_worker_server --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_worker_server`; -CREATE TABLE `t_escheduler_worker_server` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `host` varchar(45) DEFAULT NULL COMMENT 'ip', - `port` int(11) DEFAULT NULL COMMENT '进程号', - `zk_directory` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL COMMENT 'zk注册目录', - `res_info` varchar(255) DEFAULT NULL COMMENT '集群资源信息：json格式{"cpu":xxx,"memroy":xxx}', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `last_heartbeat_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- Records of t_escheduler_user,user : admin , password : escheduler123 -INSERT INTO `t_escheduler_user` VALUES ('1', 'admin', '055a97b5fcd6d120372ad1976518f371', '0', 'xxx@qq.com', 'xxxx', '0', '2018-03-27 15:48:50', '2018-10-24 17:40:22'); -INSERT INTO `t_escheduler_alertgroup` VALUES (1, 'escheduler管理员告警组', '0', 'escheduler管理员告警组','2018-11-29 10:20:39', '2018-11-29 10:20:39'); -INSERT INTO `t_escheduler_relation_user_alertgroup` VALUES ('1', '1', '1', '2018-11-29 10:22:33', '2018-11-29 10:22:33'); - --- Records of t_escheduler_queue,default queue name : default -INSERT INTO `t_escheduler_queue` VALUES ('1', 'default', 'default'); - - diff --git a/sql/quartz.sql b/sql/quartz.sql deleted file mode 100644 index 22754b39dc..0000000000 --- a/sql/quartz.sql +++ /dev/null @@ -1,179 +0,0 @@ - # - # In your Quartz properties file, you'll need to set - # org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.StdJDBCDelegate - # - # - # By: Ron Cordell - roncordell - # I didn't see this anywhere, so I thought I'd post it here. This is the script from Quartz to create the tables in a MySQL database, modified to use INNODB instead of MYISAM. - - DROP TABLE IF EXISTS QRTZ_FIRED_TRIGGERS; - DROP TABLE IF EXISTS QRTZ_PAUSED_TRIGGER_GRPS; - DROP TABLE IF EXISTS QRTZ_SCHEDULER_STATE; - DROP TABLE IF EXISTS QRTZ_LOCKS; - DROP TABLE IF EXISTS QRTZ_SIMPLE_TRIGGERS; - DROP TABLE IF EXISTS QRTZ_SIMPROP_TRIGGERS; - DROP TABLE IF EXISTS QRTZ_CRON_TRIGGERS; - DROP TABLE IF EXISTS QRTZ_BLOB_TRIGGERS; - DROP TABLE IF EXISTS QRTZ_TRIGGERS; - DROP TABLE IF EXISTS QRTZ_JOB_DETAILS; - DROP TABLE IF EXISTS QRTZ_CALENDARS; - - CREATE TABLE QRTZ_JOB_DETAILS( - SCHED_NAME VARCHAR(120) NOT NULL, - JOB_NAME VARCHAR(200) NOT NULL, - JOB_GROUP VARCHAR(200) NOT NULL, - DESCRIPTION VARCHAR(250) NULL, - JOB_CLASS_NAME VARCHAR(250) NOT NULL, - IS_DURABLE VARCHAR(1) NOT NULL, - IS_NONCONCURRENT VARCHAR(1) NOT NULL, - IS_UPDATE_DATA VARCHAR(1) NOT NULL, - REQUESTS_RECOVERY VARCHAR(1) NOT NULL, - JOB_DATA BLOB NULL, - PRIMARY KEY (SCHED_NAME,JOB_NAME,JOB_GROUP)) - ENGINE=InnoDB; - - CREATE TABLE QRTZ_TRIGGERS ( - SCHED_NAME VARCHAR(120) NOT NULL, - TRIGGER_NAME VARCHAR(200) NOT NULL, - TRIGGER_GROUP VARCHAR(200) NOT NULL, - JOB_NAME VARCHAR(200) NOT NULL, - JOB_GROUP VARCHAR(200) NOT NULL, - DESCRIPTION VARCHAR(250) NULL, - NEXT_FIRE_TIME BIGINT(13) NULL, - PREV_FIRE_TIME BIGINT(13) NULL, - PRIORITY INTEGER NULL, - TRIGGER_STATE VARCHAR(16) NOT NULL, - TRIGGER_TYPE VARCHAR(8) NOT NULL, - START_TIME BIGINT(13) NOT NULL, - END_TIME BIGINT(13) NULL, - CALENDAR_NAME VARCHAR(200) NULL, - MISFIRE_INSTR SMALLINT(2) NULL, - JOB_DATA BLOB NULL, - PRIMARY KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP), - FOREIGN KEY (SCHED_NAME,JOB_NAME,JOB_GROUP) - REFERENCES QRTZ_JOB_DETAILS(SCHED_NAME,JOB_NAME,JOB_GROUP)) - ENGINE=InnoDB; - - CREATE TABLE QRTZ_SIMPLE_TRIGGERS ( - SCHED_NAME VARCHAR(120) NOT NULL, - TRIGGER_NAME VARCHAR(200) NOT NULL, - TRIGGER_GROUP VARCHAR(200) NOT NULL, - REPEAT_COUNT BIGINT(7) NOT NULL, - REPEAT_INTERVAL BIGINT(12) NOT NULL, - TIMES_TRIGGERED BIGINT(10) NOT NULL, - PRIMARY KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP), - FOREIGN KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP) - REFERENCES QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP)) - ENGINE=InnoDB; - - CREATE TABLE QRTZ_CRON_TRIGGERS ( - SCHED_NAME VARCHAR(120) NOT NULL, - TRIGGER_NAME VARCHAR(200) NOT NULL, - TRIGGER_GROUP VARCHAR(200) NOT NULL, - CRON_EXPRESSION VARCHAR(120) NOT NULL, - TIME_ZONE_ID VARCHAR(80), - PRIMARY KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP), - FOREIGN KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP) - REFERENCES QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP)) - ENGINE=InnoDB; - - CREATE TABLE QRTZ_SIMPROP_TRIGGERS - ( - SCHED_NAME VARCHAR(120) NOT NULL, - TRIGGER_NAME VARCHAR(200) NOT NULL, - TRIGGER_GROUP VARCHAR(200) NOT NULL, - STR_PROP_1 VARCHAR(512) NULL, - STR_PROP_2 VARCHAR(512) NULL, - STR_PROP_3 VARCHAR(512) NULL, - INT_PROP_1 INT NULL, - INT_PROP_2 INT NULL, - LONG_PROP_1 BIGINT NULL, - LONG_PROP_2 BIGINT NULL, - DEC_PROP_1 NUMERIC(13,4) NULL, - DEC_PROP_2 NUMERIC(13,4) NULL, - BOOL_PROP_1 VARCHAR(1) NULL, - BOOL_PROP_2 VARCHAR(1) NULL, - PRIMARY KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP), - FOREIGN KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP) - REFERENCES QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP)) - ENGINE=InnoDB; - - CREATE TABLE QRTZ_BLOB_TRIGGERS ( - SCHED_NAME VARCHAR(120) NOT NULL, - TRIGGER_NAME VARCHAR(200) NOT NULL, - TRIGGER_GROUP VARCHAR(200) NOT NULL, - BLOB_DATA BLOB NULL, - PRIMARY KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP), - INDEX (SCHED_NAME,TRIGGER_NAME, TRIGGER_GROUP), - FOREIGN KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP) - REFERENCES QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP)) - ENGINE=InnoDB; - - CREATE TABLE QRTZ_CALENDARS ( - SCHED_NAME VARCHAR(120) NOT NULL, - CALENDAR_NAME VARCHAR(200) NOT NULL, - CALENDAR BLOB NOT NULL, - PRIMARY KEY (SCHED_NAME,CALENDAR_NAME)) - ENGINE=InnoDB; - - CREATE TABLE QRTZ_PAUSED_TRIGGER_GRPS ( - SCHED_NAME VARCHAR(120) NOT NULL, - TRIGGER_GROUP VARCHAR(200) NOT NULL, - PRIMARY KEY (SCHED_NAME,TRIGGER_GROUP)) - ENGINE=InnoDB; - - CREATE TABLE QRTZ_FIRED_TRIGGERS ( - SCHED_NAME VARCHAR(120) NOT NULL, - ENTRY_ID VARCHAR(95) NOT NULL, - TRIGGER_NAME VARCHAR(200) NOT NULL, - TRIGGER_GROUP VARCHAR(200) NOT NULL, - INSTANCE_NAME VARCHAR(200) NOT NULL, - FIRED_TIME BIGINT(13) NOT NULL, - SCHED_TIME BIGINT(13) NOT NULL, - PRIORITY INTEGER NOT NULL, - STATE VARCHAR(16) NOT NULL, - JOB_NAME VARCHAR(200) NULL, - JOB_GROUP VARCHAR(200) NULL, - IS_NONCONCURRENT VARCHAR(1) NULL, - REQUESTS_RECOVERY VARCHAR(1) NULL, - PRIMARY KEY (SCHED_NAME,ENTRY_ID)) - ENGINE=InnoDB; - - CREATE TABLE QRTZ_SCHEDULER_STATE ( - SCHED_NAME VARCHAR(120) NOT NULL, - INSTANCE_NAME VARCHAR(200) NOT NULL, - LAST_CHECKIN_TIME BIGINT(13) NOT NULL, - CHECKIN_INTERVAL BIGINT(13) NOT NULL, - PRIMARY KEY (SCHED_NAME,INSTANCE_NAME)) - ENGINE=InnoDB; - - CREATE TABLE QRTZ_LOCKS ( - SCHED_NAME VARCHAR(120) NOT NULL, - LOCK_NAME VARCHAR(40) NOT NULL, - PRIMARY KEY (SCHED_NAME,LOCK_NAME)) - ENGINE=InnoDB; - - CREATE INDEX IDX_QRTZ_J_REQ_RECOVERY ON QRTZ_JOB_DETAILS(SCHED_NAME,REQUESTS_RECOVERY); - CREATE INDEX IDX_QRTZ_J_GRP ON QRTZ_JOB_DETAILS(SCHED_NAME,JOB_GROUP); - - CREATE INDEX IDX_QRTZ_T_J ON QRTZ_TRIGGERS(SCHED_NAME,JOB_NAME,JOB_GROUP); - CREATE INDEX IDX_QRTZ_T_JG ON QRTZ_TRIGGERS(SCHED_NAME,JOB_GROUP); - CREATE INDEX IDX_QRTZ_T_C ON QRTZ_TRIGGERS(SCHED_NAME,CALENDAR_NAME); - CREATE INDEX IDX_QRTZ_T_G ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_GROUP); - CREATE INDEX IDX_QRTZ_T_STATE ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_STATE); - CREATE INDEX IDX_QRTZ_T_N_STATE ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP,TRIGGER_STATE); - CREATE INDEX IDX_QRTZ_T_N_G_STATE ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_GROUP,TRIGGER_STATE); - CREATE INDEX IDX_QRTZ_T_NEXT_FIRE_TIME ON QRTZ_TRIGGERS(SCHED_NAME,NEXT_FIRE_TIME); - CREATE INDEX IDX_QRTZ_T_NFT_ST ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_STATE,NEXT_FIRE_TIME); - CREATE INDEX IDX_QRTZ_T_NFT_MISFIRE ON QRTZ_TRIGGERS(SCHED_NAME,MISFIRE_INSTR,NEXT_FIRE_TIME); - CREATE INDEX IDX_QRTZ_T_NFT_ST_MISFIRE ON QRTZ_TRIGGERS(SCHED_NAME,MISFIRE_INSTR,NEXT_FIRE_TIME,TRIGGER_STATE); - CREATE INDEX IDX_QRTZ_T_NFT_ST_MISFIRE_GRP ON QRTZ_TRIGGERS(SCHED_NAME,MISFIRE_INSTR,NEXT_FIRE_TIME,TRIGGER_GROUP,TRIGGER_STATE); - - CREATE INDEX IDX_QRTZ_FT_TRIG_INST_NAME ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,INSTANCE_NAME); - CREATE INDEX IDX_QRTZ_FT_INST_JOB_REQ_RCVRY ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,INSTANCE_NAME,REQUESTS_RECOVERY); - CREATE INDEX IDX_QRTZ_FT_J_G ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,JOB_NAME,JOB_GROUP); - CREATE INDEX IDX_QRTZ_FT_JG ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,JOB_GROUP); - CREATE INDEX IDX_QRTZ_FT_T_G ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP); - CREATE INDEX IDX_QRTZ_FT_TG ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,TRIGGER_GROUP); - - commit; \ No newline at end of file From e33d97676bd7986714ff5d3f5e8afa5b2a10cce4 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Wed, 10 Jul 2019 11:06:04 +0800 Subject: [PATCH 12/15] install.sh update --- install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index 33acdf5eec..57cccdf22a 100644 --- a/install.sh +++ b/install.sh @@ -134,7 +134,7 @@ s3Endpoint="http://192.168.199.91:9010" s3AccessKey="A3DXS30FO22544RE" s3SecretKey="OloCLq3n+8+sdPHUhJ21XrSxTC+JK" -# resourcemanager HA配置，如果是单resourcemanager,这里为空即可 +# resourcemanager HA配置，如果是单resourcemanager,这里为yarnHaIps="" yarnHaIps="192.168.xx.xx,192.168.xx.xx" # 如果是单 resourcemanager,只需要配置一个主机名称,如果是resourcemanager HA,则默认配置就好 @@ -144,7 +144,7 @@ singleYarnIp="ark1" hdfsPath="/escheduler" # 拥有在hdfs根路径/下创建目录权限的用户 -# 注意：如果开启了kerberos，则直接hdfsRootUser=，就可以 +# 注意：如果开启了kerberos，则直接hdfsRootUser=""，就可以 hdfsRootUser="hdfs" # common 配置 From 01eaedfad7c3288acdcacb7462cc17dee4cc34d5 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Wed, 10 Jul 2019 11:49:52 +0800 Subject: [PATCH 13/15] determine whether the tenant path exists when the tenant deletes --- .../escheduler/api/service/TenantService.java | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java index 555275273f..21d5f270fb 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java @@ -239,24 +239,25 @@ public class TenantService extends BaseService{ if (PropertyUtils.getResUploadStartupState()){ String tenantPath = HadoopUtils.getHdfsDataBasePath() + "/" + tenant.getTenantCode(); - String resourcePath = HadoopUtils.getHdfsDir(tenant.getTenantCode()); - FileStatus[] fileStatus = HadoopUtils.getInstance().listFileStatus(resourcePath); - if (fileStatus.length > 0) { - putMsg(result, Status.HDFS_TERANT_RESOURCES_FILE_EXISTS); - return result; - } - fileStatus = HadoopUtils.getInstance().listFileStatus(HadoopUtils.getHdfsUdfDir(tenant.getTenantCode())); - if (fileStatus.length > 0) { - putMsg(result, Status.HDFS_TERANT_UDFS_FILE_EXISTS); - return result; - } + if (HadoopUtils.getInstance().exists(tenantPath)){ + String resourcePath = HadoopUtils.getHdfsDir(tenant.getTenantCode()); + FileStatus[] fileStatus = HadoopUtils.getInstance().listFileStatus(resourcePath); + if (fileStatus.length > 0) { + putMsg(result, Status.HDFS_TERANT_RESOURCES_FILE_EXISTS); + return result; + } + fileStatus = HadoopUtils.getInstance().listFileStatus(HadoopUtils.getHdfsUdfDir(tenant.getTenantCode())); + if (fileStatus.length > 0) { + putMsg(result, Status.HDFS_TERANT_UDFS_FILE_EXISTS); + return result; + } - HadoopUtils.getInstance().delete(tenantPath, true); + HadoopUtils.getInstance().delete(tenantPath, true); + } } tenantMapper.deleteById(id); putMsg(result, Status.SUCCESS); - return result; } @@ -269,9 +270,6 @@ public class TenantService extends BaseService{ public Map queryTenantList(User loginUser) { Map result = new HashMap<>(5); -// if (checkAdmin(loginUser, result)) { -// return result; -// } List resourceList = tenantMapper.queryAllTenant(); result.put(Constants.DATA_LIST, resourceList); From a71bae3eb2cdffd514c5979b05758b46fd7b1c38 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Wed, 10 Jul 2019 12:41:59 +0800 Subject: [PATCH 14/15] update change tenant --- .../escheduler/api/service/UsersService.java | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java index 512369ea8a..2d0012c18f 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java @@ -250,30 +250,30 @@ public class UsersService extends BaseService { String oldResourcePath = HadoopUtils.getHdfsDataBasePath() + "/" + oldTenant.getTenantCode() + "/resources"; String oldUdfsPath = HadoopUtils.getHdfsUdfDir(oldTenant.getTenantCode()); - - String newResourcePath = HadoopUtils.getHdfsDataBasePath() + "/" + newTenantCode + "/resources"; - String newUdfsPath = HadoopUtils.getHdfsUdfDir(newTenantCode); - - //file resources list - List fileResourcesList = resourceMapper.queryResourceCreatedByUser(userId, 0); - if (CollectionUtils.isNotEmpty(fileResourcesList)) { - for (Resource resource : fileResourcesList) { - HadoopUtils.getInstance().copy(oldResourcePath + "/" + resource.getAlias(), newResourcePath, false, true); + if (HadoopUtils.getInstance().exists(oldResourcePath)){ + String newResourcePath = HadoopUtils.getHdfsDataBasePath() + "/" + newTenantCode + "/resources"; + String newUdfsPath = HadoopUtils.getHdfsUdfDir(newTenantCode); + + //file resources list + List fileResourcesList = resourceMapper.queryResourceCreatedByUser(userId, 0); + if (CollectionUtils.isNotEmpty(fileResourcesList)) { + for (Resource resource : fileResourcesList) { + HadoopUtils.getInstance().copy(oldResourcePath + "/" + resource.getAlias(), newResourcePath, false, true); + } } - } - //udf resources - List udfResourceList = resourceMapper.queryResourceCreatedByUser(userId, 1); - if (CollectionUtils.isNotEmpty(udfResourceList)) { - for (Resource resource : udfResourceList) { - HadoopUtils.getInstance().copy(oldUdfsPath + "/" + resource.getAlias(), newUdfsPath, false, true); + //udf resources + List udfResourceList = resourceMapper.queryResourceCreatedByUser(userId, 1); + if (CollectionUtils.isNotEmpty(udfResourceList)) { + for (Resource resource : udfResourceList) { + HadoopUtils.getInstance().copy(oldUdfsPath + "/" + resource.getAlias(), newUdfsPath, false, true); + } } - } - - //Delete the user from the old tenant directory - String oldUserPath = HadoopUtils.getHdfsDataBasePath() + "/" + oldTenant.getTenantCode() + "/home/" + userId; - HadoopUtils.getInstance().delete(oldUserPath, true); + //Delete the user from the old tenant directory + String oldUserPath = HadoopUtils.getHdfsDataBasePath() + "/" + oldTenant.getTenantCode() + "/home/" + userId; + HadoopUtils.getInstance().delete(oldUserPath, true); + } //create user in the new tenant directory String newUserPath = HadoopUtils.getHdfsDataBasePath() + "/" + newTenant.getTenantCode() + "/home/" + user.getId(); From 5d0e64f228b75bc290f4907a504a7f0e7aac0ccb Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Wed, 10 Jul 2019 12:54:20 +0800 Subject: [PATCH 15/15] user chanage tenant update --- .../src/main/java/cn/escheduler/api/service/UsersService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java index 2d0012c18f..8004117e91 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java @@ -245,7 +245,7 @@ public class UsersService extends BaseService { Tenant newTenant = tenantMapper.queryById(tenantId); if (newTenant != null) { // if hdfs startup - if (PropertyUtils.getResUploadStartupState()){ + if (PropertyUtils.getResUploadStartupState() && oldTenant != null){ String newTenantCode = newTenant.getTenantCode(); String oldResourcePath = HadoopUtils.getHdfsDataBasePath() + "/" + oldTenant.getTenantCode() + "/resources"; String oldUdfsPath = HadoopUtils.getHdfsUdfDir(oldTenant.getTenantCode());