From d7110c66151f820e0a4574684d9dfa6d3d410f14 Mon Sep 17 00:00:00 2001 From: lenboo Date: Tue, 4 Jun 2019 14:58:14 +0800 Subject: [PATCH 001/149] fix bug: failover process instance would change the state --- .../src/main/java/cn/escheduler/dao/ProcessDao.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java index 09e2149d88..d7dfa6225f 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java @@ -581,6 +581,8 @@ public class ProcessDao extends AbstractBaseDao { processInstance.setScheduleTime(command.getScheduleTime()); } processInstance.setHost(host); + + ExecutionStatus runStatus = ExecutionStatus.RUNNING_EXEUTION; int runTime = processInstance.getRunTimes(); switch (commandType){ case START_PROCESS: @@ -621,6 +623,7 @@ public class ProcessDao extends AbstractBaseDao { case RECOVER_TOLERANCE_FAULT_PROCESS: // recover tolerance fault process processInstance.setRecovery(Flag.YES); + runStatus = processInstance.getState(); break; case COMPLEMENT_DATA: // delete all the valid tasks when complement data @@ -652,7 +655,7 @@ public class ProcessDao extends AbstractBaseDao { default: break; } - processInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + processInstance.setState(runStatus); return processInstance; } From d0ca95ead29da7755805997592152a91ef481faf Mon Sep 17 00:00:00 2001 From: lenboo Date: Sat, 8 Jun 2019 00:18:43 +0800 Subject: [PATCH 002/149] change the failover process. --- .../java/cn/escheduler/dao/ProcessDao.java | 2 +- .../escheduler/server/utils/ProcessUtils.java | 5 +- .../escheduler/server/zk/ZKMasterClient.java | 76 ++++++++++++------- 3 files changed, 52 insertions(+), 31 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java index d7dfa6225f..946bf3496b 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java @@ -58,6 +58,7 @@ public class ProcessDao extends AbstractBaseDao { private final int[] stateArray = new int[]{ExecutionStatus.SUBMITTED_SUCCESS.ordinal(), ExecutionStatus.RUNNING_EXEUTION.ordinal(), ExecutionStatus.READY_PAUSE.ordinal(), + ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal(), ExecutionStatus.READY_STOP.ordinal()}; @Autowired @@ -1569,7 +1570,6 @@ public class ProcessDao extends AbstractBaseDao { for (ProcessInstance processInstance:processInstanceList){ processNeedFailoverProcessInstances(processInstance); } - } @Transactional(value = "TransactionManager",rollbackFor = Exception.class) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java b/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java index baf82de0df..4980ef3b41 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java @@ -273,9 +273,8 @@ public class ProcessUtils { /** * find logs and kill yarn tasks * @param taskInstance - * @throws IOException */ - public static void killYarnJob(TaskInstance taskInstance) throws Exception { + public static void killYarnJob(TaskInstance taskInstance) { try { Thread.sleep(Constants.SLEEP_TIME_MILLIS); LogClient logClient = new LogClient(taskInstance.getHost(), Constants.RPC_PORT); @@ -295,7 +294,7 @@ public class ProcessUtils { } catch (Exception e) { logger.error("kill yarn job failed : " + e.getMessage(),e); - throw new RuntimeException("kill yarn job fail"); +// throw new RuntimeException("kill yarn job fail"); } } } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java index 85f9f6a7d2..fbabb2a82b 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java @@ -30,6 +30,7 @@ import cn.escheduler.dao.model.ProcessInstance; import cn.escheduler.dao.model.TaskInstance; import cn.escheduler.server.ResInfo; import cn.escheduler.server.utils.ProcessUtils; +import org.apache.commons.lang.StringUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.imps.CuratorFrameworkState; import org.apache.curator.framework.recipes.cache.PathChildrenCache; @@ -279,17 +280,9 @@ public class ZKMasterClient extends AbstractZKClient { for (int i = 0; i < Constants.ESCHEDULER_WARN_TIMES_FAILOVER;i++) { alertDao.sendServerStopedAlert(1, masterHost, "Master-Server"); } - - logger.info("start master failover ..."); - - List needFailoverProcessInstanceList = processDao.queryNeedFailoverProcessInstances(masterHost); - - //updateProcessInstance host is null and insert into command - for(ProcessInstance processInstance : needFailoverProcessInstanceList){ - processDao.processNeedFailoverProcessInstances(processInstance); + if(StringUtils.isNotEmpty(masterHost)){ + FailoverMaster(masterHost); } - - logger.info("master failover end"); }catch (Exception e){ logger.error("master failover failed : " + e.getMessage(),e); }finally { @@ -331,6 +324,8 @@ public class ZKMasterClient extends AbstractZKClient { } + + /** * monitor worker */ @@ -369,23 +364,9 @@ public class ZKMasterClient extends AbstractZKClient { alertDao.sendServerStopedAlert(1, workerHost, "Worker-Server"); } - logger.info("start worker failover ..."); - - - List needFailoverTaskInstanceList = processDao.queryNeedFailoverTaskInstances(workerHost); - for(TaskInstance taskInstance : needFailoverTaskInstanceList){ - ProcessInstance instance = processDao.findProcessInstanceDetailById(taskInstance.getProcessInstanceId()); - if(instance!=null){ - taskInstance.setProcessInstance(instance); - } - // only kill yarn job if exists , the local thread has exited - ProcessUtils.killYarnJob(taskInstance); - } - - //updateProcessInstance state value is NEED_FAULT_TOLERANCE - processDao.updateNeedFailoverTaskInstances(workerHost); - - logger.info("worker failover end"); + if(StringUtils.isNotEmpty(workerHost)){ + FailoverWorker(workerHost); + } }catch (Exception e){ logger.error("worker failover failed : " + e.getMessage(),e); } @@ -476,6 +457,46 @@ public class ZKMasterClient extends AbstractZKClient { } + /** + * failover worker tasks + * 1. kill yarn job if there are yarn jobs in tasks. + * 2. change task state from running to need failover. + * @param workerHost + */ + private void FailoverWorker(String workerHost){ + logger.info("start worker[{}] failover ...", workerHost); + + List needFailoverTaskInstanceList = processDao.queryNeedFailoverTaskInstances(workerHost); + for(TaskInstance taskInstance : needFailoverTaskInstanceList){ + ProcessInstance instance = processDao.findProcessInstanceDetailById(taskInstance.getProcessInstanceId()); + if(instance!=null){ + taskInstance.setProcessInstance(instance); + } + // only kill yarn job if exists , the local thread has exited + ProcessUtils.killYarnJob(taskInstance); + } + + //updateProcessInstance state value is NEED_FAULT_TOLERANCE + processDao.updateNeedFailoverTaskInstances(workerHost); + logger.info("end worker[{}] failover ...", workerHost); + } + + /** + * failover master tasks + * @param masterHost + */ + private void FailoverMaster(String masterHost) { + logger.info("start master failover ..."); + + List needFailoverProcessInstanceList = processDao.queryNeedFailoverProcessInstances(masterHost); + + //updateProcessInstance host is null and insert into command + for(ProcessInstance processInstance : needFailoverProcessInstanceList){ + processDao.processNeedFailoverProcessInstances(processInstance); + } + + logger.info("master failover end"); + } /** * get host ip @@ -488,6 +509,7 @@ public class ZKMasterClient extends AbstractZKClient { if(startIndex >= endIndex){ logger.error("parse ip error"); + return ""; } return path.substring(startIndex, endIndex); } From 32fae4380b53bc605219c851a8cbed733964aa4b Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Wed, 12 Jun 2019 17:10:59 +0800 Subject: [PATCH 003/149] add preview schedule execute count --- .../src/main/java/cn/escheduler/common/Constants.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java index b12145973e..41e5aa49e3 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java @@ -827,6 +827,7 @@ public final class Constants { /** - * + * preview schedule execute count */ + public static final int PREVIEW_SCHEDULE_EXECUTE_COUNT = 5; } From 8db1e36ccc710f5d8857cee055f4ed31b9b3c962 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Wed, 12 Jun 2019 17:34:58 +0800 Subject: [PATCH 004/149] add PREVIEW_SCHEDULE_ERROR and PARSE_TO_CRON_EXPRESSION_ERROR --- .../src/main/java/cn/escheduler/api/enums/Status.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java b/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java index 6e25d91825..df4cebac13 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java @@ -161,7 +161,8 @@ public enum Status { SAVE_ERROR(10136, "save error"), DELETE_PROJECT_ERROR_DEFINES_NOT_NULL(10137, "please delete the process definitions in project first!"), BATCH_DELETE_PROCESS_INSTANCE_BY_IDS_ERROR(10117,"batch delete process instance by ids {0} error"), - + PREVIEW_SCHEDULE_ERROR(10139,"preview schedule error"), + PARSE_TO_CRON_EXPRESSION_ERROR(10140,"parse cron to cron expression error"), UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found"), From e44d882aa14e2cd6d80d53774cefcc7a016d42e7 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Wed, 12 Jun 2019 17:53:52 +0800 Subject: [PATCH 005/149] add preview schedule --- .../api/controller/SchedulerController.java | 29 +++++++++++++++ .../api/service/SchedulerService.java | 36 +++++++++++++++++-- 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/controller/SchedulerController.java b/escheduler-api/src/main/java/cn/escheduler/api/controller/SchedulerController.java index ab8cba2d77..37c1ab8672 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/controller/SchedulerController.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/controller/SchedulerController.java @@ -304,4 +304,33 @@ public class SchedulerController extends BaseController { return error(Status.QUERY_SCHEDULE_LIST_ERROR.getCode(), Status.QUERY_SCHEDULE_LIST_ERROR.getMsg()); } } + + /** + * preview schedule + * + * @param loginUser + * @param projectName + * @param schedule + * @return + */ + @ApiOperation(value = "previewSchedule", notes= "PREVIEW_SCHEDULE_NOTES") + @ApiImplicitParams({ + @ApiImplicitParam(name = "schedule", value = "SCHEDULE", dataType = "String", example = "{'startTime':'2019-06-10 00:00:00','endTime':'2019-06-13 00:00:00','crontab':'0 0 3/6 * * ? *'}"), + }) + @PostMapping("/preview") + @ResponseStatus(HttpStatus.CREATED) + public Result previewSchedule(@ApiIgnore @RequestAttribute(value = SESSION_USER) User loginUser, + @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, + @RequestParam(value = "schedule") String schedule + ){ + logger.info("login user {}, project name: {}, preview schedule: {}", + loginUser.getUserName(), projectName, schedule); + try { + Map result = schedulerService.previewSchedule(loginUser, projectName, schedule); + return returnDataList(result); + } catch (Exception e) { + logger.error(PREVIEW_SCHEDULE_ERROR.getMsg(), e); + return error(PREVIEW_SCHEDULE_ERROR.getCode(), PREVIEW_SCHEDULE_ERROR.getMsg()); + } + } } diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java index 5006573702..ab2ef9018a 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java @@ -19,14 +19,13 @@ package cn.escheduler.api.service; import cn.escheduler.api.dto.ScheduleParam; import cn.escheduler.api.enums.Status; -import cn.escheduler.server.quartz.ProcessScheduleJob; -import cn.escheduler.server.quartz.QuartzExecutors; import cn.escheduler.api.utils.Constants; import cn.escheduler.api.utils.PageInfo; import cn.escheduler.common.enums.FailureStrategy; import cn.escheduler.common.enums.Priority; import cn.escheduler.common.enums.ReleaseState; import cn.escheduler.common.enums.WarningType; +import cn.escheduler.common.utils.DateUtils; import cn.escheduler.common.utils.JSONUtils; import cn.escheduler.dao.ProcessDao; import cn.escheduler.dao.mapper.MasterServerMapper; @@ -34,7 +33,11 @@ import cn.escheduler.dao.mapper.ProcessDefinitionMapper; import cn.escheduler.dao.mapper.ProjectMapper; import cn.escheduler.dao.mapper.ScheduleMapper; import cn.escheduler.dao.model.*; +import cn.escheduler.dao.utils.cron.CronUtils; +import cn.escheduler.server.quartz.ProcessScheduleJob; +import cn.escheduler.server.quartz.QuartzExecutors; import org.apache.commons.lang3.StringUtils; +import org.quartz.CronExpression; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -42,6 +45,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import java.io.IOException; +import java.text.ParseException; import java.util.*; /** @@ -537,4 +541,32 @@ public class SchedulerService extends BaseService { } return result; } + + /** + * preview schedule + * @param loginUser + * @param projectName + * @param schedule + * @return + */ + public Map previewSchedule(User loginUser, String projectName, String schedule) { + Map result = new HashMap<>(5); + CronExpression cronExpression; + ScheduleParam scheduleParam = JSONUtils.parseObject(schedule, ScheduleParam.class); + Date now = new Date(); + + Date startTime = now.after(scheduleParam.getStartTime()) ? now : scheduleParam.getStartTime(); + Date endTime = scheduleParam.getEndTime(); + try { + cronExpression = CronUtils.parse2CronExpression(scheduleParam.getCrontab()); + } catch (ParseException e) { + logger.error(e.getMessage(),e); + putMsg(result,Status.PARSE_TO_CRON_EXPRESSION_ERROR); + return result; + } + List selfFireDateList = CronUtils.getSelfFireDateList(startTime, endTime,cronExpression); + result.put(Constants.DATA_LIST, selfFireDateList.stream().map(t -> DateUtils.dateToString(t)).limit(cn.escheduler.common.Constants.PREVIEW_SCHEDULE_EXECUTE_COUNT)); + putMsg(result, Status.SUCCESS); + return result; + } } \ No newline at end of file From 444af9395c29029bee7d4a284c47cc8977c8b4b4 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Wed, 12 Jun 2019 17:56:48 +0800 Subject: [PATCH 006/149] add preview schedule test --- .../api/controller/SchedulerControllerTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/escheduler-api/src/test/java/cn/escheduler/api/controller/SchedulerControllerTest.java b/escheduler-api/src/test/java/cn/escheduler/api/controller/SchedulerControllerTest.java index aee53715c4..e58f436923 100644 --- a/escheduler-api/src/test/java/cn/escheduler/api/controller/SchedulerControllerTest.java +++ b/escheduler-api/src/test/java/cn/escheduler/api/controller/SchedulerControllerTest.java @@ -64,4 +64,17 @@ public class SchedulerControllerTest { Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); } + @Test + public void previewSchedule() throws Exception { + MvcResult mvcResult = mockMvc.perform(post("/projects/{projectName}/schedule/preview","li_test_1") + .header("sessionId", "c24ed9d9-1c20-48a0-bd9c-5cfca14a4dcb") + .param("schedule","{'startTime':'2019-06-10 00:00:00','endTime':'2019-06-13 00:00:00','crontab':'0 0 3/6 * * ? *'}")) + .andExpect(status().isCreated()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) + .andReturn(); + + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); + logger.info(mvcResult.getResponse().getContentAsString()); + } } \ No newline at end of file From b1fbd2170e1635788337ea05326afcb5e3d7ca91 Mon Sep 17 00:00:00 2001 From: lenboo Date: Wed, 12 Jun 2019 19:52:00 +0800 Subject: [PATCH 007/149] chang master/worker failover process. --- .../java/cn/escheduler/dao/ProcessDao.java | 17 ++- .../mapper/ProcessInstanceMapperProvider.java | 7 +- .../mapper/TaskInstanceMapperProvider.java | 7 +- .../dao/mapper/WorkerServerMapper.java | 17 +++ .../mapper/WorkerServerMapperProvider.java | 15 ++ .../cn/escheduler/dao/utils/DagHelper.java | 3 +- .../master/runner/MasterExecThread.java | 27 +++- .../escheduler/server/zk/ZKMasterClient.java | 134 ++++++++++++++---- 8 files changed, 189 insertions(+), 38 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java index 946bf3496b..0241016941 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java @@ -58,7 +58,7 @@ public class ProcessDao extends AbstractBaseDao { private final int[] stateArray = new int[]{ExecutionStatus.SUBMITTED_SUCCESS.ordinal(), ExecutionStatus.RUNNING_EXEUTION.ordinal(), ExecutionStatus.READY_PAUSE.ordinal(), - ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal(), +// ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal(), ExecutionStatus.READY_STOP.ordinal()}; @Autowired @@ -97,6 +97,9 @@ public class ProcessDao extends AbstractBaseDao { @Autowired private ErrorCommandMapper errorCommandMapper; + @Autowired + private WorkerServerMapper workerServerMapper; + /** * task queue impl */ @@ -122,6 +125,7 @@ public class ProcessDao extends AbstractBaseDao { udfFuncMapper = getMapper(UdfFuncMapper.class); resourceMapper = getMapper(ResourceMapper.class); workerGroupMapper = getMapper(WorkerGroupMapper.class); + workerServerMapper = getMapper(WorkerServerMapper.class); taskQueue = TaskQueueFactory.getTaskQueueInstance(); } @@ -1636,6 +1640,17 @@ public class ProcessDao extends AbstractBaseDao { return workerGroupMapper.queryById(workerGroupId); } + /** + * query worker server by host + * @param host + * @return + */ + public List queryWorkerServerByHost(String host){ + + return workerServerMapper.queryWorkerByHost(host); + + } + } diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ProcessInstanceMapperProvider.java b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ProcessInstanceMapperProvider.java index a6d09d431b..9ff57fc3c4 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ProcessInstanceMapperProvider.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ProcessInstanceMapperProvider.java @@ -402,7 +402,12 @@ public class ProcessInstanceMapperProvider { FROM(TABLE_NAME); - WHERE("`host` = #{host} and `state` in (" + strStates.toString() +")"); + Object host = parameter.get("host"); + if(host != null && StringUtils.isNotEmpty(host.toString())){ + + WHERE("`host` = #{host} "); + } + WHERE("`state` in (" + strStates.toString() +")"); ORDER_BY("`id` asc"); diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/TaskInstanceMapperProvider.java b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/TaskInstanceMapperProvider.java index 511b0970fe..ce1e69f197 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/TaskInstanceMapperProvider.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/TaskInstanceMapperProvider.java @@ -228,7 +228,12 @@ public class TaskInstanceMapperProvider { SELECT("*, UNIX_TIMESTAMP(end_time)-UNIX_TIMESTAMP(start_time) as duration"); FROM(TABLE_NAME); - WHERE("`host` = #{host} and `state` in (" + strStates.toString() +")"); + Object host = parameter.get("host"); + if(host != null && StringUtils.isNotEmpty(host.toString())){ + + WHERE("`host` = #{host} "); + } + WHERE("`state` in (" + strStates.toString() +")"); ORDER_BY("`id` asc"); } }.toString(); diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/WorkerServerMapper.java b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/WorkerServerMapper.java index b5ea3aa878..5e511a4edd 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/WorkerServerMapper.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/WorkerServerMapper.java @@ -42,6 +42,23 @@ public interface WorkerServerMapper { @SelectProvider(type = WorkerServerMapperProvider.class, method = "queryAllWorker") List queryAllWorker(); + /** + * query worker list + * + * @return + */ + @Results(value = { + @Result(property = "id", column = "id", javaType = Integer.class, jdbcType = JdbcType.INTEGER), + @Result(property = "host", column = "host", javaType = String.class, jdbcType = JdbcType.VARCHAR), + @Result(property = "port", column = "port", javaType = int.class, jdbcType = JdbcType.INTEGER), + @Result(property = "zkDirectory", column = "zk_directory", javaType = String.class, jdbcType = JdbcType.VARCHAR), + @Result(property = "resInfo", column = "res_info", javaType = String.class, jdbcType = JdbcType.VARCHAR), + @Result(property = "createTime", column = "create_time", javaType = Date.class, jdbcType = JdbcType.TIMESTAMP), + @Result(property = "lastHeartbeatTime", column = "last_heartbeat_time", javaType = Date.class, jdbcType = JdbcType.TIMESTAMP) + }) + @SelectProvider(type = WorkerServerMapperProvider.class, method = "queryWorkerByHost") + List queryWorkerByHost(@Param("host") String host); + /** * insert worker server * diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/WorkerServerMapperProvider.java b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/WorkerServerMapperProvider.java index bd5af7deda..15b330e077 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/WorkerServerMapperProvider.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/WorkerServerMapperProvider.java @@ -37,6 +37,21 @@ public class WorkerServerMapperProvider { }}.toString(); } + /** + * query worker list + * @return + */ + public String queryWorkerByHost(Map parameter) { + return new SQL() {{ + SELECT("*"); + + FROM(TABLE_NAME); + + WHERE("host = #{host}"); + }}.toString(); + } + + /** * insert worker server * @param parameter diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java b/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java index bc52e85062..a80a74d2d2 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java @@ -105,8 +105,7 @@ public class DagHelper { } for (TaskNode taskNode : tmpTaskNodeList) { - if ( !taskNode.isForbidden() - && null == findNodeByName(destTaskNodeList, taskNode.getName())) { + if (null == findNodeByName(destTaskNodeList, taskNode.getName())) { destTaskNodeList.add(taskNode); } } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java b/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java index f12726c3ab..359cff1ac2 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java @@ -411,6 +411,25 @@ public class MasterExecThread implements Runnable { return taskInstance; } + private Collection getStartVertex(String parentNodeName, DAG dag){ + Collection startVertex = null; + if(StringUtils.isNotEmpty(parentNodeName)){ + startVertex = dag.getSubsequentNodes(parentNodeName); + }else{ + startVertex = dag.getBeginNode(); + } + + for(String start : startVertex){ + TaskNode node = dag.getNode(start); + if(node.isForbidden()){ + + } + + } + + return startVertex; + } + /** * get post task instance by node * @@ -421,12 +440,8 @@ public class MasterExecThread implements Runnable { private List getPostTaskInstanceByNode(DAG dag, String parentNodeName){ List postTaskList = new ArrayList<>(); - Collection startVertex = null; - if(StringUtils.isNotEmpty(parentNodeName)){ - startVertex = dag.getSubsequentNodes(parentNodeName); - }else{ - startVertex = dag.getBeginNode(); - } + Collection startVertex = getStartVertex(parentNodeName, dag); + for (String nodeName : startVertex){ // encapsulation task instance diff --git a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java index fbabb2a82b..3066504a46 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java @@ -18,6 +18,7 @@ package cn.escheduler.server.zk; import cn.escheduler.common.Constants; import cn.escheduler.common.enums.ExecutionStatus; +import cn.escheduler.common.enums.ZKNodeType; import cn.escheduler.common.utils.CollectionUtils; import cn.escheduler.common.utils.DateUtils; import cn.escheduler.common.utils.OSUtils; @@ -28,11 +29,11 @@ import cn.escheduler.dao.ProcessDao; import cn.escheduler.dao.ServerDao; import cn.escheduler.dao.model.ProcessInstance; import cn.escheduler.dao.model.TaskInstance; +import cn.escheduler.dao.model.WorkerServer; import cn.escheduler.server.ResInfo; import cn.escheduler.server.utils.ProcessUtils; import org.apache.commons.lang.StringUtils; import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.imps.CuratorFrameworkState; import org.apache.curator.framework.recipes.cache.PathChildrenCache; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; @@ -135,7 +136,8 @@ public class ZKMasterClient extends AbstractZKClient { // check if fault tolerance is required，failure and tolerance if (getActiveMasterNum() == 1) { - processDao.masterStartupFaultTolerant(); +// processDao.masterStartupFaultTolerant(); + failoverMaster(null); } }catch (Exception e){ @@ -191,31 +193,20 @@ public class ZKMasterClient extends AbstractZKClient { Date now = new Date(); createTime = now ; try { + String osHost = OSUtils.getHost(); - // encapsulation master znnode - masterZNode = masterZNodeParentPath + "/" + OSUtils.getHost() + "_"; - List masterZNodeList = zkClient.getChildren().forPath(masterZNodeParentPath); - - if (CollectionUtils.isNotEmpty(masterZNodeList)){ - boolean flag = false; - for (String masterZNode : masterZNodeList){ - if (masterZNode.startsWith(OSUtils.getHost())){ - flag = true; - break; - } - } - - if (flag){ - logger.error("register failure , master already started on host : {}" , OSUtils.getHost()); - // exit system - System.exit(-1); - } + // zookeeper node exists, cannot start a new one. + if(checkZKNodeExists(osHost, ZKNodeType.MASTER)){ + logger.error("register failure , master already started on host : {}" , osHost); + // exit system + System.exit(-1); } // specify the format of stored data in ZK nodes String heartbeatZKInfo = getOsInfo(now); // create temporary sequence nodes for master znode - masterZNode = zkClient.create().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(masterZNode, heartbeatZKInfo.getBytes()); + masterZNode = zkClient.create().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath( + masterZNodeParentPath + "/" + OSUtils.getHost() + "_", heartbeatZKInfo.getBytes()); logger.info("register master node {} success" , masterZNode); @@ -239,6 +230,46 @@ public class ZKMasterClient extends AbstractZKClient { } + /** + * check the zookeeper node already exists + * @param host + * @param zkNodeType + * @return + * @throws Exception + */ + private boolean checkZKNodeExists(String host, ZKNodeType zkNodeType) throws Exception { + + String path = null; + switch (zkNodeType){ + case MASTER: + path = masterZNodeParentPath; + break; + case WORKER: + path = workerZNodeParentPath; + break; + case DEAD_SERVER: + path = deadServerZNodeParentPath; + break; + default: + break; + } + if(StringUtils.isEmpty(path)){ + logger.error("check zk node exists error, host:{}, zk node type:{}", host, zkNodeType.toString()); + return false; + } + + List masterZNodeList = null; + masterZNodeList = zkClient.getChildren().forPath(path); + if (CollectionUtils.isNotEmpty(masterZNodeList)){ + for (String masterZNode : masterZNodeList){ + if (masterZNode.startsWith(host)){ + return true; + } + } + } + return false; + } + /** * monitor master */ @@ -281,7 +312,7 @@ public class ZKMasterClient extends AbstractZKClient { alertDao.sendServerStopedAlert(1, masterHost, "Master-Server"); } if(StringUtils.isNotEmpty(masterHost)){ - FailoverMaster(masterHost); + failoverMaster(masterHost); } }catch (Exception e){ logger.error("master failover failed : " + e.getMessage(),e); @@ -365,7 +396,7 @@ public class ZKMasterClient extends AbstractZKClient { } if(StringUtils.isNotEmpty(workerHost)){ - FailoverWorker(workerHost); + failoverWorker(workerHost, true); } }catch (Exception e){ logger.error("worker failover failed : " + e.getMessage(),e); @@ -457,27 +488,76 @@ public class ZKMasterClient extends AbstractZKClient { } + /** + * task needs failover if task start before worker starts + * + * @param taskInstance + * @return + */ + private boolean checkTaskInstanceNeedFailover(TaskInstance taskInstance) throws Exception { + + boolean taskNeedFailover = true; + + // if the worker node exists in zookeeper, we must check the task starts after the worker + if(checkZKNodeExists(taskInstance.getHost(), ZKNodeType.WORKER)){ + //if task start after worker starts, there is no need to failover the task. + if(checkTaskAfterWorkerStart(taskInstance)){ + taskNeedFailover = false; + } + } + return taskNeedFailover; + } + + /** + * check task start after the worker server starts. + * @param taskInstance + * @return + */ + private boolean checkTaskAfterWorkerStart(TaskInstance taskInstance) { + Date workerServerStartDate = null; + List workerServers = processDao.queryWorkerServerByHost(taskInstance.getHost()); + if(workerServers.size() > 0){ + workerServerStartDate = workerServers.get(0).getCreateTime(); + } + + if(workerServerStartDate != null){ + return taskInstance.getStartTime().after(workerServerStartDate); + + }else{ + return false; + } + } + /** * failover worker tasks * 1. kill yarn job if there are yarn jobs in tasks. * 2. change task state from running to need failover. * @param workerHost */ - private void FailoverWorker(String workerHost){ + private void failoverWorker(String workerHost, boolean needCheckWorkerAlive) throws Exception { logger.info("start worker[{}] failover ...", workerHost); List needFailoverTaskInstanceList = processDao.queryNeedFailoverTaskInstances(workerHost); for(TaskInstance taskInstance : needFailoverTaskInstanceList){ + if(needCheckWorkerAlive){ + if(!checkTaskInstanceNeedFailover(taskInstance)){ + continue; + } + } + ProcessInstance instance = processDao.findProcessInstanceDetailById(taskInstance.getProcessInstanceId()); if(instance!=null){ taskInstance.setProcessInstance(instance); } // only kill yarn job if exists , the local thread has exited ProcessUtils.killYarnJob(taskInstance); + + taskInstance.setState(ExecutionStatus.NEED_FAULT_TOLERANCE); + processDao.saveTaskInstance(taskInstance); } - //updateProcessInstance state value is NEED_FAULT_TOLERANCE - processDao.updateNeedFailoverTaskInstances(workerHost); + //update task Instance state value is NEED_FAULT_TOLERANCE + // processDao.updateNeedFailoverTaskInstances(workerHost); logger.info("end worker[{}] failover ...", workerHost); } @@ -485,7 +565,7 @@ public class ZKMasterClient extends AbstractZKClient { * failover master tasks * @param masterHost */ - private void FailoverMaster(String masterHost) { + private void failoverMaster(String masterHost) { logger.info("start master failover ..."); List needFailoverProcessInstanceList = processDao.queryNeedFailoverProcessInstances(masterHost); From 4e3b655bcc5bf2497d9e208e3d45d06caaf7028a Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Thu, 13 Jun 2019 11:34:56 +0800 Subject: [PATCH 008/149] hdfs add kerberos authentication --- .../java/cn/escheduler/common/Constants.java | 40 ++++++++++++++++--- .../escheduler/common/utils/HadoopUtils.java | 17 ++++++-- .../main/resources/common/common.properties | 16 +++++++- install.sh | 15 ++++++- 4 files changed, 76 insertions(+), 12 deletions(-) diff --git a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java index 41e5aa49e3..07495c5e48 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java @@ -241,7 +241,11 @@ public final class Constants { */ public static final String SCHEDULER_TASKS_QUEUE = "tasks_queue"; + /** + * escheduler need kill tasks queue + */ public static final String SCHEDULER_TASKS_KILL = "tasks_kill"; + public static final String ZOOKEEPER_SCHEDULER_ROOT = "zookeeper.escheduler.root"; public static final String SCHEDULER_QUEUE_IMPL = "escheduler.queue.impl"; @@ -336,11 +340,6 @@ public final class Constants { */ public static final int MAX_TASK_TIMEOUT = 24 * 3600; - /** - * max task timeout - */ - public static final int MAX_PROCESS_TIMEOUT = Integer.MAX_VALUE; - /** * heartbeat threads number @@ -830,4 +829,35 @@ public final class Constants { * preview schedule execute count */ public static final int PREVIEW_SCHEDULE_EXECUTE_COUNT = 5; + + /** + * java.security.krb5.conf + */ + public static final String JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; + + /** + * java.security.krb5.conf.path + */ + public static final String JAVA_SECURITY_KRB5_CONF_PATH = "java.security.krb5.conf.path"; + + /** + * hadoop.security.authentication + */ + public static final String HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authentication"; + + /** + * hadoop.security.authentication + */ + public static final String HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE = "hadoop.security.authentication.startup.state"; + + + /** + * loginUserFromKeytab user + */ + public static final String LOGIN_USER_KEY_TAB_USERNAME = "login.user.keytab.username"; + + /** + * loginUserFromKeytab path + */ + public static final String LOGIN_USER_KEY_TAB_PATH = "login.user.keytab.path"; } diff --git a/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java b/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java index bba9e610fc..bedf030e0c 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java @@ -29,6 +29,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.client.cli.RMAdminCLI; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,9 +41,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import static cn.escheduler.common.Constants.*; -import static cn.escheduler.common.utils.PropertyUtils.getInt; -import static cn.escheduler.common.utils.PropertyUtils.getString; -import static cn.escheduler.common.utils.PropertyUtils.getPrefixedProperties; +import static cn.escheduler.common.utils.PropertyUtils.*; /** * hadoop utils @@ -73,6 +72,16 @@ public class HadoopUtils implements Closeable { if (configuration == null) { try { configuration = new Configuration(); + + if (getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE)){ + System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, + getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)); + configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION,"kerberos"); + UserGroupInformation.setConfiguration(configuration); + UserGroupInformation.loginUserFromKeytab(getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), + getString(Constants.LOGIN_USER_KEY_TAB_PATH)); + } + String defaultFS = configuration.get(FS_DEFAULTFS); //first get key from core-site.xml hdfs-site.xml ,if null ,then try to get from properties file // the default is the local file system @@ -155,7 +164,7 @@ public class HadoopUtils implements Closeable { */ public List catFile(String hdfsFilePath, int skipLineNums, int limit) throws IOException { - if(StringUtils.isBlank(hdfsFilePath)){ + if (StringUtils.isBlank(hdfsFilePath)){ logger.error("hdfs file path:{} is blank",hdfsFilePath); return null; } diff --git a/escheduler-common/src/main/resources/common/common.properties b/escheduler-common/src/main/resources/common/common.properties index 1cb995ba0e..6a40a992ee 100644 --- a/escheduler-common/src/main/resources/common/common.properties +++ b/escheduler-common/src/main/resources/common/common.properties @@ -14,7 +14,19 @@ process.exec.basepath=/tmp/escheduler/exec data.store2hdfs.basepath=/escheduler # whether hdfs starts -hdfs.startup.state=true +hdfs.startup.state=false + +# whether kerberos starts +hadoop.security.authentication.startup.state=false + +# java.security.krb5.conf path +java.security.krb5.conf.path=/opt/krb5.conf + +# loginUserFromKeytab user +login.user.keytab.username="hdfs-mycluster@ESZ.COM" + +# loginUserFromKeytab path +login.user.keytab.path="/opt/hdfs.headless.keytab" # system env path. self configuration, please make sure the directory and file exists and have read write execute permissions escheduler.env.path=/opt/.escheduler_env.sh @@ -23,5 +35,5 @@ escheduler.env.path=/opt/.escheduler_env.sh resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml # is development state? default "false" -development.state=false +development.state=true diff --git a/install.sh b/install.sh index 6fd9e83de2..90ba6e910a 100644 --- a/install.sh +++ b/install.sh @@ -125,7 +125,7 @@ yarnHaIps="192.168.xx.xx,192.168.xx.xx" # 如果是单 resourcemanager,只需要配置一个主机名称,如果是resourcemanager HA,则默认配置就好 singleYarnIp="ark1" -# hdfs根路径,根路径的owner必须是部署用户 +# hdfs根路径，根路径的owner必须是部署用户。1.1.0之前版本不会自动创建hdfs根目录，需要自行创建 hdfsPath="/escheduler" # common 配置 @@ -147,6 +147,19 @@ resSuffixs="txt,log,sh,conf,cfg,py,java,sql,hql,xml" # 开发状态,如果是true,对于SHELL脚本可以在execPath目录下查看封装后的SHELL脚本,如果是false则执行完成直接删除 devState="true" +# kerberos 配置 +# kerberos 是否启动 +kerberosStartUp="false" + +# kdc krb5 配置文件路径 +krb5ConfPath="$installPath/conf/krb5.conf" + +# keytab 用户名 +keytabUserName="hdfs-mycluster@ESZ.COM" + +# 用户 keytab路径 +keytabPath="$installPath/conf/hdfs.headless.keytab" + # zk 配置 # zk根目录 zkRoot="/escheduler" From 72b4c856333df6183fb22f3bc514876ce39eca91 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Thu, 13 Jun 2019 15:48:14 +0800 Subject: [PATCH 009/149] hdfs add kerberos authentication update --- install.sh | 5 +++++ sql/create/release-1.0.0_schema/mysql/escheduler_dml.sql | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 90ba6e910a..cda6fde520 100644 --- a/install.sh +++ b/install.sh @@ -274,6 +274,7 @@ sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${namenodeFs}#g" conf/common/hadoop sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common/hadoop/hadoop.properties sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common/hadoop/hadoop.properties + sed -i ${txt} "s#data.basedir.path.*#data.basedir.path=${programPath}#g" conf/common/common.properties sed -i ${txt} "s#data.download.basedir.path.*#data.download.basedir.path=${downloadPath}#g" conf/common/common.properties sed -i ${txt} "s#process.exec.basepath.*#process.exec.basepath=${execPath}#g" conf/common/common.properties @@ -282,6 +283,10 @@ sed -i ${txt} "s#hdfs.startup.state.*#hdfs.startup.state=${hdfsStartupSate}#g" c sed -i ${txt} "s#escheduler.env.path.*#escheduler.env.path=${shellEnvPath}#g" conf/common/common.properties sed -i ${txt} "s#resource.view.suffixs.*#resource.view.suffixs=${resSuffixs}#g" conf/common/common.properties sed -i ${txt} "s#development.state.*#development.state=${devState}#g" conf/common/common.properties +sed -i ${txt} "s#hadoop.security.authentication.startup.state.*#hadoop.security.authentication.startup.state=${kerberosStartUp}#g" conf/common/common.properties +sed -i ${txt} "s#java.security.krb5.conf.path.*#java.security.krb5.conf.path=${krb5ConfPath}#g" conf/common/common.properties +sed -i ${txt} "s#login.user.keytab.username.*#login.user.keytab.username=${keytabUserName}#g" conf/common/common.properties +sed -i ${txt} "s#login.user.keytab.path.*#login.user.keytab.path=${keytabPath}#g" conf/common/common.properties sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/zookeeper.properties sed -i ${txt} "s#zookeeper.escheduler.root.*#zookeeper.escheduler.root=${zkRoot}#g" conf/zookeeper.properties diff --git a/sql/create/release-1.0.0_schema/mysql/escheduler_dml.sql b/sql/create/release-1.0.0_schema/mysql/escheduler_dml.sql index b7f25d76e1..b075475270 100644 --- a/sql/create/release-1.0.0_schema/mysql/escheduler_dml.sql +++ b/sql/create/release-1.0.0_schema/mysql/escheduler_dml.sql @@ -1,5 +1,5 @@ -- Records of t_escheduler_user,user : admin , password : escheduler123 -INSERT INTO `t_escheduler_user` VALUES ('1', 'admin', '055a97b5fcd6d120372ad1976518f371', '0', '825193156@qq.com', '15001335629', '0', '2018-03-27 15:48:50', '2018-10-24 17:40:22'); +INSERT INTO `t_escheduler_user` VALUES ('1', 'admin', '055a97b5fcd6d120372ad1976518f371', '0', 'xxx@qq.com', 'xx', '0', '2018-03-27 15:48:50', '2018-10-24 17:40:22'); INSERT INTO `t_escheduler_alertgroup` VALUES (1, 'escheduler管理员告警组', '0', 'escheduler管理员告警组','2018-11-29 10:20:39', '2018-11-29 10:20:39'); INSERT INTO `t_escheduler_relation_user_alertgroup` VALUES ('1', '1', '1', '2018-11-29 10:22:33', '2018-11-29 10:22:33'); From de5736bec02e6f22a5ba20a0579ff8e57e98a43c Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Thu, 13 Jun 2019 17:33:33 +0800 Subject: [PATCH 010/149] add --- .../java/cn/escheduler/common/Constants.java | 5 ++++ .../escheduler/common/utils/HadoopUtils.java | 27 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java index 07495c5e48..ddd27170db 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java @@ -70,6 +70,11 @@ public final class Constants { */ public static final String YARN_APPLICATION_STATUS_ADDRESS = "yarn.application.status.address"; + /** + * hdfs configuration + * hdfs.root.user + */ + public static final String HDFS_ROOT_USER = "hdfs.root.user"; /** * hdfs configuration diff --git a/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java b/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java index bedf030e0c..71a2eb712d 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java @@ -35,6 +35,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; +import java.security.PrivilegedAction; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -63,6 +66,30 @@ public class HadoopUtils implements Closeable { return instance; } + /** + * init escheduler root path in hdfs + */ + private void initHdfsPath(){ + String hdfsUser = getString(Constants.HDFS_ROOT_USER); + String hdfsPath = getString(Constants.DATA_STORE_2_HDFS_BASEPATH); + Path path = new Path(hdfsPath); + try { + UserGroupInformation ugi = UserGroupInformation.createProxyUser(hdfsUser,UserGroupInformation.getLoginUser()); + ugi.doAs(new PrivilegedExceptionAction() { + @Override + public Boolean run() throws Exception { + + if(!fs.exists(path)){ + return fs.mkdirs(path); + } + return true; + } + }); + } catch (Exception e) { + logger.error(e.getMessage(),e); + } + } + /** * init hadoop configuration */ From 6444a7af7e8cbfda41f7b212cf666732613a9fde Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Thu, 13 Jun 2019 17:55:20 +0800 Subject: [PATCH 011/149] add hdfsRootUser --- install.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install.sh b/install.sh index 90ba6e910a..36ac3fed65 100644 --- a/install.sh +++ b/install.sh @@ -128,6 +128,9 @@ singleYarnIp="ark1" # hdfs根路径，根路径的owner必须是部署用户。1.1.0之前版本不会自动创建hdfs根目录，需要自行创建 hdfsPath="/escheduler" +# 拥有在hdfs根路径/下创建目录权限的用户 +hdfsRootUser="hdfs" + # common 配置 # 程序路径 programPath="/tmp/escheduler" From dff5a26a2a260f72bc63eae8d735d11686a52339 Mon Sep 17 00:00:00 2001 From: lenboo Date: Thu, 13 Jun 2019 17:59:58 +0800 Subject: [PATCH 012/149] add zk node type --- .../cn/escheduler/common/enums/ZKNodeType.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 escheduler-common/src/main/java/cn/escheduler/common/enums/ZKNodeType.java diff --git a/escheduler-common/src/main/java/cn/escheduler/common/enums/ZKNodeType.java b/escheduler-common/src/main/java/cn/escheduler/common/enums/ZKNodeType.java new file mode 100644 index 0000000000..371231f727 --- /dev/null +++ b/escheduler-common/src/main/java/cn/escheduler/common/enums/ZKNodeType.java @@ -0,0 +1,15 @@ +package cn.escheduler.common.enums; + +/** + * zk node type + */ +public enum ZKNodeType { + + /** + * 0 do not send warning; + * 1 send if process success; + * 2 send if process failed; + * 3 send if process ending; + */ + MASTER, WORKER, DEAD_SERVER, TASK_QUEUE; +} From 2d9351cba0c39be433c704c70ba782a8352d5cc6 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Thu, 13 Jun 2019 18:00:17 +0800 Subject: [PATCH 013/149] add hdfs.root.user --- escheduler-common/src/main/resources/common/common.properties | 3 +++ 1 file changed, 3 insertions(+) diff --git a/escheduler-common/src/main/resources/common/common.properties b/escheduler-common/src/main/resources/common/common.properties index 6a40a992ee..357ba2cbc6 100644 --- a/escheduler-common/src/main/resources/common/common.properties +++ b/escheduler-common/src/main/resources/common/common.properties @@ -10,6 +10,9 @@ data.download.basedir.path=/tmp/escheduler/download # process execute directory. self configuration, please make sure the directory exists and have read write permissions process.exec.basepath=/tmp/escheduler/exec +# Users who have permission to create directories under the HDFS root path +hdfs.root.user=hdfs + # data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/escheduler" is recommended data.store2hdfs.basepath=/escheduler From 69f8eea5f00e0b00ddcbaaa41451ac8bbc927e3b Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Thu, 13 Jun 2019 18:03:29 +0800 Subject: [PATCH 014/149] add sed hdfsRootUser --- install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/install.sh b/install.sh index 36ac3fed65..de2b06478c 100644 --- a/install.sh +++ b/install.sh @@ -280,6 +280,7 @@ sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.addre sed -i ${txt} "s#data.basedir.path.*#data.basedir.path=${programPath}#g" conf/common/common.properties sed -i ${txt} "s#data.download.basedir.path.*#data.download.basedir.path=${downloadPath}#g" conf/common/common.properties sed -i ${txt} "s#process.exec.basepath.*#process.exec.basepath=${execPath}#g" conf/common/common.properties +sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common/common.properties sed -i ${txt} "s#data.store2hdfs.basepath.*#data.store2hdfs.basepath=${hdfsPath}#g" conf/common/common.properties sed -i ${txt} "s#hdfs.startup.state.*#hdfs.startup.state=${hdfsStartupSate}#g" conf/common/common.properties sed -i ${txt} "s#escheduler.env.path.*#escheduler.env.path=${shellEnvPath}#g" conf/common/common.properties From bb786b28c69f47e0d55142dba5c85c094567fd81 Mon Sep 17 00:00:00 2001 From: lenboo Date: Fri, 14 Jun 2019 15:23:51 +0800 Subject: [PATCH 015/149] change forbidden nodes run process. --- .../cn/escheduler/dao/utils/DagHelper.java | 112 ++++++++++++++++++ .../master/runner/MasterExecThread.java | 61 ++-------- .../server/master/MasterCommandTest.java | 40 +++++++ 3 files changed, 162 insertions(+), 51 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java b/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java index a80a74d2d2..58267f78db 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java @@ -18,16 +18,22 @@ package cn.escheduler.dao.utils; import cn.escheduler.common.enums.TaskDependType; +import cn.escheduler.common.graph.DAG; import cn.escheduler.common.model.TaskNode; import cn.escheduler.common.model.TaskNodeRelation; import cn.escheduler.common.process.ProcessDag; import cn.escheduler.common.utils.JSONUtils; import cn.escheduler.dao.model.ProcessData; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; /** * dag tools @@ -192,6 +198,24 @@ public class DagHelper { return processDag; } + /** + * parse the forbidden task nodes in process definition. + * @param processDefinitionJson + * @return + */ + public static Map getForbiddenTaskNodeMaps(String processDefinitionJson){ + Map forbidTaskNodeMap = new ConcurrentHashMap<>(); + ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); + + List taskNodeList = processData.getTasks(); + for(TaskNode node : taskNodeList){ + if(node.isForbidden()){ + forbidTaskNodeMap.putIfAbsent(node.getName(), node); + } + } + return forbidTaskNodeMap; + } + /** * find node by node name @@ -209,4 +233,92 @@ public class DagHelper { } return null; } + + + /** + * get start vertex in one dag + * it would find the post node if the start vertex is forbidden running + * @param parentNodeName the previous node + * @param dag + * @return + */ + public static Collection getStartVertex(String parentNodeName, DAG dag){ + Collection startVertexs = null; + if(StringUtils.isNotEmpty(parentNodeName)){ + startVertexs = dag.getSubsequentNodes(parentNodeName); + }else{ + startVertexs = dag.getBeginNode(); + } + + Collection tmpStartVertexs = new ArrayList<>(); + tmpStartVertexs.addAll(startVertexs); + + for(String start : tmpStartVertexs){ + TaskNode startNode = dag.getNode(start); + if(!startNode.isForbidden()){ + continue; + } + Collection postNodes = getStartVertex(start, dag); + + for(String post : postNodes){ + if(checkForbiddenPostCanSubmit(post, dag)){ + startVertexs.add(post); + } + } + startVertexs.remove(start); + } + + return startVertexs; + } + + /** + * + * @param postNodeName + * @param dag + * @return + */ + private static boolean checkForbiddenPostCanSubmit(String postNodeName, DAG dag){ + + TaskNode postNode = dag.getNode(postNodeName); + List dependList = postNode.getDepList(); + + for(String dependNodeName : dependList){ + TaskNode dependNode = dag.getNode(dependNodeName); + if(!dependNode.isForbidden()){ + return false; + } + } + return true; + } + + + + /*** + * generate dag graph + * @param processDag + * @return + */ + public static DAG buildDagGraph(ProcessDag processDag) { + + DAG dag = new DAG<>(); + + /** + * add vertex + */ + if (CollectionUtils.isNotEmpty(processDag.getNodes())){ + for (TaskNode node : processDag.getNodes()){ + dag.addNode(node.getName(),node); + } + } + + /** + * add edge + */ + if (CollectionUtils.isNotEmpty(processDag.getEdges())){ + for (TaskNodeRelation edge : processDag.getEdges()){ + dag.addEdge(edge.getStartNode(),edge.getEndNode()); + } + } + return dag; + } } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java b/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java index 359cff1ac2..d4997e48d5 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java @@ -79,6 +79,7 @@ public class MasterExecThread implements Runnable { private Map completeTaskList = new ConcurrentHashMap<>(); private Map readyToSubmitTaskList = new ConcurrentHashMap<>(); private Map dependFailedTask = new ConcurrentHashMap<>(); + private Map forbiddenTaskList = new ConcurrentHashMap<>(); private List recoverToleranceFaultTaskList = new ArrayList<>(); private AlertManager alertManager = new AlertManager(); @@ -269,6 +270,7 @@ public class MasterExecThread implements Runnable { private void buildFlowDag() throws Exception { recoverNodeIdList = getStartTaskInstanceList(processInstance.getCommandParam()); + forbiddenTaskList = DagHelper.getForbiddenTaskNodeMaps(processInstance.getProcessInstanceJson()); // generate process to get DAG info List recoveryNameList = getRecoveryNodeNameList(); List startNodeNameList = parseStartNodeName(processInstance.getCommandParam()); @@ -279,7 +281,8 @@ public class MasterExecThread implements Runnable { return; } // generate process dag - dag = buildDagGraph(processDag); + dag = DagHelper.buildDagGraph(processDag); + } private void initTaskQueue(){ @@ -411,24 +414,7 @@ public class MasterExecThread implements Runnable { return taskInstance; } - private Collection getStartVertex(String parentNodeName, DAG dag){ - Collection startVertex = null; - if(StringUtils.isNotEmpty(parentNodeName)){ - startVertex = dag.getSubsequentNodes(parentNodeName); - }else{ - startVertex = dag.getBeginNode(); - } - - for(String start : startVertex){ - TaskNode node = dag.getNode(start); - if(node.isForbidden()){ - } - - } - - return startVertex; - } /** * get post task instance by node @@ -440,10 +426,12 @@ public class MasterExecThread implements Runnable { private List getPostTaskInstanceByNode(DAG dag, String parentNodeName){ List postTaskList = new ArrayList<>(); - Collection startVertex = getStartVertex(parentNodeName, dag); + Collection startVertex = DagHelper.getStartVertex(parentNodeName, dag); + if(startVertex == null){ + return postTaskList; + } for (String nodeName : startVertex){ - // encapsulation task instance TaskInstance taskInstance = createTaskInstance(processInstance, nodeName , dag.getNode(nodeName),parentNodeName); @@ -532,8 +520,8 @@ public class MasterExecThread implements Runnable { List depsNameList = taskNode.getDepList(); for(String depsNode : depsNameList ){ - // dependencies must be all complete - if(!completeTaskList.containsKey(depsNode)){ + // dependencies must be fully completed or run prohibited + if(!completeTaskList.containsKey(depsNode) || !forbiddenTaskList.containsKey(depsNode)){ return DependResult.WAITING; } ExecutionStatus taskState = completeTaskList.get(depsNode).getState(); @@ -919,35 +907,6 @@ public class MasterExecThread implements Runnable { } } - /*** - * generate dag graph - * @param processDag - * @return - */ - public DAG buildDagGraph(ProcessDag processDag) { - - DAG dag = new DAG<>(); - - /** - * add vertex - */ - if (CollectionUtils.isNotEmpty(processDag.getNodes())){ - for (TaskNode node : processDag.getNodes()){ - dag.addNode(node.getName(),node); - } - } - - /** - * add edge - */ - if (CollectionUtils.isNotEmpty(processDag.getEdges())){ - for (TaskNodeRelation edge : processDag.getEdges()){ - dag.addEdge(edge.getStartNode(),edge.getEndNode()); - } - } - return dag; - } - /** * whether the retry interval is timed out * @param taskInstance diff --git a/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java b/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java index 3aed212618..cfa723105d 100644 --- a/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java +++ b/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java @@ -18,15 +18,27 @@ package cn.escheduler.server.master; import cn.escheduler.common.enums.CommandType; import cn.escheduler.common.enums.FailureStrategy; +import cn.escheduler.common.enums.TaskDependType; import cn.escheduler.common.enums.WarningType; +import cn.escheduler.common.graph.DAG; +import cn.escheduler.common.model.TaskNode; +import cn.escheduler.common.model.TaskNodeRelation; +import cn.escheduler.common.process.ProcessDag; import cn.escheduler.dao.datasource.ConnectionFactory; import cn.escheduler.dao.mapper.CommandMapper; +import cn.escheduler.dao.mapper.ProcessDefinitionMapper; import cn.escheduler.dao.model.Command; +import cn.escheduler.dao.model.ProcessDefinition; +import cn.escheduler.dao.utils.DagHelper; +import org.codehaus.jackson.SerializableString; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.Collection; + /** * master test */ @@ -36,9 +48,14 @@ public class MasterCommandTest { private CommandMapper commandMapper; + private ProcessDefinitionMapper processDefinitionMapper; + + @Before public void before(){ + commandMapper = ConnectionFactory.getSqlSession().getMapper(CommandMapper.class); + processDefinitionMapper = ConnectionFactory.getSqlSession().getMapper(ProcessDefinitionMapper.class); } @@ -104,4 +121,27 @@ public class MasterCommandTest { } + @Test + public void testDagHelper(){ + + ProcessDefinition processDefinition = processDefinitionMapper.queryByDefineId(19); + + try { + ProcessDag processDag = DagHelper.generateFlowDag(processDefinition.getProcessDefinitionJson(), + new ArrayList<>(), new ArrayList<>(), TaskDependType.TASK_POST); + + DAG dag = DagHelper.buildDagGraph(processDag); + Collection start = DagHelper.getStartVertex("1", dag); + + System.out.println(start.toString()); + + } catch (Exception e) { + e.printStackTrace(); + } + + } + + + + } From a7db4c3f2479c6af25d77247cddde7a69e9ec691 Mon Sep 17 00:00:00 2001 From: lenboo Date: Fri, 14 Jun 2019 16:33:17 +0800 Subject: [PATCH 016/149] change forbidden task process. --- .../escheduler/server/master/runner/MasterExecThread.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java b/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java index d4997e48d5..118922c822 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java @@ -520,8 +520,11 @@ public class MasterExecThread implements Runnable { List depsNameList = taskNode.getDepList(); for(String depsNode : depsNameList ){ - // dependencies must be fully completed or run prohibited - if(!completeTaskList.containsKey(depsNode) || !forbiddenTaskList.containsKey(depsNode)){ + if(forbiddenTaskList.containsKey(depsNode)){ + continue; + } + // dependencies must be fully completed + if(!completeTaskList.containsKey(depsNode)){ return DependResult.WAITING; } ExecutionStatus taskState = completeTaskList.get(depsNode).getState(); From 79b05a440573e401f22aafa3f233f1626248b02b Mon Sep 17 00:00:00 2001 From: lenboo Date: Fri, 14 Jun 2019 16:58:48 +0800 Subject: [PATCH 017/149] change forbidden task process --- .../main/java/cn/escheduler/dao/utils/DagHelper.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java b/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java index 58267f78db..c101d2791c 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java @@ -250,10 +250,10 @@ public class DagHelper { startVertexs = dag.getBeginNode(); } - Collection tmpStartVertexs = new ArrayList<>(); + List tmpStartVertexs = new ArrayList<>(); tmpStartVertexs.addAll(startVertexs); - for(String start : tmpStartVertexs){ + for(String start : startVertexs){ TaskNode startNode = dag.getNode(start); if(!startNode.isForbidden()){ continue; @@ -262,13 +262,13 @@ public class DagHelper { for(String post : postNodes){ if(checkForbiddenPostCanSubmit(post, dag)){ - startVertexs.add(post); + tmpStartVertexs.add(post); } } - startVertexs.remove(start); + tmpStartVertexs.remove(start); } - return startVertexs; + return tmpStartVertexs; } /** From 0865aeca39f866f05856c011c88e05afacdc95eb Mon Sep 17 00:00:00 2001 From: lenboo Date: Fri, 14 Jun 2019 18:01:43 +0800 Subject: [PATCH 018/149] change forbidden task process. --- .../src/main/java/cn/escheduler/dao/utils/DagHelper.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java b/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java index c101d2791c..d811cf78bc 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java @@ -251,7 +251,9 @@ public class DagHelper { } List tmpStartVertexs = new ArrayList<>(); - tmpStartVertexs.addAll(startVertexs); + if(startVertexs!= null){ + tmpStartVertexs.addAll(startVertexs); + } for(String start : startVertexs){ TaskNode startNode = dag.getNode(start); From 4f09ff36ddea487bd54ada8b4b5167aca57bd191 Mon Sep 17 00:00:00 2001 From: lenboo Date: Fri, 14 Jun 2019 18:06:54 +0800 Subject: [PATCH 019/149] update --- .../cn/escheduler/server/master/runner/MasterExecThread.java | 3 +++ .../src/main/java/cn/escheduler/server/zk/ZKMasterClient.java | 1 + .../java/cn/escheduler/server/master/MasterCommandTest.java | 3 +++ 3 files changed, 7 insertions(+) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java b/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java index 118922c822..cb38c7a722 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java @@ -432,6 +432,9 @@ public class MasterExecThread implements Runnable { } for (String nodeName : startVertex){ + if(completeTaskList.containsKey(nodeName)){ + continue; + } // encapsulation task instance TaskInstance taskInstance = createTaskInstance(processInstance, nodeName , dag.getNode(nodeName),parentNodeName); diff --git a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java index 3066504a46..85c805e2fc 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java @@ -136,6 +136,7 @@ public class ZKMasterClient extends AbstractZKClient { // check if fault tolerance is required，failure and tolerance if (getActiveMasterNum() == 1) { + failoverWorker(null, true); // processDao.masterStartupFaultTolerant(); failoverMaster(null); } diff --git a/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java b/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java index cfa723105d..114d9d3a84 100644 --- a/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java +++ b/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java @@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Collection; +import java.util.Map; /** * master test @@ -135,6 +136,8 @@ public class MasterCommandTest { System.out.println(start.toString()); + Map forbidden = DagHelper.getForbiddenTaskNodeMaps(processDefinition.getProcessDefinitionJson()); + System.out.println(forbidden); } catch (Exception e) { e.printStackTrace(); } From 1cac85f2effc928a853f5ab2f8d878d710837394 Mon Sep 17 00:00:00 2001 From: lenboo Date: Fri, 14 Jun 2019 18:31:16 +0800 Subject: [PATCH 020/149] update --- .../cn/escheduler/dao/utils/DagHelper.java | 18 +++++++++++------- .../server/master/runner/MasterExecThread.java | 5 +---- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java b/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java index d811cf78bc..bc509b1e99 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java @@ -24,15 +24,13 @@ import cn.escheduler.common.model.TaskNodeRelation; import cn.escheduler.common.process.ProcessDag; import cn.escheduler.common.utils.JSONUtils; import cn.escheduler.dao.model.ProcessData; +import cn.escheduler.dao.model.TaskInstance; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; /** @@ -240,9 +238,15 @@ public class DagHelper { * it would find the post node if the start vertex is forbidden running * @param parentNodeName the previous node * @param dag + * @param completeTaskList * @return */ - public static Collection getStartVertex(String parentNodeName, DAG dag){ + public static Collection getStartVertex(String parentNodeName, DAG dag, + Map completeTaskList){ + + if(completeTaskList == null){ + completeTaskList = new HashMap<>(); + } Collection startVertexs = null; if(StringUtils.isNotEmpty(parentNodeName)){ startVertexs = dag.getSubsequentNodes(parentNodeName); @@ -257,10 +261,10 @@ public class DagHelper { for(String start : startVertexs){ TaskNode startNode = dag.getNode(start); - if(!startNode.isForbidden()){ + if(!startNode.isForbidden() && !completeTaskList.containsKey(start)){ continue; } - Collection postNodes = getStartVertex(start, dag); + Collection postNodes = getStartVertex(start, dag, completeTaskList); for(String post : postNodes){ if(checkForbiddenPostCanSubmit(post, dag)){ diff --git a/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java b/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java index cb38c7a722..9ce08d691a 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java @@ -426,15 +426,12 @@ public class MasterExecThread implements Runnable { private List getPostTaskInstanceByNode(DAG dag, String parentNodeName){ List postTaskList = new ArrayList<>(); - Collection startVertex = DagHelper.getStartVertex(parentNodeName, dag); + Collection startVertex = DagHelper.getStartVertex(parentNodeName, dag, completeTaskList); if(startVertex == null){ return postTaskList; } for (String nodeName : startVertex){ - if(completeTaskList.containsKey(nodeName)){ - continue; - } // encapsulation task instance TaskInstance taskInstance = createTaskInstance(processInstance, nodeName , dag.getNode(nodeName),parentNodeName); From 1a09f56d5011b31280be1db34763ab9d549a8e51 Mon Sep 17 00:00:00 2001 From: lenboo Date: Mon, 17 Jun 2019 10:28:03 +0800 Subject: [PATCH 021/149] update --- .../src/main/resources/dao/data_source.properties | 6 +++--- .../java/cn/escheduler/server/master/MasterCommandTest.java | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/escheduler-dao/src/main/resources/dao/data_source.properties b/escheduler-dao/src/main/resources/dao/data_source.properties index cac3aa5e20..3c89dd1fd2 100644 --- a/escheduler-dao/src/main/resources/dao/data_source.properties +++ b/escheduler-dao/src/main/resources/dao/data_source.properties @@ -1,9 +1,9 @@ # base spring data source configuration spring.datasource.type=com.alibaba.druid.pool.DruidDataSource spring.datasource.driver-class-name=com.mysql.jdbc.Driver -spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/escheduler?characterEncoding=UTF-8 -spring.datasource.username=xx -spring.datasource.password=xx +spring.datasource.url=jdbc:mysql://192.168.220.188:3306/escheduler_new?characterEncoding=UTF-8 +spring.datasource.username=root +spring.datasource.password=root@123 # connection configuration spring.datasource.initialSize=5 diff --git a/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java b/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java index 114d9d3a84..9c71795179 100644 --- a/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java +++ b/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java @@ -30,7 +30,6 @@ import cn.escheduler.dao.mapper.ProcessDefinitionMapper; import cn.escheduler.dao.model.Command; import cn.escheduler.dao.model.ProcessDefinition; import cn.escheduler.dao.utils.DagHelper; -import org.codehaus.jackson.SerializableString; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; @@ -132,7 +131,7 @@ public class MasterCommandTest { new ArrayList<>(), new ArrayList<>(), TaskDependType.TASK_POST); DAG dag = DagHelper.buildDagGraph(processDag); - Collection start = DagHelper.getStartVertex("1", dag); + Collection start = DagHelper.getStartVertex("1", dag, null); System.out.println(start.toString()); From b482eedd3e3a5c4c5775222560257af16e233ad9 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Mon, 17 Jun 2019 18:30:36 +0800 Subject: [PATCH 022/149] add init hdfs path --- .../escheduler/common/utils/HadoopUtils.java | 45 +++++++++++-------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java b/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java index 71a2eb712d..85716bec81 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java @@ -24,19 +24,14 @@ import com.alibaba.fastjson.JSONObject; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.*; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.client.cli.RMAdminCLI; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; -import java.security.PrivilegedAction; -import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; import java.util.List; import java.util.Map; @@ -54,12 +49,18 @@ public class HadoopUtils implements Closeable { private static final Logger logger = LoggerFactory.getLogger(HadoopUtils.class); + private static String hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER); private static volatile HadoopUtils instance = new HadoopUtils(); private static volatile Configuration configuration; private static FileSystem fs; + private HadoopUtils(){ + if(StringUtils.isEmpty(hdfsUser)){ + hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER); + } init(); + initHdfsPath(); } public static HadoopUtils getInstance(){ @@ -70,26 +71,19 @@ public class HadoopUtils implements Closeable { * init escheduler root path in hdfs */ private void initHdfsPath(){ - String hdfsUser = getString(Constants.HDFS_ROOT_USER); String hdfsPath = getString(Constants.DATA_STORE_2_HDFS_BASEPATH); Path path = new Path(hdfsPath); - try { - UserGroupInformation ugi = UserGroupInformation.createProxyUser(hdfsUser,UserGroupInformation.getLoginUser()); - ugi.doAs(new PrivilegedExceptionAction() { - @Override - public Boolean run() throws Exception { - if(!fs.exists(path)){ - return fs.mkdirs(path); - } - return true; - } - }); + try { + if (!fs.exists(path)) { + fs.mkdirs(path); + } } catch (Exception e) { logger.error(e.getMessage(),e); } } + /** * init hadoop configuration */ @@ -127,7 +121,20 @@ public class HadoopUtils implements Closeable { } if (fs == null) { - fs = FileSystem.get(configuration); + if(StringUtils.isNotEmpty(hdfsUser)){ + //UserGroupInformation ugi = UserGroupInformation.createProxyUser(hdfsUser,UserGroupInformation.getLoginUser()); + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser); + ugi.doAs(new PrivilegedExceptionAction() { + @Override + public Boolean run() throws Exception { + fs = FileSystem.get(configuration); + return true; + } + }); + }else{ + logger.warn("hdfs.root.user is not set value!"); + fs = FileSystem.get(configuration); + } } String rmHaIds = getString(YARN_RESOURCEMANAGER_HA_RM_IDS); String appAddress = getString(Constants.YARN_APPLICATION_STATUS_ADDRESS); From 9c369ccde492025c550cc40ee2fbd47763248344 Mon Sep 17 00:00:00 2001 From: lenboo Date: Fri, 21 Jun 2019 16:42:00 +0800 Subject: [PATCH 023/149] move queue count and command count to statistics. --- .../home/pages/projects/pages/index/index.vue | 24 ------------------- .../src/js/conf/home/router/index.js | 8 +++++++ .../components/secondaryMenu/_source/menu.js | 18 +++++++++++++- .../src/js/module/i18n/locale/en_US.js | 8 ++++++- .../src/js/module/i18n/locale/zh_CN.js | 6 +++++ 5 files changed, 38 insertions(+), 26 deletions(-) diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue index 28f61f9e65..f695f3a678 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue @@ -33,30 +33,6 @@ -
-
-
-
-
- {{$t('Queue statistics')}} -
-
- - -
-
-
-
-
-
- {{$t('Command status statistics')}} -
-
- - -
-
-
diff --git a/escheduler-ui/src/js/conf/home/router/index.js b/escheduler-ui/src/js/conf/home/router/index.js index 97a7e81a10..c1aa86d6ec 100644 --- a/escheduler-ui/src/js/conf/home/router/index.js +++ b/escheduler-ui/src/js/conf/home/router/index.js @@ -439,6 +439,14 @@ const router = new Router({ meta: { title: `Mysql` } + }, + { + path: '/monitor/servers/statistics', + name: 'statistics', + component: resolve => require(['../pages/monitor/pages/servers/statistics'], resolve), + meta: { + title: `statistics` + } } ] } diff --git a/escheduler-ui/src/js/module/components/secondaryMenu/_source/menu.js b/escheduler-ui/src/js/module/components/secondaryMenu/_source/menu.js index 20182ce0b1..1c0aefbec7 100644 --- a/escheduler-ui/src/js/module/components/secondaryMenu/_source/menu.js +++ b/escheduler-ui/src/js/module/components/secondaryMenu/_source/menu.js @@ -193,7 +193,7 @@ let menu = { monitor: [ { name: `${i18n.$t('Servers manage')}`, - id: 0, + id: 1, path: '', isOpen: true, disabled: true, @@ -242,6 +242,22 @@ let menu = { disabled: true } ] + }, + { + name: `${i18n.$t('Statistics manage')}`, + id: 0, + path: '', + isOpen: true, + disabled: true, + icon: 'fa-server', + children: [ + { + name: "Statistics", + path: 'statistics', + id: 0, + disabled: true + } + ] } ] } diff --git a/escheduler-ui/src/js/module/i18n/locale/en_US.js b/escheduler-ui/src/js/module/i18n/locale/en_US.js index fbc621e18b..cf81d9c95c 100644 --- a/escheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/escheduler-ui/src/js/module/i18n/locale/en_US.js @@ -458,5 +458,11 @@ export default { 'Post Statement': 'Post Statement', 'Statement cannot be empty': 'Statement cannot be empty', 'Process Define Count': 'Process Define Count', - 'Process Instance Running Count': 'Process Instance Running Count' + 'Process Instance Running Count': 'Process Instance Running Count', + 'process number of waiting for running': 'process number of waiting for running', + 'failure command number': 'failure command number', + 'tasks number of waiting running': 'tasks number of waiting running', + 'task number of ready to kill': '待杀死任务数', + 'Statistics manage': 'Statistics manage', + 'statistics': 'statistics', } diff --git a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js index d77c55715a..78ef8323aa 100644 --- a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -460,4 +460,10 @@ export default { 'Process Define Count': '流程定义个数', 'Process Instance Running Count': '运行流程实例个数', 'Please select a queue': '请选择队列', + 'process number of waiting for running': '待执行的流程数', + 'failure command number': '执行失败的命令数', + 'tasks number of waiting running': '待运行任务数', + 'task number of ready to kill': '待杀死任务数', + 'Statistics manage': '统计管理', + 'statistics': '统计', } From ef4c36d256f7b1a4d2439d5e5cf94f84fcb4816b Mon Sep 17 00:00:00 2001 From: lenboo Date: Fri, 21 Jun 2019 16:48:35 +0800 Subject: [PATCH 024/149] update --- .../monitor/pages/servers/statistics.vue | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 escheduler-ui/src/js/conf/home/pages/monitor/pages/servers/statistics.vue diff --git a/escheduler-ui/src/js/conf/home/pages/monitor/pages/servers/statistics.vue b/escheduler-ui/src/js/conf/home/pages/monitor/pages/servers/statistics.vue new file mode 100644 index 0000000000..e6302fdd1f --- /dev/null +++ b/escheduler-ui/src/js/conf/home/pages/monitor/pages/servers/statistics.vue @@ -0,0 +1,115 @@ + + + + From b49d218df46ce2d60847a46fadeaadd833a7132a Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Mon, 24 Jun 2019 14:15:03 +0800 Subject: [PATCH 025/149] user list display admin --- .../main/java/cn/escheduler/dao/mapper/UserMapperProvider.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/UserMapperProvider.java b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/UserMapperProvider.java index 4a7df12a0f..363cd38f6a 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/UserMapperProvider.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/UserMapperProvider.java @@ -187,7 +187,6 @@ public class UserMapperProvider { return new SQL() {{ SELECT("count(0)"); FROM(TABLE_NAME); - WHERE("user_type = 1"); Object searchVal = parameter.get("searchVal"); if(searchVal != null && StringUtils.isNotEmpty(searchVal.toString())){ WHERE( " user_name like concat('%', #{searchVal}, '%') "); @@ -209,7 +208,6 @@ public class UserMapperProvider { FROM(TABLE_NAME + " u "); LEFT_OUTER_JOIN("t_escheduler_tenant t on u.tenant_id = t.id"); LEFT_OUTER_JOIN("t_escheduler_queue q on t.queue_id = q.id"); - WHERE("u.user_type = 1"); Object searchVal = parameter.get("searchVal"); if(searchVal != null && StringUtils.isNotEmpty(searchVal.toString())){ WHERE( " u.user_name like concat('%', #{searchVal}, '%') "); From 7a33a5f1901db9e8a4b0dc0dd00958f4afd67cd4 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Mon, 24 Jun 2019 14:58:44 +0800 Subject: [PATCH 026/149] datasource project resource for admin operate --- .../cn/escheduler/api/service/DataSourceService.java | 3 ++- .../java/cn/escheduler/api/service/ProjectService.java | 9 --------- .../java/cn/escheduler/api/service/ResourcesService.java | 3 ++- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java index 36b9b45861..5e6fac13e0 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java @@ -21,6 +21,7 @@ import cn.escheduler.api.utils.Constants; import cn.escheduler.api.utils.PageInfo; import cn.escheduler.api.utils.Result; import cn.escheduler.common.enums.DbType; +import cn.escheduler.common.enums.UserType; import cn.escheduler.common.job.db.*; import cn.escheduler.dao.mapper.DataSourceMapper; import cn.escheduler.dao.mapper.DatasourceUserMapper; @@ -537,7 +538,7 @@ public class DataSourceService extends BaseService{ putMsg(result, Status.RESOURCE_NOT_EXIST); return result; } - if(loginUser.getId() != dataSource.getUserId()){ + if(loginUser.getId() != dataSource.getUserId() || loginUser.getUserType() != UserType.ADMIN_USER){ putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ProjectService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ProjectService.java index c90d4da779..30d8f827aa 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ProjectService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ProjectService.java @@ -76,15 +76,6 @@ public class ProjectService extends BaseService{ return descCheck; } - /** - * only general users can create projects. administrators have no corresponding tenants and can only view - * 管理员没有对应的租户,只能查看,只有普通用户才可以创建项目 - */ - if (!userService.isGeneral(loginUser)) { - putMsg(result, Status.USER_NO_OPERATION_PERM); - return result; - } - Project project = projectMapper.queryByName(name); if (project != null) { putMsg(result, Status.PROJECT_ALREADY_EXISTS, name); diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java index 291c59ea22..0dea2e00d3 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java @@ -21,6 +21,7 @@ import cn.escheduler.api.utils.Constants; import cn.escheduler.api.utils.PageInfo; import cn.escheduler.api.utils.Result; import cn.escheduler.common.enums.ResourceType; +import cn.escheduler.common.enums.UserType; import cn.escheduler.common.utils.FileUtils; import cn.escheduler.common.utils.HadoopUtils; import cn.escheduler.common.utils.PropertyUtils; @@ -399,7 +400,7 @@ public class ResourcesService extends BaseService { putMsg(result, Status.RESOURCE_NOT_EXIST); return result; } - if (loginUser.getId() != resource.getUserId()) { + if (loginUser.getId() != resource.getUserId() || loginUser.getUserType() != UserType.ADMIN_USER) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } From ab05b7edcaeab716c6bbff471874af341b3406b5 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Mon, 24 Jun 2019 15:20:11 +0800 Subject: [PATCH 027/149] datasource and resource for admin operation --- .../main/java/cn/escheduler/api/service/DataSourceService.java | 2 +- .../main/java/cn/escheduler/api/service/ResourcesService.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java index 5e6fac13e0..9081a436cf 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java @@ -538,7 +538,7 @@ public class DataSourceService extends BaseService{ putMsg(result, Status.RESOURCE_NOT_EXIST); return result; } - if(loginUser.getId() != dataSource.getUserId() || loginUser.getUserType() != UserType.ADMIN_USER){ + if(loginUser.getId() != dataSource.getUserId() && loginUser.getUserType() != UserType.ADMIN_USER){ putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java index 0dea2e00d3..b59e43387b 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java @@ -400,7 +400,7 @@ public class ResourcesService extends BaseService { putMsg(result, Status.RESOURCE_NOT_EXIST); return result; } - if (loginUser.getId() != resource.getUserId() || loginUser.getUserType() != UserType.ADMIN_USER) { + if (loginUser.getId() != resource.getUserId() && loginUser.getUserType() != UserType.ADMIN_USER) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } From 15ce0733df2002984c8ac79fcbb480f653994cfd Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Mon, 24 Jun 2019 18:11:53 +0800 Subject: [PATCH 028/149] admin support delete process define --- .../cn/escheduler/api/service/ProcessDefinitionService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessDefinitionService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessDefinitionService.java index 45ff487f5e..76c7173f02 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessDefinitionService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessDefinitionService.java @@ -24,6 +24,7 @@ import cn.escheduler.api.utils.PageInfo; import cn.escheduler.common.enums.Flag; import cn.escheduler.common.enums.ReleaseState; import cn.escheduler.common.enums.TaskType; +import cn.escheduler.common.enums.UserType; import cn.escheduler.common.graph.DAG; import cn.escheduler.common.model.TaskNode; import cn.escheduler.common.model.TaskNodeRelation; @@ -365,7 +366,7 @@ public class ProcessDefinitionService extends BaseDAGService { } // Determine if the login user is the owner of the process definition - if (loginUser.getId() != processDefinition.getUserId()) { + if (loginUser.getId() != processDefinition.getUserId() && loginUser.getUserType() != UserType.ADMIN_USER) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } From 947f3ade4c82ea4a6d984e8c2a878746428bd366 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Mon, 24 Jun 2019 18:18:11 +0800 Subject: [PATCH 029/149] =?UTF-8?q?admin=20support=20project=EF=BC=8Cdatas?= =?UTF-8?q?ource=20and=20resource=20operation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../js/conf/home/pages/dag/_source/config.js | 2 +- .../js/conf/home/pages/dag/_source/dag.vue | 22 +++++++++---------- .../pages/dag/_source/formModel/formModel.vue | 2 +- .../pages/dag/_source/plugIn/jsPlumbHandle.js | 2 +- .../conf/home/pages/dag/_source/udp/udp.vue | 2 +- .../datasource/pages/list/_source/list.vue | 4 +--- .../pages/datasource/pages/list/index.vue | 2 +- .../definition/pages/list/_source/list.vue | 15 ++++++------- .../definition/pages/list/_source/start.vue | 2 +- .../pages/definition/pages/list/index.vue | 2 +- .../instance/pages/list/_source/list.vue | 8 +------ .../projects/pages/list/_source/list.vue | 6 ++--- .../home/pages/projects/pages/list/index.vue | 4 ++-- .../pages/file/pages/create/index.vue | 2 +- .../pages/file/pages/list/_source/list.vue | 14 +++++------- .../resource/pages/file/pages/list/index.vue | 6 ++--- .../pages/udf/pages/function/_source/list.vue | 8 +++---- .../pages/udf/pages/function/index.vue | 2 +- .../pages/udf/pages/resource/_source/list.vue | 9 +++----- .../pages/udf/pages/resource/index.vue | 2 +- .../pages/users/_source/createUser.vue | 11 +++++++--- .../security/pages/users/_source/list.vue | 10 ++++++++- .../src/js/module/mixin/disabledState.js | 4 ++-- 23 files changed, 67 insertions(+), 74 deletions(-) diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/config.js b/escheduler-ui/src/js/conf/home/pages/dag/_source/config.js index 740846890c..9c1065870f 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/config.js +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/config.js @@ -26,7 +26,7 @@ import Permissions from '@/module/permissions' * @desc tooltip */ const toolOper = (dagThis) => { - let disabled = Permissions.getAuth() === false ? false : !dagThis.$store.state.dag.isDetails + let disabled =!dagThis.$store.state.dag.isDetails// Permissions.getAuth() === false ? false : !dagThis.$store.state.dag.isDetails return [ { code: 'pointer', diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue b/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue index 37c4119676..2c0422c701 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue @@ -8,7 +8,7 @@ :id="v" v-for="(item,v) in tasksTypeList" @mousedown="_getDagId(v)"> -
+
@@ -68,10 +68,9 @@ type="primary" size="xsmall" :loading="spinnerLoading" - v-ps="['GENERAL_USER']" @click="_saveChart" icon="fa fa-save" - :disabled="isDetails"> + > {{spinnerLoading ? 'Loading...' : $t('Save')}}
@@ -205,9 +204,9 @@ * @param item */ _getDagId (v) { - if (this.isDetails) { - return - } + // if (this.isDetails) { + // return + // } this.dagBarId = v }, /** @@ -239,11 +238,12 @@ }) }, _operationClass (item) { - if (item.disable) { - return this.toolOperCode === item.code ? 'active' : '' - } else { - return 'disable' - } + return this.toolOperCode === item.code ? 'active' : '' + // if (item.disable) { + // return this.toolOperCode === item.code ? 'active' : '' + // } else { + // return 'disable' + // } }, /** * Storage interface diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index 7f36417a3a..863a44abf5 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -165,7 +165,7 @@
{{$t('Cancel')}} - {{spinnerLoading ? 'Loading...' : $t('Confirm add')}} + {{spinnerLoading ? 'Loading...' : $t('Confirm add')}}
diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js b/escheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js index e2c0c677a9..440deb48fb 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js @@ -71,7 +71,7 @@ JSP.prototype.init = function ({ dag, instance }) { this.setConfig({ isDrag: !store.state.dag.isDetails, isAttachment: false, - isNewNodes: Permissions.getAuth() === false ? false : !store.state.dag.isDetails, + isNewNodes: !store.state.dag.isDetails,//Permissions.getAuth() === false ? false : !store.state.dag.isDetails, isDblclick: true, isContextmenu: true, isClick: false diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue b/escheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue index 8e11dca356..f7de034db5 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue @@ -62,7 +62,7 @@
{{$t('Cancel')}} - {{$t('Add')}} + {{$t('Add')}} diff --git a/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/list.vue b/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/list.vue index b32aeeb6bc..a08051bc35 100644 --- a/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/list.vue +++ b/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/list.vue @@ -56,7 +56,6 @@ shape="circle" size="xsmall" data-toggle="tooltip" - v-ps="['GENERAL_USER']" :title="$t('Edit')" icon="iconfont icon-bianjixiugai" @click="_edit(item)"> @@ -77,8 +76,7 @@ size="xsmall" icon="iconfont icon-shanchu" data-toggle="tooltip" - :title="$t('delete')" - v-ps="['GENERAL_USER']"> + :title="$t('delete')"> diff --git a/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/index.vue b/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/index.vue index de94e0ee21..8d5012700f 100644 --- a/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/index.vue +++ b/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/index.vue @@ -3,7 +3,7 @@ diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue index aa2be8ef8e..f9e8dba231 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue @@ -58,12 +58,12 @@ - - - - - - - + + + + + + + :title="$t('delete')"> diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue index ddb6b0a156..c2e3c33728 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue @@ -137,7 +137,7 @@
{{$t('Cancel')}} - {{spinnerLoading ? 'Loading...' : $t('Start')}} + {{spinnerLoading ? 'Loading...' : $t('Start')}}
diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/index.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/index.vue index bc63896c17..bf8612dd98 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/index.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/index.vue @@ -3,7 +3,7 @@ diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue index f98383c558..2bc1cad066 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue @@ -73,7 +73,6 @@ data-toggle="tooltip" :title="$t('Edit')" @click="_reEdit(item)" - v-ps="['GENERAL_USER']" icon="iconfont icon-bianjixiugai" :disabled="item.state !== 'SUCCESS' && item.state !== 'PAUSE' && item.state !== 'FAILURE' && item.state !== 'STOP'"> + :title="$t('delete')"> diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/list/_source/list.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/list/_source/list.vue index 087b50032a..21a58fc1ea 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/list/_source/list.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/list/_source/list.vue @@ -63,8 +63,7 @@ data-toggle="tooltip" :title="$t('Edit')" @click="_edit(item)" - icon="iconfont icon-bianjixiugai" - v-ps="['GENERAL_USER']"> + icon="iconfont icon-bianjixiugai"> + icon="iconfont icon-shanchu"> diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/list/index.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/list/index.vue index 6031b590e0..7b2f555192 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/list/index.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/list/index.vue @@ -3,7 +3,7 @@ @@ -113,4 +113,4 @@ }, components: { mListConstruction, mSpin, mConditions, mList, mCreateProject, mNoData } } - \ No newline at end of file + diff --git a/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue b/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue index d08ed7f8dd..bf3ebe044d 100644 --- a/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue +++ b/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue @@ -49,7 +49,7 @@ diff --git a/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/_source/list.vue b/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/_source/list.vue index 40c03ff7a0..89acc74dd6 100644 --- a/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/_source/list.vue +++ b/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/_source/list.vue @@ -51,8 +51,7 @@ :title="$t('Edit')" :disabled="_rtDisb(item)" @click="_edit(item,$index)" - icon="iconfont icon-bianjixiugai" - v-ps="['GENERAL_USER']"> + icon="iconfont icon-bianjixiugai"> + @click="_rename(item,$index)"> + icon="iconfont icon-download"> + :title="$t('delete')"> @@ -210,4 +206,4 @@ }, components: { } } - \ No newline at end of file + diff --git a/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/index.vue b/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/index.vue index 6555d7a6d4..7434772899 100644 --- a/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/index.vue +++ b/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/index.vue @@ -4,8 +4,8 @@ @@ -98,4 +98,4 @@ }, components: { mListConstruction, mConditions, mList, mSpin, mNoData } } - \ No newline at end of file + diff --git a/escheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/list.vue b/escheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/list.vue index 8d96ede359..ad0c510c87 100644 --- a/escheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/list.vue +++ b/escheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/list.vue @@ -1,4 +1,4 @@ - diff --git a/escheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/resource/index.vue b/escheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/resource/index.vue index ed9a81a705..228501dbed 100644 --- a/escheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/resource/index.vue +++ b/escheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/resource/index.vue @@ -3,7 +3,7 @@ diff --git a/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue b/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue index 9d3ab042d8..b02db7848e 100644 --- a/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue +++ b/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue @@ -98,7 +98,9 @@ userName: '', userPassword: '', tenantId: {}, - queueName: {}, + queueName: { + id:'' + }, email: '', phone: '', tenantList: [], @@ -197,6 +199,7 @@ }, _submit () { this.$refs['popup'].spinnerLoading = true + console.log(this.tenantId.id) let param = { userName: this.userName, userPassword: this.userPassword, @@ -205,9 +208,11 @@ queue: this.queueName.code, phone: this.phone } + if (this.item) { param.id = this.item.id } + this.store.dispatch(`security/${this.item ? 'updateUser' : 'createUser'}`, param).then(res => { setTimeout(() => { this.$refs['popup'].spinnerLoading = false @@ -232,7 +237,7 @@ this.phone = this.item.phone this.tenantId = _.find(this.tenantList, ['id', this.item.tenantId]) this.$nextTick(() => { - this.queueName = _.find(this.queueList, ['code', this.item.queue]) + this.queueName = _.find(this.queueList, ['code', this.item.queue])||{id:''} }) } }) @@ -243,7 +248,7 @@ this.email = this.item.email this.phone = this.item.phone this.tenantId.id = this.item.tenantId - this.queueName = { queue: this.item.queue } + this.queueName = { queue: this.item.queue} } } }, diff --git a/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/list.vue b/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/list.vue index 125a3dfd93..e97886a61b 100644 --- a/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/list.vue +++ b/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/list.vue @@ -9,6 +9,9 @@ {{$t('User Name')}} + + 用户类型 + {{$t('Tenant')}} @@ -21,6 +24,7 @@ {{$t('Phone')}} + {{$t('Create Time')}} @@ -40,6 +44,9 @@ {{item.userName || '-'}} + + {{item.userType === 'GENERAL_USER' ? `${$t('Ordinary users')}` : `${$t('Administrator')}`}} + {{item.tenantName || '-'}} {{item.queue || '-'}} @@ -62,7 +69,7 @@ {{$t('UDF Function')}} @@ -84,6 +91,7 @@ size="xsmall" data-toggle="tooltip" :title="$t('delete')" + :disabled="item.userType === 'ADMIN_USER'" icon="iconfont icon-shanchu"> diff --git a/escheduler-ui/src/js/module/mixin/disabledState.js b/escheduler-ui/src/js/module/mixin/disabledState.js index 7c0b1f8e92..4b814a1908 100644 --- a/escheduler-ui/src/js/module/mixin/disabledState.js +++ b/escheduler-ui/src/js/module/mixin/disabledState.js @@ -28,11 +28,11 @@ export default { } }, created () { - this.isDetails = Permissions.getAuth() ? this.store.state.dag.isDetails : true + this.isDetails =this.store.state.dag.isDetails// Permissions.getAuth() ? this.store.state.dag.isDetails : true }, computed: { _isDetails () { - return this.isDetails ? 'icon-disabled' : '' + return ''// this.isDetails ? 'icon-disabled' : '' } } } From 9135b1420e30aedaf3ba49ecff6ef53d8868784b Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Mon, 24 Jun 2019 19:33:17 +0800 Subject: [PATCH 030/149] add resource upload s3 --- .../api/service/ResourcesService.java | 40 ++++----- .../escheduler/api/service/TenantService.java | 4 +- .../api/service/UdfFuncService.java | 12 +-- .../escheduler/api/service/UsersService.java | 6 +- .../java/cn/escheduler/common/Constants.java | 21 ++++- .../common/enums/ResUploadType.java | 29 +++++++ .../escheduler/common/utils/HadoopUtils.java | 87 +++++++++++-------- .../common/utils/PropertyUtils.java | 14 ++- .../main/resources/common/common.properties | 4 +- .../resources/common/hadoop/hadoop.properties | 12 ++- install.sh | 24 +++-- 11 files changed, 168 insertions(+), 85 deletions(-) create mode 100644 escheduler-common/src/main/java/cn/escheduler/common/enums/ResUploadType.java diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java index b59e43387b..4f2c62271c 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java @@ -86,8 +86,8 @@ public class ResourcesService extends BaseService { Result result = new Result(); // if hdfs not startup - if (!PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ - logger.error("hdfs startup state: {}", PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)); + if (!PropertyUtils.getResUploadStartupState()){ + logger.error("resource upload startup state: {}", PropertyUtils.getResUploadStartupState()); putMsg(result, Status.HDFS_NOT_STARTUP); return result; } @@ -185,9 +185,9 @@ public class ResourcesService extends BaseService { ResourceType type) { Result result = new Result(); - // if hdfs not startup - if (!PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ - logger.error("hdfs startup state: {}", PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)); + // if resource upload startup + if (!PropertyUtils.getResUploadStartupState()){ + logger.error("resource upload startup state: {}", PropertyUtils.getResUploadStartupState()); putMsg(result, Status.HDFS_NOT_STARTUP); return result; } @@ -386,9 +386,9 @@ public class ResourcesService extends BaseService { public Result delete(User loginUser, int resourceId) throws Exception { Result result = new Result(); - // if hdfs not startup - if (!PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ - logger.error("hdfs startup state: {}", PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)); + // if resource upload startup + if (!PropertyUtils.getResUploadStartupState()){ + logger.error("resource upload startup state: {}", PropertyUtils.getResUploadStartupState()); putMsg(result, Status.HDFS_NOT_STARTUP); return result; } @@ -449,9 +449,9 @@ public class ResourcesService extends BaseService { public Result readResource(int resourceId, int skipLineNum, int limit) { Result result = new Result(); - // if hdfs not startup - if (!PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ - logger.error("hdfs startup state: {}", PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)); + // if resource upload startup + if (!PropertyUtils.getResUploadStartupState()){ + logger.error("resource upload startup state: {}", PropertyUtils.getResUploadStartupState()); putMsg(result, Status.HDFS_NOT_STARTUP); return result; } @@ -510,9 +510,9 @@ public class ResourcesService extends BaseService { @Transactional(value = "TransactionManager",rollbackFor = Exception.class) public Result onlineCreateResource(User loginUser, ResourceType type, String fileName, String fileSuffix, String desc, String content) { Result result = new Result(); - // if hdfs not startup - if (!PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ - logger.error("hdfs startup state: {}", PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)); + // if resource upload startup + if (!PropertyUtils.getResUploadStartupState()){ + logger.error("resource upload startup state: {}", PropertyUtils.getResUploadStartupState()); putMsg(result, Status.HDFS_NOT_STARTUP); return result; } @@ -573,9 +573,9 @@ public class ResourcesService extends BaseService { public Result updateResourceContent(int resourceId, String content) { Result result = new Result(); - // if hdfs not startup - if (!PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ - logger.error("hdfs startup state: {}", PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)); + // if resource upload startup + if (!PropertyUtils.getResUploadStartupState()){ + logger.error("resource upload startup state: {}", PropertyUtils.getResUploadStartupState()); putMsg(result, Status.HDFS_NOT_STARTUP); return result; } @@ -663,9 +663,9 @@ public class ResourcesService extends BaseService { * @return */ public org.springframework.core.io.Resource downloadResource(int resourceId) throws Exception { - // if hdfs not startup - if (!PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ - logger.error("hdfs startup state: {}", PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)); + // if resource upload startup + if (!PropertyUtils.getResUploadStartupState()){ + logger.error("resource upload startup state: {}", PropertyUtils.getResUploadStartupState()); throw new RuntimeException("hdfs not startup"); } diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java index 68fbc55348..7538071e3a 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java @@ -96,7 +96,7 @@ public class TenantService extends BaseService{ tenantMapper.insert(tenant); // if hdfs startup - if (PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ + if (PropertyUtils.getResUploadStartupState()){ String resourcePath = HadoopUtils.getHdfsDataBasePath() + "/" + tenantCode + "/resources"; String udfsPath = HadoopUtils.getHdfsUdfDir(tenantCode); /** @@ -178,7 +178,7 @@ public class TenantService extends BaseService{ Tenant newTenant = tenantMapper.queryByTenantCode(tenantCode); if (newTenant == null){ // if hdfs startup - if (PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ + if (PropertyUtils.getResUploadStartupState()){ String resourcePath = HadoopUtils.getHdfsDataBasePath() + "/" + tenantCode + "/resources"; String udfsPath = HadoopUtils.getHdfsUdfDir(tenantCode); //init hdfs resource diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/UdfFuncService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/UdfFuncService.java index 52e605f711..8ca399d17f 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/UdfFuncService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/UdfFuncService.java @@ -80,9 +80,9 @@ public class UdfFuncService extends BaseService{ int resourceId) { Result result = new Result(); - // if hdfs not startup - if (!PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ - logger.error("hdfs startup state: {}", PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)); + // if resource upload startup + if (!PropertyUtils.getResUploadStartupState()){ + logger.error("resource upload startup state: {}", PropertyUtils.getResUploadStartupState()); putMsg(result, Status.HDFS_NOT_STARTUP); return result; } @@ -167,9 +167,9 @@ public class UdfFuncService extends BaseService{ // verify udfFunc is exist UdfFunc udf = udfFuncMapper.queryUdfById(udfFuncId); - // if hdfs not startup - if (!PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ - logger.error("hdfs startup state: {}", PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)); + // if resource upload startup + if (!PropertyUtils.getResUploadStartupState()){ + logger.error("resource upload startup state: {}", PropertyUtils.getResUploadStartupState()); putMsg(result, Status.HDFS_NOT_STARTUP); return result; } diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java index 5db8662c07..c0dab22b69 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java @@ -125,7 +125,7 @@ public class UsersService extends BaseService { Tenant tenant = tenantMapper.queryById(tenantId); // if hdfs startup - if (PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ + if (PropertyUtils.getResUploadStartupState()){ String userPath = HadoopUtils.getHdfsDataBasePath() + "/" + tenant.getTenantCode() + "/home/" + user.getId(); HadoopUtils.getInstance().mkdir(userPath); @@ -245,7 +245,7 @@ public class UsersService extends BaseService { Tenant newTenant = tenantMapper.queryById(tenantId); if (newTenant != null) { // if hdfs startup - if (PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ + if (PropertyUtils.getResUploadStartupState()){ String newTenantCode = newTenant.getTenantCode(); String oldResourcePath = HadoopUtils.getHdfsDataBasePath() + "/" + oldTenant.getTenantCode() + "/resources"; String oldUdfsPath = HadoopUtils.getHdfsUdfDir(oldTenant.getTenantCode()); @@ -308,7 +308,7 @@ public class UsersService extends BaseService { User user = userMapper.queryTenantCodeByUserId(id); - if (PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ + if (PropertyUtils.getResUploadStartupState()){ String userPath = HadoopUtils.getHdfsDataBasePath() + "/" + user.getTenantCode() + "/home/" + id; HadoopUtils.getInstance().delete(userPath, true); diff --git a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java index ddd27170db..30d2e26ded 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java @@ -60,6 +60,23 @@ public final class Constants { */ public static final String FS_DEFAULTFS = "fs.defaultFS"; + + /** + * fs s3a endpoint + */ + public static final String FS_S3A_ENDPOINT = "fs.s3a.endpoint"; + + /** + * fs s3a access key + */ + public static final String FS_S3A_ACCESS_KEY = "fs.s3a.access.key"; + + /** + * fs s3a secret key + */ + public static final String FS_S3A_SECRET_KEY = "fs.s3a.secret.key"; + + /** * yarn.resourcemanager.ha.rm.idsfs.defaultFS */ @@ -123,9 +140,9 @@ public final class Constants { public static final String DEVELOPMENT_STATE = "development.state"; /** - * hdfs.startup.state + * res.upload.startup.type */ - public static final String HDFS_STARTUP_STATE = "hdfs.startup.state"; + public static final String RES_UPLOAD_STARTUP_TYPE = "res.upload.startup.type"; /** * zookeeper quorum diff --git a/escheduler-common/src/main/java/cn/escheduler/common/enums/ResUploadType.java b/escheduler-common/src/main/java/cn/escheduler/common/enums/ResUploadType.java new file mode 100644 index 0000000000..65d8be8f92 --- /dev/null +++ b/escheduler-common/src/main/java/cn/escheduler/common/enums/ResUploadType.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package cn.escheduler.common.enums; + +/** + * data base types + */ +public enum ResUploadType { + /** + * 0 hdfs + * 1 s3 + * 2 none + */ + HDFS,S3,NONE +} diff --git a/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java b/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java index 85716bec81..6f3e5e2198 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java @@ -18,6 +18,7 @@ package cn.escheduler.common.utils; import cn.escheduler.common.Constants; import cn.escheduler.common.enums.ExecutionStatus; +import cn.escheduler.common.enums.ResUploadType; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONException; import com.alibaba.fastjson.JSONObject; @@ -40,6 +41,7 @@ import java.util.stream.Stream; import static cn.escheduler.common.Constants.*; import static cn.escheduler.common.utils.PropertyUtils.*; +import static cn.escheduler.common.utils.PropertyUtils.getString; /** * hadoop utils @@ -94,48 +96,61 @@ public class HadoopUtils implements Closeable { try { configuration = new Configuration(); - if (getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE)){ - System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, - getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)); - configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION,"kerberos"); - UserGroupInformation.setConfiguration(configuration); - UserGroupInformation.loginUserFromKeytab(getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), - getString(Constants.LOGIN_USER_KEY_TAB_PATH)); - } + String resUploadStartupType = PropertyUtils.getString(Constants.RES_UPLOAD_STARTUP_TYPE); + ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); + + if (resUploadType == ResUploadType.HDFS){ + if (getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE)){ + System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, + getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)); + configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION,"kerberos"); + UserGroupInformation.setConfiguration(configuration); + UserGroupInformation.loginUserFromKeytab(getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), + getString(Constants.LOGIN_USER_KEY_TAB_PATH)); + } - String defaultFS = configuration.get(FS_DEFAULTFS); - //first get key from core-site.xml hdfs-site.xml ,if null ,then try to get from properties file - // the default is the local file system - if(defaultFS.startsWith("file")){ - String defaultFSProp = getString(FS_DEFAULTFS); - if(StringUtils.isNotBlank(defaultFSProp)){ - Map fsRelatedProps = getPrefixedProperties("fs."); - configuration.set(FS_DEFAULTFS,defaultFSProp); - fsRelatedProps.entrySet().stream().forEach(entry -> configuration.set(entry.getKey(), entry.getValue())); + String defaultFS = configuration.get(FS_DEFAULTFS); + //first get key from core-site.xml hdfs-site.xml ,if null ,then try to get from properties file + // the default is the local file system + if(defaultFS.startsWith("file")){ + String defaultFSProp = getString(FS_DEFAULTFS); + if(StringUtils.isNotBlank(defaultFSProp)){ + Map fsRelatedProps = getPrefixedProperties("fs."); + configuration.set(FS_DEFAULTFS,defaultFSProp); + fsRelatedProps.entrySet().stream().forEach(entry -> configuration.set(entry.getKey(), entry.getValue())); + }else{ + logger.error("property:{} can not to be empty, please set!"); + throw new RuntimeException("property:{} can not to be empty, please set!"); + } }else{ - logger.error("property:{} can not to be empty, please set!"); - throw new RuntimeException("property:{} can not to be empty, please set!"); + logger.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", FS_DEFAULTFS, defaultFS); } - }else{ - logger.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", FS_DEFAULTFS, defaultFS); - } - if (fs == null) { - if(StringUtils.isNotEmpty(hdfsUser)){ - //UserGroupInformation ugi = UserGroupInformation.createProxyUser(hdfsUser,UserGroupInformation.getLoginUser()); - UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser); - ugi.doAs(new PrivilegedExceptionAction() { - @Override - public Boolean run() throws Exception { - fs = FileSystem.get(configuration); - return true; - } - }); - }else{ - logger.warn("hdfs.root.user is not set value!"); - fs = FileSystem.get(configuration); + if (fs == null) { + if(StringUtils.isNotEmpty(hdfsUser)){ + //UserGroupInformation ugi = UserGroupInformation.createProxyUser(hdfsUser,UserGroupInformation.getLoginUser()); + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser); + ugi.doAs(new PrivilegedExceptionAction() { + @Override + public Boolean run() throws Exception { + fs = FileSystem.get(configuration); + return true; + } + }); + }else{ + logger.warn("hdfs.root.user is not set value!"); + fs = FileSystem.get(configuration); + } } + }else if (resUploadType == ResUploadType.S3){ + configuration.set(FS_DEFAULTFS,getString(FS_DEFAULTFS)); + configuration.set(FS_S3A_ENDPOINT,getString(FS_S3A_ENDPOINT)); + configuration.set(FS_S3A_ACCESS_KEY,getString(FS_S3A_ACCESS_KEY)); + configuration.set(FS_S3A_SECRET_KEY,getString(FS_S3A_SECRET_KEY)); + fs = FileSystem.get(configuration); } + + String rmHaIds = getString(YARN_RESOURCEMANAGER_HA_RM_IDS); String appAddress = getString(Constants.YARN_APPLICATION_STATUS_ADDRESS); if (!StringUtils.isEmpty(rmHaIds)) { diff --git a/escheduler-common/src/main/java/cn/escheduler/common/utils/PropertyUtils.java b/escheduler-common/src/main/java/cn/escheduler/common/utils/PropertyUtils.java index f5dab12618..475cbfb72e 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/utils/PropertyUtils.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/utils/PropertyUtils.java @@ -16,6 +16,8 @@ */ package cn.escheduler.common.utils; +import cn.escheduler.common.Constants; +import cn.escheduler.common.enums.ResUploadType; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,11 +67,15 @@ public class PropertyUtils { } } -/* - public static PropertyUtils getInstance(){ - return propertyUtils; + /** + * judge whether resource upload startup + * @return + */ + public static Boolean getResUploadStartupState(){ + String resUploadStartupType = PropertyUtils.getString(Constants.RES_UPLOAD_STARTUP_TYPE); + ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); + return resUploadType == ResUploadType.HDFS || resUploadType == ResUploadType.S3; } -*/ /** * get property value diff --git a/escheduler-common/src/main/resources/common/common.properties b/escheduler-common/src/main/resources/common/common.properties index 357ba2cbc6..874d3d0f0b 100644 --- a/escheduler-common/src/main/resources/common/common.properties +++ b/escheduler-common/src/main/resources/common/common.properties @@ -16,8 +16,8 @@ hdfs.root.user=hdfs # data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/escheduler" is recommended data.store2hdfs.basepath=/escheduler -# whether hdfs starts -hdfs.startup.state=false +# resource upload startup type : HDFS,S3,NONE +res.upload.startup.type=NONE # whether kerberos starts hadoop.security.authentication.startup.state=false diff --git a/escheduler-common/src/main/resources/common/hadoop/hadoop.properties b/escheduler-common/src/main/resources/common/hadoop/hadoop.properties index f210ae7533..81452a83a2 100644 --- a/escheduler-common/src/main/resources/common/hadoop/hadoop.properties +++ b/escheduler-common/src/main/resources/common/hadoop/hadoop.properties @@ -1,6 +1,16 @@ -# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml to the conf directory +# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml +# to the conf directory，support s3，for example : s3a://escheduler fs.defaultFS=hdfs://mycluster:8020 +# s3 need，s3 endpoint +fs.s3a.endpoint=http://192.168.199.91:9010 + +# s3 need，s3 access key +fs.s3a.access.key=A3DXS30FO22544RE + +# s3 need，s3 secret key +fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK + #resourcemanager ha note this need ips , this empty if single yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx diff --git a/install.sh b/install.sh index 2245409986..dc26cc59ef 100644 --- a/install.sh +++ b/install.sh @@ -110,14 +110,17 @@ xlsFilePath="/tmp/xls" #是否启动监控自启动脚本 monitorServerState="false" -# hadoop 配置 -# 是否启动hdfs,如果启动则为true,需要配置以下hadoop相关参数; -# 不启动设置为false,如果为false,以下配置不需要修改 -# 特别注意：如果启动hdfs，需要自行创建hdfs根路径，也就是install.sh中的 hdfsPath -hdfsStartupSate="false" +# 资源中心上传选择存储方式：HDFS,S3,NONE +resUploadStartupType="NONE" -# namenode地址，支持HA,需要将core-site.xml和hdfs-site.xml放到conf目录下 -namenodeFs="hdfs://mycluster:8020" +# 如果resUploadStartupType为HDFS，defaultFS写namenode地址，支持HA,需要将core-site.xml和hdfs-site.xml放到conf目录下 +# 如果是S3，则写S3地址，比如说：s3a://escheduler，注意，一定要创建根目录/escheduler +defaultFS="hdfs://mycluster:8020" + +# 如果配置了S3，则需要有以下配置 +s3Endpoint="http://192.168.199.91:9010" +s3AccessKey="A3DXS30FO22544RE" +s3SecretKey="OloCLq3n+8+sdPHUhJ21XrSxTC+JK" # resourcemanager HA配置，如果是单resourcemanager,这里为空即可 yarnHaIps="192.168.xx.xx,192.168.xx.xx" @@ -273,7 +276,10 @@ sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.us sed -i ${txt} "s#org.quartz.dataSource.myDs.password.*#org.quartz.dataSource.myDs.password=${mysqlPassword}#g" conf/quartz.properties -sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${namenodeFs}#g" conf/common/hadoop/hadoop.properties +sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g" conf/common/hadoop/hadoop.properties +sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g" conf/common/hadoop/hadoop.properties +sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g" conf/common/hadoop/hadoop.properties +sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g" conf/common/hadoop/hadoop.properties sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common/hadoop/hadoop.properties sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common/hadoop/hadoop.properties @@ -283,7 +289,7 @@ sed -i ${txt} "s#data.download.basedir.path.*#data.download.basedir.path=${downl sed -i ${txt} "s#process.exec.basepath.*#process.exec.basepath=${execPath}#g" conf/common/common.properties sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common/common.properties sed -i ${txt} "s#data.store2hdfs.basepath.*#data.store2hdfs.basepath=${hdfsPath}#g" conf/common/common.properties -sed -i ${txt} "s#hdfs.startup.state.*#hdfs.startup.state=${hdfsStartupSate}#g" conf/common/common.properties +sed -i ${txt} "s#res.upload.startup.type.*#res.upload.startup.type=${resUploadStartupType}#g" conf/common/common.properties sed -i ${txt} "s#escheduler.env.path.*#escheduler.env.path=${shellEnvPath}#g" conf/common/common.properties sed -i ${txt} "s#resource.view.suffixs.*#resource.view.suffixs=${resSuffixs}#g" conf/common/common.properties sed -i ${txt} "s#development.state.*#development.state=${devState}#g" conf/common/common.properties From f425e0818cf1954697850ac44becb36a9bf1c4a2 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Wed, 26 Jun 2019 19:12:07 +0800 Subject: [PATCH 031/149] add preview escheduler --- .../definition/pages/list/_source/timing.vue | 30 +++++++++++++++++++ .../src/js/conf/home/store/dag/actions.js | 13 ++++++++ .../src/js/module/i18n/locale/zh_CN.js | 1 + 3 files changed, 44 insertions(+) diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue index ba67536ea2..9f7000cf28 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue @@ -21,9 +21,11 @@
+ 执行时间
{{$t('Timing')}}
+
+
+
{{$t('previewTime')}}
+ +
{{$t('Failure Strategy')}} @@ -225,6 +231,27 @@ } }, + _preview () { + if (this._verification()) { + let api = 'dag/previewSchedule' + let searchParams = { + schedule: JSON.stringify({ + startTime: this.scheduleTime[0], + endTime: this.scheduleTime[1], + crontab: this.crontab + }) + } + let msg = '' + + this.store.dispatch(api, searchParams).then(res => { + this.$message.success(msg) + this.$emit('onUpdate') + }).catch(e => { + this.$message.error(e.msg || '') + }) + } + }, + _getNotifyGroupList () { return new Promise((resolve, reject) => { let notifyGroupListS = _.cloneDeep(this.store.state.dag.notifyGroupListS) || [] @@ -248,6 +275,9 @@ }, close () { this.$emit('close') + }, + preview () { + this._preview() } }, watch: { diff --git a/escheduler-ui/src/js/conf/home/store/dag/actions.js b/escheduler-ui/src/js/conf/home/store/dag/actions.js index e41e4be760..32e5328699 100644 --- a/escheduler-ui/src/js/conf/home/store/dag/actions.js +++ b/escheduler-ui/src/js/conf/home/store/dag/actions.js @@ -377,6 +377,19 @@ export default { }) }) }, + /** + * Preview timing + */ + previewSchedule ({ state }, payload) { + return new Promise((resolve, reject) => { + io.post(`projects/${state.projectName}/schedule/preview`, payload, res => { + //resolve(res) + alert(res.data) + }).catch(e => { + reject(e) + }) + }) + }, /** * Timing list paging */ diff --git a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js index d77c55715a..a41c84c24c 100644 --- a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -460,4 +460,5 @@ export default { 'Process Define Count': '流程定义个数', 'Process Instance Running Count': '运行流程实例个数', 'Please select a queue': '请选择队列', + 'previewTime': '未来5次执行时间', } From 455d82a7400d3f5845f56ce37fb4f16066ed4fef Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Thu, 27 Jun 2019 11:52:39 +0800 Subject: [PATCH 032/149] sqlTask update --- .../main/java/cn/escheduler/server/worker/task/sql/SqlTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java index 4eb567d8c8..dd10d05ddf 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java @@ -374,7 +374,7 @@ public class SqlTask extends AbstractTask { String showTypeName = sqlParameters.getShowType().replace(Constants.COMMA,"").trim(); if(EnumUtils.isValidEnum(ShowType.class,showTypeName)){ Map mailResult = MailUtils.sendMails(receviersList, receviersCcList, title, content, ShowType.valueOf(showTypeName)); - if(!(Boolean) mailResult.get(cn.escheduler.api.utils.Constants.STATUS)){ + if(!(Boolean) mailResult.get(Constants.STATUS)){ throw new RuntimeException("send mail failed!"); } }else{ From 729963fc63caddf5451ad0c2b10ca402375e7779 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Thu, 27 Jun 2019 13:53:51 +0800 Subject: [PATCH 033/149] tenant service update --- .../main/java/cn/escheduler/api/service/TenantService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java index ffaa4de7ad..35bb35a2b4 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java @@ -235,8 +235,8 @@ public class TenantService extends BaseService{ return result; } - // if hdfs startup - if (PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ + // if resource upload startup + if (PropertyUtils.getResUploadStartupState()){ String tenantPath = HadoopUtils.getHdfsDataBasePath() + "/" + tenant.getTenantCode(); String resourcePath = HadoopUtils.getHdfsDir(tenant.getTenantCode()); From ce8645bc69b8ef57fc90e60d0e1154444fa1d0f0 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Thu, 27 Jun 2019 16:09:22 +0800 Subject: [PATCH 034/149] add kerberos interface --- .../api/controller/DataSourceController.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/controller/DataSourceController.java b/escheduler-api/src/main/java/cn/escheduler/api/controller/DataSourceController.java index 5a0b911581..4d1aaa6c28 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/controller/DataSourceController.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/controller/DataSourceController.java @@ -34,9 +34,11 @@ import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.*; import springfox.documentation.annotations.ApiIgnore; +import java.util.HashMap; import java.util.Map; import static cn.escheduler.api.enums.Status.*; +import static cn.escheduler.common.utils.PropertyUtils.getBoolean; /** @@ -429,4 +431,24 @@ public class DataSourceController extends BaseController { return error(AUTHORIZED_DATA_SOURCE.getCode(), AUTHORIZED_DATA_SOURCE.getMsg()); } } + + /** + * get user info + * + * @param loginUser + * @return + */ + @ApiOperation(value = "getKerberosStartupState", notes= "GET_USER_INFO_NOTES") + @GetMapping(value="/kerberos-startup-state") + @ResponseStatus(HttpStatus.OK) + public Result getKerberosStartupState(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser){ + logger.info("login user {},get user info : {}", loginUser.getUserName()); + try{ + Boolean kerberosStartupState = getBoolean(cn.escheduler.common.Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE); + return success(Status.SUCCESS.getMsg(),kerberosStartupState); + }catch (Exception e){ + logger.error(KERBEROS_STARTUP_STATE.getMsg(),e); + return error(Status.KERBEROS_STARTUP_STATE.getCode(), Status.KERBEROS_STARTUP_STATE.getMsg()); + } + } } From a0e1495f27f3ad05ece5167957546932fc0e740f Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Thu, 27 Jun 2019 16:12:04 +0800 Subject: [PATCH 035/149] add kerberos interface --- escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java | 1 + 1 file changed, 1 insertion(+) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java b/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java index 7d4a3c6381..bd1d5e91b3 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java @@ -233,6 +233,7 @@ public enum Status { QUEUE_COUNT_ERROR(90001,"queue count error"), + KERBEROS_STARTUP_STATE(100001,"get kerberos startup state error"), ; private int code; From 60b26651422fe41310f837b798e56380c0455842 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Thu, 27 Jun 2019 18:14:20 +0800 Subject: [PATCH 036/149] update preview time --- escheduler-ui/src/js/conf/home/store/dag/actions.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/escheduler-ui/src/js/conf/home/store/dag/actions.js b/escheduler-ui/src/js/conf/home/store/dag/actions.js index 32e5328699..bd361c4dbc 100644 --- a/escheduler-ui/src/js/conf/home/store/dag/actions.js +++ b/escheduler-ui/src/js/conf/home/store/dag/actions.js @@ -383,8 +383,8 @@ export default { previewSchedule ({ state }, payload) { return new Promise((resolve, reject) => { io.post(`projects/${state.projectName}/schedule/preview`, payload, res => { - //resolve(res) - alert(res.data) + resolve(res.data) + //alert(res.data) }).catch(e => { reject(e) }) From be804b5a6304beb4513ae00961e70931bd913bb9 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Thu, 27 Jun 2019 18:14:44 +0800 Subject: [PATCH 037/149] update verifyResourceName --- .../definition/pages/list/_source/timing.vue | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue index 9f7000cf28..277060f11a 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue @@ -47,7 +47,7 @@
{{$t('previewTime')}}
- +
@@ -168,7 +168,8 @@ receiversCc: [], i18n: i18n.globalScope.LOCALE, processInstancePriority: 'MEDIUM', - workerGroupId: -1 + workerGroupId: -1, + previewTime: 12345 } }, props: { @@ -244,10 +245,12 @@ let msg = '' this.store.dispatch(api, searchParams).then(res => { - this.$message.success(msg) - this.$emit('onUpdate') - }).catch(e => { - this.$message.error(e.msg || '') + this.previewTime = res + if (this.previewTime.length) { + resolve() + } else { + reject(new Error(0)) + } }) } }, From 3f217f5d00f21eac83320d2241448ca49a3701ed Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Fri, 28 Jun 2019 14:39:02 +0800 Subject: [PATCH 038/149] add datasource kerberos auth and FAQ modify --- docs/zh_CN/EasyScheduler-FAQ.md | 267 +++++++++++++++--- .../api/controller/DataSourceController.java | 30 +- .../api/service/DataSourceService.java | 39 ++- .../cn/escheduler/api/utils/CheckUtils.java | 14 + .../cn/escheduler/api/utils/Constants.java | 1 + .../common/job/db/BaseDataSource.java | 20 +- .../common/job/db/HiveDataSource.java | 13 +- .../common/job/db/SparkDataSource.java | 3 +- .../pages/list/_source/createDataSource.vue | 41 ++- .../js/conf/home/store/datasource/actions.js | 9 + .../src/js/module/i18n/locale/en_US.js | 3 +- .../src/js/module/i18n/locale/zh_CN.js | 1 + install.sh | 2 +- 13 files changed, 372 insertions(+), 71 deletions(-) diff --git a/docs/zh_CN/EasyScheduler-FAQ.md b/docs/zh_CN/EasyScheduler-FAQ.md index e9f9d5d7ab..e6af99cae4 100644 --- a/docs/zh_CN/EasyScheduler-FAQ.md +++ b/docs/zh_CN/EasyScheduler-FAQ.md @@ -1,96 +1,283 @@ -Q:单机运行服务老挂，应该是内存不够，测试机器4核8G。生产环境需要分布式，如果单机的话建议的配置是？ +## Q：EasyScheduler服务介绍及建议运行内存 -A: Easy Scheduler有5个服务组成，这些服务本身需要的内存和cpu不多， +A： EasyScheduler由5个服务组成，MasterServer、WorkerServer、ApiServer、AlertServer、LoggerServer和UI。 -| 服务 | 内存 | cpu核数 | -| ------------ | ---- | ------- | -| MasterServer | 2G | 2核 | -| WorkerServer | 2G | 2核 | -| ApiServer | 512M | 1核 | -| AlertServer | 512M | 1核 | -| LoggerServer | 512M | 1核 | +| 服务 | 说明 | +| ------------------------- | ------------------------------------------------------------ | +| MasterServer | 主要负责 **DAG** 的切分和任务状态的监控 | +| WorkerServer/LoggerServer | 主要负责任务的提交、执行和任务状态的更新。LoggerServer用于Rest Api通过 **RPC** 查看日志 | +| ApiServer | 提供Rest Api服务，供UI进行调用 | +| AlertServer | 提供告警服务 | +| UI | 前端页面展示 | -注意：由于如果任务较多，WorkServer所在机器建议物理内存在16G以上 +注意：**由于服务比较多，建议单机部署最好是4核16G以上** +--- + +## Q： 管理员为什么不能创建项目 + +A：管理员目前属于"**纯管理**", 没有租户，即没有linux上对应的用户，所以没有执行权限, **故没有所属的项目、资源及数据源**，所以没有创建权限。**但是有所有的查看权限**。如果需要创建项目等业务操作，**请使用管理员创建租户和普通用户，然后使用普通用户登录进行操作**。我们将会在1.1.0版本中将管理员的创建和执行权限放开，管理员将会有所有的权限 + +--- + +## Q：系统支持哪些邮箱？ + +A：支持绝大多数邮箱，qq、163、126、139、outlook、aliyun等皆支持。支持**TLS和SSL**协议，可以在alert.properties中选择性配置 + +--- + +## Q：常用的系统变量时间参数有哪些，如何使用？ + +A：请参考 https://analysys.github.io/easyscheduler_docs_cn/%E7%B3%BB%E7%BB%9F%E4%BD%BF%E7%94%A8%E6%89%8B%E5%86%8C.html#%E7%B3%BB%E7%BB%9F%E5%8F%82%E6%95%B0 + +--- +## Q：pip install kazoo 这个安装报错。是必须安装的吗？ + +A： 这个是python连接zookeeper需要使用到的，必须要安装 --- -Q: 管理员为什么不能创建项目？ +## Q: 怎么指定机器运行任务 -A: 管理员目前属于"纯管理", 没有租户，即没有linux上对应的用户，所以没有执行权限, 但是有所有的查看权限。如果需要创建项目等业务操作，请使用管理员创建租户和普通用户，然后使用普通用户登录进行操作 +A：使用 **管理员** 创建Worker分组，在 **流程定义启动** 的时候可**指定Worker分组**或者在**任务节点上指定Worker分组**。如果不指定，则使用Default，**Default默认是使用的集群里所有的Worker中随机选取一台来进行任务提交、执行** --- -Q: 系统支持哪些邮箱？ +## Q：任务的优先级 + +A：我们同时 **支持流程和任务的优先级**。优先级我们有 **HIGHEST、HIGH、MEDIUM、LOW和LOWEST** 五种级别。**可以设置不同流程实例之间的优先级，也可以设置同一个流程实例中不同任务实例的优先级**。详细内容请参考任务优先级设计 https://analysys.github.io/easyscheduler_docs_cn/%E7%B3%BB%E7%BB%9F%E6%9E%B6%E6%9E%84%E8%AE%BE%E8%AE%A1.html#%E7%B3%BB%E7%BB%9F%E6%9E%B6%E6%9E%84%E8%AE%BE%E8%AE%A1 + +---- + +## Q：escheduler-grpc报错 + +A：在根目录下执行：mvn -U clean package assembly:assembly -Dmaven.test.skip=true ， 然后刷新下整个项目 + +---- + +## Q：EasyScheduler支持windows上运行么 + +A： 理论上只有**Worker是需要在Linux上运行的**，其它的服务都是可以在windows上正常运行的。但是还是建议最好能在linux上部署使用 + +----- + +## Q：UI 在 linux 编译node-sass提示：Error：EACCESS:permission denied，mkdir xxxx -A: 支持绝大多数邮箱，qq、163、126、139、outlook、aliyun等皆可支持 +A：单独安装 **npm install node-sass --unsafe-perm**，之后再 **npm install** --- -Q:常用的系统变量时间参数有哪些,如何使用？ +## Q：UI 不能正常登陆访问 -A: 请参考使用手册中的系统参数 +A： 1，如果是node启动的查看escheduler-ui下的.env API_BASE配置是否是Api Server服务地址 + +​ 2，如果是nginx启动的并且是通过 **install-escheduler-ui.sh** 安装的，查看 **/etc/nginx/conf.d/escheduler.conf** 中的proxy_pass配置是否是Api Server服务地址 + +​ 3，如果以上配置都是正确的，那么请查看Api Server服务是否是正常的，curl http://192.168.xx.xx:12345/escheduler/users/get-user-info，查看Api Server日志，如果提示 cn.escheduler.api.interceptor.LoginHandlerInterceptor:[76] - session info is null，则证明Api Server服务是正常的 + +​ 4，如果以上都没有问题，需要查看一下 **application.properties** 中的 **server.context-path 和 server.port 配置**是否正确 --- -Q：pip install kazoo 这个安装报错。是必须安装的吗？ +## Q： 流程定义手动启动或调度启动之后，没有流程实例生成 + +A： 1，首先通过**jps 查看MasterServer服务是否存在**，或者从服务监控直接查看zk中是否存在master服务 -A: 这个是python连接zookeeper需要使用到的 +​ 2，如果存在master服务，查看 **命令状态统计** 或者 **t_escheduler_error_command** 中是否增加的新记录，如果增加了，**请查看 message 字段定位启动异常原因** --- -Q: 如果alert、api、logger服务任意一个宕机，任何还会正常执行吧 +## Q ： 任务状态一直处于提交成功状态 -A: 不影响，影响正在运行中的任务的服务有Master和Worker服务 +A： 1，首先通过**jps 查看WorkerServer服务是否存在**，或者从服务监控直接查看zk中是否存在worker服务 + +​ 2，如果 **WorkerServer** 服务正常，需要 **查看MasterServer是否把task任务放到zk队列中** ，**需要查看MasterServer日志及zk队列中是否有任务阻塞** + +​ 3，如果以上都没有问题，需要定位是否指定了Worker分组，但是 **Worker分组的机器不是在线状态** --- -Q: 这个怎么指定机器运行任务的啊 」 +## Q： 是否提供Docker镜像及Dockerfile + +A： 提供Docker镜像及Dockerfile。 -A: 通过worker分组： 这个流程只能在指定的机器组里执行。默认是Default，可以在任一worker上执行。 +Docker镜像地址：https://hub.docker.com/r/escheduler/escheduler_images + +Dockerfile地址：https://github.com/qiaozhanwei/escheduler_dockerfile/tree/master/docker_escheduler --- -Q: 跨用户的任务依赖怎么实现呢， 比如A用户写了一个任务，B用户需要依赖这个任务 +## Q ： install.sh 中需要注意问题 + +A： 1，如果替换变量中包含特殊字符，**请用 \ 转移符进行转移** + +​ 2，installPath="/data1_1T/escheduler"，**这个目录不能和当前要一键安装的install.sh目录是一样的** -就比如说 我们数仓组 写了一个 中间宽表的任务， 其他业务部门想要使用这个中间表的时候，他们应该是另外一个用户，怎么依赖这个中间表呢 +​ 3，deployUser="escheduler"，**部署用户必须具有sudo权限**，因为worker是通过sudo -u 租户 sh xxx.command进行执行的 -A: 有两种情况，一个是要运行这个宽表任务，可以使用子工作流把宽表任务放到自己的工作流里面。另一个是检查这个宽表任务有没有完成，可以使用依赖节点来检查这个宽表任务在指定的时间周期有没有完成。 +​ 4，monitorServerState="false"，服务监控脚本是否启动，默认是不启动服务监控脚本的。**如果启动服务监控脚本，则每5分钟定时来监控master和worker的服务是否down机，如果down机则会自动重启** + +​ 5，hdfsStartupSate="false"，是否开启HDFS资源上传功能。默认是不开启的，**如果不开启则资源中心是不能使用的**。如果开启，需要conf/common/hadoop/hadoop.properties中配置fs.defaultFS和yarn的相关配置，如果使用namenode HA，需要将core-site.xml和hdfs-site.xml复制到conf根目录下 + +​ 注意：**1.0.x版本是不会自动创建hdfs根目录的，需要自行创建，并且需要部署用户有hdfs的操作权限** --- -Q: 启动WorkerServer服务时不能正常启动，报以下信息是什么原因？ +## Q ： 流程定义和流程实例下线异常 + +A ： 对于 **1.0.4 以前的版本中**，修改escheduler-api cn.escheduler.api.quartz包下的代码即可 ``` -[INFO] 2019-05-06 16:39:31.492 cn.escheduler.server.zk.ZKWorkerClient:[155] - register failure , worker already started on : 127.0.0.1, please wait for a moment and try again +public boolean deleteJob(String jobName, String jobGroupName) { + lock.writeLock().lock(); + try { + JobKey jobKey = new JobKey(jobName,jobGroupName); + if(scheduler.checkExists(jobKey)){ + logger.info("try to delete job, job name: {}, job group name: {},", jobName, jobGroupName); + return scheduler.deleteJob(jobKey); + }else { + return true; + } + + } catch (SchedulerException e) { + logger.error(String.format("delete job : %s failed",jobName), e); + } finally { + lock.writeLock().unlock(); + } + return false; + } ``` -A：Worker/Master Server在启动时，会向Zookeeper注册自己的启动信息，是Zookeeper的临时节点，如果两次启动时间间隔较短的情况，上次启动的Worker/Master Server在Zookeeper的会话还未过期，会出现上述信息，处理办法是等待session过期，一般是1分钟左右 +--- ----- +## Q ： HDFS启动之前创建的租户，能正常使用资源中心吗 -Q: 编译时escheduler-grpc模块一直报错：Information:java: Errors occurred while compiling module 'escheduler-rpc'， 找不到LogParameter、RetStrInfo、RetByteInfo等class类 +A： 不能。因为在未启动HDFS创建的租户，不会在HDFS中注册租户目录。所以上次资源会报错 -A: 这是因为rpc源码包是google Grpc实现的，需要使用maven进行编译，在根目录下执行：mvn -U clean package assembly:assembly -Dmaven.test.skip=true ， 然后刷新下整个项目 +## Q : 多Master和多Worker状态下，服务掉了，怎么容错 ----- +A： **注意：Master监控Master及Worker服务。** -Q：EasyScheduler支持windows上运行么？ +​ 1，如果Master服务掉了，其它的Master会接管挂掉的Master的流程，继续监控Worker task状态 -A: 建议在Ubuntu、Centos上运行，暂不支持windows上运行，不过windows上可以进行编译。开发调试的话建议Ubuntu或者mac上进行。 +​ 2，如果Worker服务掉，Master会监控到Worker服务掉了，如果存在Yarn任务，Kill Yarn任务之后走重试 + +具体请看容错设计：https://analysys.github.io/easyscheduler_docs_cn/%E7%B3%BB%E7%BB%9F%E6%9E%B6%E6%9E%84%E8%AE%BE%E8%AE%A1.html#%E7%B3%BB%E7%BB%9F%E6%9E%B6%E6%9E%84%E8%AE%BE%E8%AE%A1 + +--- + +## Q ： 对于Master和Worker一台机器伪分布式下的容错 + +A ： 1.0.3 版本只实现了Master启动流程容错，不走Worker容错。也就是说如果Worker挂掉的时候，没有Master存在。这流程将会出现问题。我们会在 **1.1.0** 版本中增加Master和Worker启动自容错，修复这个问题。如果想手动修改这个问题，需要针对 **跨重启正在运行流程** **并且已经掉的正在运行的Worker任务，需要修改为失败**，**同时跨重启正在运行流程设置为失败状态**。然后从失败节点进行流程恢复即可 + +--- + +## Q ： 定时容易设置成每秒执行 + +A ： 设置定时的时候需要注意，如果第一位（* * * * * ? *）设置成 \* ，则表示每秒执行。**我们将会在1.1.0版本中加入显示最近调度的时间列表** ，使用http://cron.qqe2.com/ 可以在线看近5次运行时间 ------ -Q：任务为什么不执行？ -A: 不执行的原因： +## Q： 定时有有效时间范围吗 -查看command表里有没有内容？ +A：有的，**如果定时的起止时间是同一个时间，那么此定时将是无效的定时**。**如果起止时间的结束时间比当前的时间小，很有可能定时会被自动删除** -查看Master server的运行日志： -查看Worker Server的运行日志 +## Q ： 任务依赖有几种实现 +A： 1，**DAG** 之间的任务依赖关系，是从 **入度为零** 进行DAG切分的 + +​ 2，有 **任务依赖节点** ，可以实现跨流程的任务或者流程依赖，具体请参考 依赖(DEPENDENT)节点：https://analysys.github.io/easyscheduler_docs_cn/%E7%B3%BB%E7%BB%9F%E4%BD%BF%E7%94%A8%E6%89%8B%E5%86%8C.html#%E4%BB%BB%E5%8A%A1%E8%8A%82%E7%82%B9%E7%B1%BB%E5%9E%8B%E5%92%8C%E5%8F%82%E6%95%B0%E8%AE%BE%E7%BD%AE + +​ 注意：**不支持跨项目的流程或任务依赖** + +## Q： 流程定义有几种启动方式 + +A： 1，在 **流程定义列表**，点击 **启动** 按钮 + +​ 2，**流程定义列表添加定时器**，调度启动流程定义 + +​ 3，流程定义 **查看或编辑** DAG 页面，任意 **任务节点右击** 启动流程定义 + +​ 4，可以对流程定义 DAG 编辑，设置某些任务的运行标志位 **禁止运行**，则在启动流程定义的时候，将该节点的连线将从DAG中去掉 + +## Q ： Python任务设置Python版本 + +A： 1，对于1**.0.3之后的版本**只需要修改 conf/env/.escheduler_env.sh中的PYTHON_HOME + +``` +export PYTHON_HOME=/bin/python +``` + +注意：这了 **PYTHON_HOME** ，是python命令的绝对路径，而不是单纯的 PYTHON_HOME，还需要注意的是 export PATH 的时候，需要直接 + +``` +export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH +``` + +​ 2，对 1.0.3 之前的版本，Python任务只能支持系统的Python版本，不支持指定Python版本 + +## Q： Worker Task 通过sudo -u 租户 sh xxx.command会产生子进程，在kill的时候，是否会杀掉 + +A： 我们会在1.0.4中增加kill任务同时，kill掉任务产生的各种所有子进程 + + + +## Q ： EasyScheduler中的队列怎么用，用户队列和租户队列是什么意思 + +A ： EasyScheduler 中的队列可以在用户或者租户上指定队列，**用户指定的队列优先级是高于租户队列的优先级的。**，例如：对MR任务指定队列，是通过 mapreduce.job.queuename 来指定队列的。 + +注意：MR在用以上方法指定队列的时候，传递参数请使用如下方式： + +``` + Configuration conf = new Configuration(); + GenericOptionsParser optionParser = new GenericOptionsParser(conf, args); + String[] remainingArgs = optionParser.getRemainingArgs(); +``` + + + +如果是Spark任务 --queue 方式指定队列 + + + +## Q : Master 后者 Worker报如下告警 + +![1560847965302](C:\Users\Administrator\Desktop\FAQ\1560847965302.png) + + + +A ： 修改conf下的 master.properties **master.reserved.memory** 的值为更小的值，比如说0.1 或者 + +worker.properties **worker.reserved.memory** 的值为更小的值，比如说0.1 + + + +## Q : hive版本是1.1.0+cdh5.15.0，SQL hive任务连接报错 + +![EF4DA613-5129-4c7a-A0AB-61E5A866A919](C:\Users\Administrator\Desktop\FAQ\EF4DA613-5129-4c7a-A0AB-61E5A866A919.png) + + + +A ： 将 hive pom + +``` + + org.apache.hive + hive-jdbc + 2.1.0 + +``` + +修改为 + +``` + + org.apache.hive + hive-jdbc + 1.1.0 + +``` diff --git a/escheduler-api/src/main/java/cn/escheduler/api/controller/DataSourceController.java b/escheduler-api/src/main/java/cn/escheduler/api/controller/DataSourceController.java index 4d1aaa6c28..51e0c93bd6 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/controller/DataSourceController.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/controller/DataSourceController.java @@ -18,10 +18,13 @@ package cn.escheduler.api.controller; import cn.escheduler.api.enums.Status; import cn.escheduler.api.service.DataSourceService; +import cn.escheduler.api.utils.CheckUtils; import cn.escheduler.api.utils.Constants; import cn.escheduler.api.utils.Result; import cn.escheduler.common.enums.DbType; +import cn.escheduler.common.enums.ResUploadType; import cn.escheduler.common.utils.ParameterUtils; +import cn.escheduler.common.utils.PropertyUtils; import cn.escheduler.dao.model.User; import io.swagger.annotations.Api; import io.swagger.annotations.ApiImplicitParam; @@ -56,12 +59,16 @@ public class DataSourceController extends BaseController { /** * create data source - * 创建数据源 - * * @param loginUser * @param name * @param note * @param type + * @param host + * @param port + * @param database + * @param principal + * @param userName + * @param password * @param other * @return */ @@ -86,13 +93,14 @@ public class DataSourceController extends BaseController { @RequestParam(value = "host") String host, @RequestParam(value = "port") String port, @RequestParam(value = "database") String database, + @RequestParam(value = "principal") String principal, @RequestParam(value = "userName") String userName, @RequestParam(value = "password") String password, @RequestParam(value = "other") String other) { - logger.info("login user {} create datasource ame: {}, note: {}, type: {}, other: {}", - loginUser.getUserName(), name, note, type, other); + logger.info("login user {} create datasource name: {}, note: {}, type: {}, host: {},port: {},database : {},principal: {},userName : {} other: {}", + loginUser.getUserName(), name, note, type, host,port,database,principal,userName,other); try { - String parameter = dataSourceService.buildParameter(name, note, type, host, port, database, userName, password, other); + String parameter = dataSourceService.buildParameter(name, note, type, host, port, database,principal,userName, password, other); Map result = dataSourceService.createDataSource(loginUser, name, note, type, parameter); return returnDataList(result); @@ -136,13 +144,14 @@ public class DataSourceController extends BaseController { @RequestParam(value = "host") String host, @RequestParam(value = "port") String port, @RequestParam(value = "database") String database, + @RequestParam(value = "principal") String principal, @RequestParam(value = "userName") String userName, @RequestParam(value = "password") String password, @RequestParam(value = "other") String other) { logger.info("login user {} updateProcessInstance datasource name: {}, note: {}, type: {}, other: {}", loginUser.getUserName(), name, note, type, other); try { - String parameter = dataSourceService.buildParameter(name, note, type, host, port, database, userName, password, other); + String parameter = dataSourceService.buildParameter(name, note, type, host, port, database,principal, userName, password, other); Map dataSource = dataSourceService.updateDataSource(id, loginUser, name, note, type, parameter); return returnDataList(dataSource); } catch (Exception e) { @@ -271,13 +280,14 @@ public class DataSourceController extends BaseController { @RequestParam(value = "host") String host, @RequestParam(value = "port") String port, @RequestParam(value = "database") String database, + @RequestParam(value = "principal") String principal, @RequestParam(value = "userName") String userName, @RequestParam(value = "password") String password, @RequestParam(value = "other") String other) { logger.info("login user {}, connect datasource: {} failure, note: {}, type: {}, other: {}", loginUser.getUserName(), name, note, type, other); try { - String parameter = dataSourceService.buildParameter(name, note, type, host, port, database, userName, password, other); + String parameter = dataSourceService.buildParameter(name, note, type, host, port, database,principal,userName, password, other); Boolean isConnection = dataSourceService.checkConnection(type, parameter); Result result = new Result(); @@ -442,10 +452,10 @@ public class DataSourceController extends BaseController { @GetMapping(value="/kerberos-startup-state") @ResponseStatus(HttpStatus.OK) public Result getKerberosStartupState(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser){ - logger.info("login user {},get user info : {}", loginUser.getUserName()); + logger.info("login user {},get kerberos startup state : {}", loginUser.getUserName()); try{ - Boolean kerberosStartupState = getBoolean(cn.escheduler.common.Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE); - return success(Status.SUCCESS.getMsg(),kerberosStartupState); + // if upload resource is HDFS and kerberos startup is true , else false + return success(Status.SUCCESS.getMsg(), CheckUtils.getKerberosStartupState()); }catch (Exception e){ logger.error(KERBEROS_STARTUP_STATE.getMsg(),e); return error(Status.KERBEROS_STARTUP_STATE.getCode(), Status.KERBEROS_STARTUP_STATE.getMsg()); diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java index 9081a436cf..02164f971b 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java @@ -17,12 +17,15 @@ package cn.escheduler.api.service; import cn.escheduler.api.enums.Status; +import cn.escheduler.api.utils.CheckUtils; import cn.escheduler.api.utils.Constants; import cn.escheduler.api.utils.PageInfo; import cn.escheduler.api.utils.Result; import cn.escheduler.common.enums.DbType; +import cn.escheduler.common.enums.ResUploadType; import cn.escheduler.common.enums.UserType; import cn.escheduler.common.job.db.*; +import cn.escheduler.common.utils.PropertyUtils; import cn.escheduler.dao.mapper.DataSourceMapper; import cn.escheduler.dao.mapper.DatasourceUserMapper; import cn.escheduler.dao.mapper.ProjectMapper; @@ -31,6 +34,8 @@ import cn.escheduler.dao.model.Resource; import cn.escheduler.dao.model.User; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.TypeReference; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -42,6 +47,9 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.util.*; +import static cn.escheduler.common.utils.PropertyUtils.getBoolean; +import static cn.escheduler.common.utils.PropertyUtils.getString; + /** * datasource service */ @@ -55,6 +63,7 @@ public class DataSourceService extends BaseService{ public static final String TYPE = "type"; public static final String HOST = "host"; public static final String PORT = "port"; + public static final String PRINCIPAL = "principal"; public static final String DATABASE = "database"; public static final String USER_NAME = "userName"; public static final String PASSWORD = "password"; @@ -240,6 +249,7 @@ public class DataSourceService extends BaseService{ map.put(TYPE, dataSourceType); map.put(HOST, host); map.put(PORT, port); + map.put(PRINCIPAL, datasourceForm.getPrincipal()); map.put(DATABASE, database); map.put(USER_NAME, datasourceForm.getUser()); map.put(PASSWORD, datasourceForm.getPassword()); @@ -363,11 +373,21 @@ public class DataSourceService extends BaseService{ Class.forName(Constants.COM_MYSQL_JDBC_DRIVER); break; case HIVE: - datasource = JSONObject.parseObject(parameter, HiveDataSource.class); - Class.forName(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER); - break; case SPARK: - datasource = JSONObject.parseObject(parameter, SparkDataSource.class); + if (CheckUtils.getKerberosStartupState()) { + System.setProperty(cn.escheduler.common.Constants.JAVA_SECURITY_KRB5_CONF, + getString(cn.escheduler.common.Constants.JAVA_SECURITY_KRB5_CONF_PATH)); + Configuration configuration = new Configuration(); + configuration.set(cn.escheduler.common.Constants.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + UserGroupInformation.setConfiguration(configuration); + UserGroupInformation.loginUserFromKeytab(getString(cn.escheduler.common.Constants.LOGIN_USER_KEY_TAB_USERNAME), + getString(cn.escheduler.common.Constants.LOGIN_USER_KEY_TAB_PATH)); + } + if (dbType == DbType.HIVE){ + datasource = JSONObject.parseObject(parameter, HiveDataSource.class); + }else if (dbType == DbType.SPARK){ + datasource = JSONObject.parseObject(parameter, SparkDataSource.class); + } Class.forName(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER); break; case CLICKHOUSE: @@ -443,10 +463,18 @@ public class DataSourceService extends BaseService{ * @param other * @return */ - public String buildParameter(String name, String desc, DbType type, String host, String port, String database, String userName, String password, String other) { + public String buildParameter(String name, String desc, DbType type, String host, + String port, String database,String principal,String userName, + String password, String other) { String address = buildAddress(type, host, port); + String jdbcUrl = address + "/" + database; + if (CheckUtils.getKerberosStartupState() && + (type == DbType.HIVE || type == DbType.SPARK)){ + jdbcUrl += ";principal=" + principal; + } + String separator = ""; if (Constants.MYSQL.equals(type.name()) || Constants.POSTGRESQL.equals(type.name()) @@ -465,6 +493,7 @@ public class DataSourceService extends BaseService{ parameterMap.put(Constants.JDBC_URL, jdbcUrl); parameterMap.put(Constants.USER, userName); parameterMap.put(Constants.PASSWORD, password); + parameterMap.put(Constants.PRINCIPAL,principal); if (other != null && !"".equals(other)) { Map map = JSONObject.parseObject(other, new TypeReference>() { }); diff --git a/escheduler-api/src/main/java/cn/escheduler/api/utils/CheckUtils.java b/escheduler-api/src/main/java/cn/escheduler/api/utils/CheckUtils.java index 6a8c627d7b..f6330b79de 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/utils/CheckUtils.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/utils/CheckUtils.java @@ -18,8 +18,10 @@ package cn.escheduler.api.utils; import cn.escheduler.api.enums.Status; +import cn.escheduler.common.enums.ResUploadType; import cn.escheduler.common.task.AbstractParameters; import cn.escheduler.common.utils.JSONUtils; +import cn.escheduler.common.utils.PropertyUtils; import cn.escheduler.common.utils.TaskParametersUtils; import org.apache.commons.lang.StringUtils; @@ -28,6 +30,7 @@ import java.util.HashMap; import java.util.Map; import java.util.regex.Pattern; +import static cn.escheduler.common.utils.PropertyUtils.getBoolean; /** @@ -158,4 +161,15 @@ public class CheckUtils { return pattern.matcher(str).matches(); } + /** + * if upload resource is HDFS and kerberos startup is true , else false + * @return + */ + public static boolean getKerberosStartupState(){ + String resUploadStartupType = PropertyUtils.getString(cn.escheduler.common.Constants.RES_UPLOAD_STARTUP_TYPE); + ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); + Boolean kerberosStartupState = getBoolean(cn.escheduler.common.Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE); + return resUploadType == ResUploadType.HDFS && kerberosStartupState; + } + } diff --git a/escheduler-api/src/main/java/cn/escheduler/api/utils/Constants.java b/escheduler-api/src/main/java/cn/escheduler/api/utils/Constants.java index 79cf3e5b3f..1dfe3ac470 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/utils/Constants.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/utils/Constants.java @@ -111,6 +111,7 @@ public class Constants { public static final String ADDRESS = "address"; public static final String DATABASE = "database"; public static final String JDBC_URL = "jdbcUrl"; + public static final String PRINCIPAL = "principal"; public static final String USER = "user"; public static final String PASSWORD = "password"; public static final String OTHER = "other"; diff --git a/escheduler-common/src/main/java/cn/escheduler/common/job/db/BaseDataSource.java b/escheduler-common/src/main/java/cn/escheduler/common/job/db/BaseDataSource.java index af0624091a..f215d3e8c9 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/job/db/BaseDataSource.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/job/db/BaseDataSource.java @@ -45,6 +45,18 @@ public abstract class BaseDataSource { */ private String other; + /** + * principal + */ + private String principal; + + public String getPrincipal() { + return principal; + } + + public void setPrincipal(String principal) { + this.principal = principal; + } /** * test whether the data source can be connected successfully * @throws Exception @@ -73,14 +85,14 @@ public abstract class BaseDataSource { this.password = password; } - public String getAddress() { - return address; - } - public void setAddress(String address) { this.address = address; } + public String getAddress() { + return address; + } + public String getDatabase() { return database; } diff --git a/escheduler-common/src/main/java/cn/escheduler/common/job/db/HiveDataSource.java b/escheduler-common/src/main/java/cn/escheduler/common/job/db/HiveDataSource.java index 28e37991d7..719c5eb300 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/job/db/HiveDataSource.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/job/db/HiveDataSource.java @@ -17,12 +17,12 @@ package cn.escheduler.common.job.db; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; +import java.sql.*; /** * data source of hive @@ -32,6 +32,8 @@ public class HiveDataSource extends BaseDataSource { private static final Logger logger = LoggerFactory.getLogger(HiveDataSource.class); + + /** * gets the JDBC url for the data source connection * @return @@ -43,7 +45,7 @@ public class HiveDataSource extends BaseDataSource { jdbcUrl += "/"; } - jdbcUrl += getDatabase(); + jdbcUrl += getDatabase() + ";principal=" + getPrincipal(); if (StringUtils.isNotEmpty(getOther())) { jdbcUrl += ";" + getOther(); @@ -67,11 +69,10 @@ public class HiveDataSource extends BaseDataSource { try { con.close(); } catch (SQLException e) { - logger.error("Postgre datasource try conn close conn error", e); + logger.error("hive datasource try conn close conn error", e); throw e; } } } - } } diff --git a/escheduler-common/src/main/java/cn/escheduler/common/job/db/SparkDataSource.java b/escheduler-common/src/main/java/cn/escheduler/common/job/db/SparkDataSource.java index d9a24eef22..13aa06eaae 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/job/db/SparkDataSource.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/job/db/SparkDataSource.java @@ -31,7 +31,6 @@ public class SparkDataSource extends BaseDataSource { private static final Logger logger = LoggerFactory.getLogger(SparkDataSource.class); - /** * gets the JDBC url for the data source connection * @return @@ -43,7 +42,7 @@ public class SparkDataSource extends BaseDataSource { jdbcUrl += "/"; } - jdbcUrl += getDatabase(); + jdbcUrl += getDatabase() + ";principal=" + getPrincipal(); if (StringUtils.isNotEmpty(getOther())) { jdbcUrl += ";" + getOther(); diff --git a/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue b/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue index 8b09570862..cc6d25e7de 100644 --- a/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue +++ b/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue @@ -63,6 +63,17 @@ + + + + +
+ {{$t('select tenant')}} + +
超时告警 @@ -73,6 +77,7 @@ import mLocalParams from '../formModel/tasks/_source/localParams' import disabledState from '@/module/mixin/disabledState' import Affirm from '../jumpAffirm' + import FormTenant from "./_source/selectTenant"; export default { name: 'udp', @@ -90,6 +95,8 @@ syncDefine: true, // Timeout alarm timeout: 0, + + tenantId: -1, // checked Timeout alarm checkedTimeout: true } @@ -116,6 +123,7 @@ this.store.commit('dag/setGlobalParams', _.cloneDeep(this.udpList)) this.store.commit('dag/setName', _.cloneDeep(this.name)) this.store.commit('dag/setTimeout', _.cloneDeep(this.timeout)) + this.store.commit('dag/setTenantId', _.cloneDeep(this.tenantId)) this.store.commit('dag/setDesc', _.cloneDeep(this.desc)) this.store.commit('dag/setSyncDefine', this.syncDefine) }, @@ -181,9 +189,10 @@ this.syncDefine = dag.syncDefine this.timeout = dag.timeout || 0 this.checkedTimeout = this.timeout !== 0 + this.tenantId = dag.tenantId || -1 }, mounted () {}, - components: { mLocalParams } + components: {FormTenant, mLocalParams } } diff --git a/escheduler-ui/src/js/conf/home/pages/dag/definitionDetails.vue b/escheduler-ui/src/js/conf/home/pages/dag/definitionDetails.vue index e377ebfa7f..c8849d594e 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/definitionDetails.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/definitionDetails.vue @@ -26,7 +26,7 @@ methods: { ...mapMutations('dag', ['resetParams', 'setIsDetails']), ...mapActions('dag', ['getProcessList', 'getResourcesList', 'getProcessDetails']), - ...mapActions('security', ['getWorkerGroupsAll']), + ...mapActions('security', ['getTenantList','getWorkerGroupsAll']), /** * init */ @@ -43,7 +43,8 @@ // get resource this.getResourcesList(), // get worker group list - this.getWorkerGroupsAll() + this.getWorkerGroupsAll(), + this.getTenantList() ]).then((data) => { let item = data[0] this.setIsDetails(item.releaseState === 'ONLINE') diff --git a/escheduler-ui/src/js/conf/home/pages/dag/index.vue b/escheduler-ui/src/js/conf/home/pages/dag/index.vue index b9aee791dd..7a3429f3b1 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/index.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/index.vue @@ -25,7 +25,7 @@ methods: { ...mapMutations('dag', ['resetParams']), ...mapActions('dag', ['getProcessList', 'getResourcesList']), - ...mapActions('security', ['getWorkerGroupsAll']), + ...mapActions('security', ['getTenantList','getWorkerGroupsAll']), /** * init */ @@ -40,7 +40,8 @@ // get resource this.getResourcesList(), // get worker group list - this.getWorkerGroupsAll() + this.getWorkerGroupsAll(), + this.getTenantList() ]).then((data) => { this.isLoading = false // Whether to pop up the box? @@ -65,4 +66,4 @@ }, components: { mDag, mSpin } } - \ No newline at end of file + diff --git a/escheduler-ui/src/js/conf/home/pages/dag/instanceDetails.vue b/escheduler-ui/src/js/conf/home/pages/dag/instanceDetails.vue index 3e6c49c1f6..cbe220ad36 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/instanceDetails.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/instanceDetails.vue @@ -26,7 +26,7 @@ methods: { ...mapMutations('dag', ['setIsDetails', 'resetParams']), ...mapActions('dag', ['getProcessList', 'getResourcesList', 'getInstancedetail']), - ...mapActions('security', ['getWorkerGroupsAll']), + ...mapActions('security', ['getTenantList','getWorkerGroupsAll']), /** * init */ @@ -43,7 +43,8 @@ // get resources this.getResourcesList(), // get worker group list - this.getWorkerGroupsAll() + this.getWorkerGroupsAll(), + this.getTenantList() ]).then((data) => { let item = data[0] let flag = false @@ -92,4 +93,4 @@ }, components: { mDag, mSpin, mVariable } } - \ No newline at end of file + diff --git a/escheduler-ui/src/js/conf/home/store/dag/actions.js b/escheduler-ui/src/js/conf/home/store/dag/actions.js index e41e4be760..e756cfdff4 100644 --- a/escheduler-ui/src/js/conf/home/store/dag/actions.js +++ b/escheduler-ui/src/js/conf/home/store/dag/actions.js @@ -115,6 +115,7 @@ export default { // timeout state.timeout = processDefinitionJson.timeout + state.tenantId = processDefinitionJson.tenantId resolve(res.data) }).catch(res => { reject(res) @@ -146,6 +147,8 @@ export default { // timeout state.timeout = processInstanceJson.timeout + state.tenantId = processInstanceJson.tenantId + resolve(res.data) }).catch(res => { reject(res) @@ -160,6 +163,7 @@ export default { let data = { globalParams: state.globalParams, tasks: state.tasks, + tenantId: state.tenantId, timeout: state.timeout } io.post(`projects/${state.projectName}/process/save`, { @@ -183,6 +187,7 @@ export default { let data = { globalParams: state.globalParams, tasks: state.tasks, + tenantId: state.tenantId, timeout: state.timeout } io.post(`projects/${state.projectName}/process/update`, { @@ -207,6 +212,7 @@ export default { let data = { globalParams: state.globalParams, tasks: state.tasks, + tenantId: state.tenantId, timeout: state.timeout } io.post(`projects/${state.projectName}/instance/update`, { diff --git a/escheduler-ui/src/js/conf/home/store/dag/mutations.js b/escheduler-ui/src/js/conf/home/store/dag/mutations.js index 15f87fab30..d3386bc76a 100644 --- a/escheduler-ui/src/js/conf/home/store/dag/mutations.js +++ b/escheduler-ui/src/js/conf/home/store/dag/mutations.js @@ -58,6 +58,12 @@ export default { setTimeout (state, payload) { state.timeout = payload }, + /** + * set tenantId + */ + setTenantId (state, payload) { + state.tenantId = payload + }, /** * set global params */ @@ -100,6 +106,7 @@ export default { state.name = payload && payload.name || '' state.desc = payload && payload.desc || '' state.timeout = payload && payload.timeout || 0 + state.tenantId = payload && payload.tenantId || -1 state.processListS = payload && payload.processListS || [] state.resourcesListS = payload && payload.resourcesListS || [] state.isDetails = payload && payload.isDetails || false diff --git a/escheduler-ui/src/js/conf/home/store/dag/state.js b/escheduler-ui/src/js/conf/home/store/dag/state.js index 9679893e60..37df7b8b86 100644 --- a/escheduler-ui/src/js/conf/home/store/dag/state.js +++ b/escheduler-ui/src/js/conf/home/store/dag/state.js @@ -31,6 +31,8 @@ export default { tasks: [], // Timeout alarm timeout: 0, + // tenant id + tenantId:-1, // Node location information locations: {}, // Node-to-node connection diff --git a/escheduler-ui/src/js/conf/home/store/security/actions.js b/escheduler-ui/src/js/conf/home/store/security/actions.js index 9fda7663ab..ff96adccf9 100644 --- a/escheduler-ui/src/js/conf/home/store/security/actions.js +++ b/escheduler-ui/src/js/conf/home/store/security/actions.js @@ -240,7 +240,13 @@ export default { getTenantList ({ state }, payload) { return new Promise((resolve, reject) => { io.get(`tenant/list`, payload, res => { - resolve(res.data) + let list=res.data + list.unshift({ + id: -1, + tenantName: 'Default' + }) + state.tenantAllList = list + resolve(list) }).catch(e => { reject(e) }) diff --git a/escheduler-ui/src/js/conf/home/store/security/state.js b/escheduler-ui/src/js/conf/home/store/security/state.js index be52d7838c..cbb67a1823 100644 --- a/escheduler-ui/src/js/conf/home/store/security/state.js +++ b/escheduler-ui/src/js/conf/home/store/security/state.js @@ -15,5 +15,6 @@ * limitations under the License. */ export default { - workerGroupsListAll: [] + workerGroupsListAll: [], + tenantAllList : [] } diff --git a/escheduler-ui/src/js/module/i18n/locale/en_US.js b/escheduler-ui/src/js/module/i18n/locale/en_US.js index cf81d9c95c..933b6be99d 100644 --- a/escheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/escheduler-ui/src/js/module/i18n/locale/en_US.js @@ -465,4 +465,5 @@ export default { 'task number of ready to kill': '待杀死任务数', 'Statistics manage': 'Statistics manage', 'statistics': 'statistics', + 'select tenant':'select tenant', } diff --git a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js index 78ef8323aa..a5ab26dfcb 100644 --- a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -466,4 +466,5 @@ export default { 'task number of ready to kill': '待杀死任务数', 'Statistics manage': '统计管理', 'statistics': '统计', + 'select tenant':'选择租户', } From ec7406ddababf2858ccfc76607dca34c43f9a18c Mon Sep 17 00:00:00 2001 From: lenboo Date: Mon, 1 Jul 2019 10:14:04 +0800 Subject: [PATCH 046/149] add tenant id in process definition --- .../src/main/java/cn/escheduler/dao/ProcessDao.java | 6 +----- .../cn/escheduler/dao/model/ProcessInstance.java | 9 ++++++++- .../server/worker/runner/FetchTaskThread.java | 12 ++++++++---- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java index 387c61635a..2aa59c7013 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java @@ -494,11 +494,7 @@ public class ProcessDao extends AbstractBaseDao { processInstance.setProcessInstancePriority(command.getProcessInstancePriority()); processInstance.setWorkerGroupId(command.getWorkerGroupId()); processInstance.setTimeout(processDefinition.getTimeout()); - Tenant tenant = getTenantForProcess(processDefinition.getTenantId(), - processDefinition.getUserId()); - if(tenant!= null){ - processInstance.setTenantCode(tenant.getTenantCode()); - } + processInstance.setTenantId(processDefinition.getTenantId()); return processInstance; } diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/model/ProcessInstance.java b/escheduler-dao/src/main/java/cn/escheduler/dao/model/ProcessInstance.java index 0519118404..24902c0121 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/model/ProcessInstance.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/model/ProcessInstance.java @@ -192,7 +192,7 @@ public class ProcessInstance { /** * tenant id */ - private String tenantId; + private int tenantId; public ProcessInstance(){ @@ -553,4 +553,11 @@ public class ProcessInstance { '}'; } + public void setTenantId(int tenantId) { + this.tenantId = tenantId; + } + + public int getTenantId() { + return this.tenantId ; + } } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java index 3ecafde57a..e741966f06 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java @@ -23,10 +23,7 @@ import cn.escheduler.common.thread.ThreadUtils; import cn.escheduler.common.utils.FileUtils; import cn.escheduler.common.utils.OSUtils; import cn.escheduler.dao.ProcessDao; -import cn.escheduler.dao.model.ProcessDefinition; -import cn.escheduler.dao.model.ProcessInstance; -import cn.escheduler.dao.model.TaskInstance; -import cn.escheduler.dao.model.WorkerGroup; +import cn.escheduler.dao.model.*; import cn.escheduler.server.zk.ZKWorkerClient; import com.cronutils.utils.StringUtils; import org.apache.commons.configuration.Configuration; @@ -194,9 +191,16 @@ public class FetchTaskThread implements Runnable{ // get process instance ProcessInstance processInstance = processDao.findProcessInstanceDetailById(taskInstance.getProcessInstanceId()); + // get process define ProcessDefinition processDefine = processDao.findProcessDefineById(taskInstance.getProcessDefinitionId()); + Tenant tenant = processDao.getTenantForProcess(processInstance.getTenantId(), + processDefine.getUserId()); + + if(tenant != null){ + processInstance.setTenantCode(tenant.getTenantCode()); + } taskInstance.setProcessInstance(processInstance); taskInstance.setProcessDefine(processDefine); From 8feb245e7259cc46bc9d93c33ca2a575d607898f Mon Sep 17 00:00:00 2001 From: huyuanming Date: Mon, 1 Jul 2019 18:08:08 +0800 Subject: [PATCH 047/149] the email input --- .../pages/definition/pages/list/_source/email.vue | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/email.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/email.vue index cc6cb57646..f5c38f9a12 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/email.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/email.vue @@ -32,9 +32,12 @@ v-model="email" :disabled="disabled" :placeholder="$t('Please enter email')" + @blur="_emailEnter" @keydown.tab="_emailTab" @keyup.delete="_emailDelete" @keyup.enter="_emailEnter" + @keyup.space="_emailEnter" + @keyup.186="_emailEnter" @keyup.up="_emailKeyup('up')" @keyup.down="_emailKeyup('down')"> @@ -78,6 +81,11 @@ * Manually add a mailbox */ _manualEmail () { + if (this.email === '') { + return + } + this.email = _.trim(this.email).replace(/(;$)|(；$)/g, "") + let email = this.email let is = (n) => { From 867048892a61a84bccbe68f2f10970bfac7aa404 Mon Sep 17 00:00:00 2001 From: lenboo Date: Mon, 1 Jul 2019 19:00:07 +0800 Subject: [PATCH 048/149] remove the auth for query list. --- .../main/java/cn/escheduler/api/service/TenantService.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java index 35bb35a2b4..694266d4ad 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java @@ -125,9 +125,9 @@ public class TenantService extends BaseService{ public Map queryTenantList(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { Map result = new HashMap<>(5); - if (checkAdmin(loginUser, result)) { - return result; - } +// if (checkAdmin(loginUser, result)) { +// return result; +// } Integer count = tenantMapper.countTenantPaging(searchVal); From d07916e1b6f1bd03a461be00a67684b8865a0077 Mon Sep 17 00:00:00 2001 From: huyuanming Date: Tue, 2 Jul 2019 09:54:48 +0800 Subject: [PATCH 049/149] enter key event of search input, the width of the mr project type input --- .../src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue | 2 +- .../pages/projects/pages/_source/instanceConditions/index.vue | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue index 14b50a5fe4..3787d13541 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue @@ -3,7 +3,7 @@
{{$t('Program Type')}}
- +
- +
- +
From 2deb22223406f0d8b7f6dc3f2a31d7e24a53457a Mon Sep 17 00:00:00 2001 From: lenboo Date: Tue, 2 Jul 2019 10:57:33 +0800 Subject: [PATCH 050/149] add tenant_id field in mysql --- .../1.1.0_schema/mysql/escheduler_ddl.sql | 41 +++++++++++++++++++ .../1.1.0_schema/mysql/escheduler_dml.sql | 1 + 2 files changed, 42 insertions(+) create mode 100644 sql/upgrade/1.1.0_schema/mysql/escheduler_ddl.sql create mode 100644 sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql diff --git a/sql/upgrade/1.1.0_schema/mysql/escheduler_ddl.sql b/sql/upgrade/1.1.0_schema/mysql/escheduler_ddl.sql new file mode 100644 index 0000000000..c43b3d86b0 --- /dev/null +++ b/sql/upgrade/1.1.0_schema/mysql/escheduler_ddl.sql @@ -0,0 +1,41 @@ +SET sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY','')); + +-- ac_escheduler_T_t_escheduler_process_definition_C_tenant_id +drop PROCEDURE if EXISTS ac_escheduler_T_t_escheduler_process_definition_C_tenant_id; +delimiter d// +CREATE PROCEDURE ac_escheduler_T_t_escheduler_process_definition_C_tenant_id() + BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_escheduler_process_definition' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME='tenant_id') + THEN + ALTER TABLE `t_escheduler_process_definition` ADD COLUMN `tenant_id` int(11) NOT NULL DEFAULT -1 COMMENT '租户id' AFTER `timeout`; + END IF; + END; + +d// + +delimiter ; +CALL ac_escheduler_T_t_escheduler_process_definition_C_tenant_id; +DROP PROCEDURE ac_escheduler_T_t_escheduler_process_definition_C_tenant_id; + +-- ac_escheduler_T_t_escheduler_process_instance_C_tenant_id +drop PROCEDURE if EXISTS ac_escheduler_T_t_escheduler_process_instance_C_tenant_id; +delimiter d// +CREATE PROCEDURE ac_escheduler_T_t_escheduler_process_instance_C_tenant_id() + BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_escheduler_process_instance' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME='tenant_id') + THEN + ALTER TABLE `t_escheduler_process_instance` ADD COLUMN `tenant_id` int(11) NOT NULL DEFAULT -1 COMMENT '租户id' AFTER `timeout`; + END IF; + END; + +d// + +delimiter ; +CALL ac_escheduler_T_t_escheduler_process_instance_C_tenant_id; +DROP PROCEDURE ac_escheduler_T_t_escheduler_process_instance_C_tenant_id; diff --git a/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql b/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql new file mode 100644 index 0000000000..97da34884c --- /dev/null +++ b/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql @@ -0,0 +1 @@ +INSERT INTO `t_escheduler_version` (`version`) VALUES ('1.0.2'); \ No newline at end of file From 50e36b5ecec8982cb0a081cef2d808390355b3f3 Mon Sep 17 00:00:00 2001 From: lenboo Date: Tue, 2 Jul 2019 10:59:35 +0800 Subject: [PATCH 051/149] remove the auth for query list. --- sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql b/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql index 97da34884c..4ddd9ddcdc 100644 --- a/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql +++ b/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql @@ -1 +1 @@ -INSERT INTO `t_escheduler_version` (`version`) VALUES ('1.0.2'); \ No newline at end of file +INSERT INTO `t_escheduler_version` (`version`) VALUES ('1.1.0'); \ No newline at end of file From 9c9c940f4858782700e79e587009ec8497bab1d7 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Tue, 2 Jul 2019 11:10:44 +0800 Subject: [PATCH 052/149] sql task add kerbos auth --- .../api/controller/DataSourceController.java | 3 ++- .../api/service/DataSourceService.java | 5 +++-- .../cn/escheduler/api/utils/CheckUtils.java | 12 ------------ .../cn/escheduler/common/utils/CommonUtils.java | 11 +++++++++++ .../server/worker/task/sql/SqlTask.java | 17 +++++++++++++++-- 5 files changed, 31 insertions(+), 17 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/controller/DataSourceController.java b/escheduler-api/src/main/java/cn/escheduler/api/controller/DataSourceController.java index 51e0c93bd6..7301f419cc 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/controller/DataSourceController.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/controller/DataSourceController.java @@ -23,6 +23,7 @@ import cn.escheduler.api.utils.Constants; import cn.escheduler.api.utils.Result; import cn.escheduler.common.enums.DbType; import cn.escheduler.common.enums.ResUploadType; +import cn.escheduler.common.utils.CommonUtils; import cn.escheduler.common.utils.ParameterUtils; import cn.escheduler.common.utils.PropertyUtils; import cn.escheduler.dao.model.User; @@ -455,7 +456,7 @@ public class DataSourceController extends BaseController { logger.info("login user {},get kerberos startup state : {}", loginUser.getUserName()); try{ // if upload resource is HDFS and kerberos startup is true , else false - return success(Status.SUCCESS.getMsg(), CheckUtils.getKerberosStartupState()); + return success(Status.SUCCESS.getMsg(), CommonUtils.getKerberosStartupState()); }catch (Exception e){ logger.error(KERBEROS_STARTUP_STATE.getMsg(),e); return error(Status.KERBEROS_STARTUP_STATE.getCode(), Status.KERBEROS_STARTUP_STATE.getMsg()); diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java index 02164f971b..b11e34913f 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java @@ -25,6 +25,7 @@ import cn.escheduler.common.enums.DbType; import cn.escheduler.common.enums.ResUploadType; import cn.escheduler.common.enums.UserType; import cn.escheduler.common.job.db.*; +import cn.escheduler.common.utils.CommonUtils; import cn.escheduler.common.utils.PropertyUtils; import cn.escheduler.dao.mapper.DataSourceMapper; import cn.escheduler.dao.mapper.DatasourceUserMapper; @@ -374,7 +375,7 @@ public class DataSourceService extends BaseService{ break; case HIVE: case SPARK: - if (CheckUtils.getKerberosStartupState()) { + if (CommonUtils.getKerberosStartupState()) { System.setProperty(cn.escheduler.common.Constants.JAVA_SECURITY_KRB5_CONF, getString(cn.escheduler.common.Constants.JAVA_SECURITY_KRB5_CONF_PATH)); Configuration configuration = new Configuration(); @@ -470,7 +471,7 @@ public class DataSourceService extends BaseService{ String address = buildAddress(type, host, port); String jdbcUrl = address + "/" + database; - if (CheckUtils.getKerberosStartupState() && + if (CommonUtils.getKerberosStartupState() && (type == DbType.HIVE || type == DbType.SPARK)){ jdbcUrl += ";principal=" + principal; } diff --git a/escheduler-api/src/main/java/cn/escheduler/api/utils/CheckUtils.java b/escheduler-api/src/main/java/cn/escheduler/api/utils/CheckUtils.java index f6330b79de..00c50f8263 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/utils/CheckUtils.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/utils/CheckUtils.java @@ -160,16 +160,4 @@ public class CheckUtils { return pattern.matcher(str).matches(); } - - /** - * if upload resource is HDFS and kerberos startup is true , else false - * @return - */ - public static boolean getKerberosStartupState(){ - String resUploadStartupType = PropertyUtils.getString(cn.escheduler.common.Constants.RES_UPLOAD_STARTUP_TYPE); - ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); - Boolean kerberosStartupState = getBoolean(cn.escheduler.common.Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE); - return resUploadType == ResUploadType.HDFS && kerberosStartupState; - } - } diff --git a/escheduler-common/src/main/java/cn/escheduler/common/utils/CommonUtils.java b/escheduler-common/src/main/java/cn/escheduler/common/utils/CommonUtils.java index d0164791d2..43087fbd9c 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/utils/CommonUtils.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/utils/CommonUtils.java @@ -17,6 +17,7 @@ package cn.escheduler.common.utils; import cn.escheduler.common.Constants; +import cn.escheduler.common.enums.ResUploadType; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,4 +64,14 @@ public class CommonUtils { + /** + * if upload resource is HDFS and kerberos startup is true , else false + * @return + */ + public static boolean getKerberosStartupState(){ + String resUploadStartupType = PropertyUtils.getString(cn.escheduler.common.Constants.RES_UPLOAD_STARTUP_TYPE); + ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); + Boolean kerberosStartupState = getBoolean(cn.escheduler.common.Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE); + return resUploadType == ResUploadType.HDFS && kerberosStartupState; + } } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java index dd10d05ddf..26d682f132 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java @@ -29,6 +29,7 @@ import cn.escheduler.common.task.sql.SqlBinds; import cn.escheduler.common.task.sql.SqlParameters; import cn.escheduler.common.task.sql.SqlType; import cn.escheduler.common.utils.CollectionUtils; +import cn.escheduler.common.utils.CommonUtils; import cn.escheduler.common.utils.ParameterUtils; import cn.escheduler.dao.AlertDao; import cn.escheduler.dao.DaoFactory; @@ -43,6 +44,8 @@ import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.serializer.SerializerFeature; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.EnumUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import java.sql.*; @@ -51,6 +54,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; +import static cn.escheduler.common.utils.PropertyUtils.getString; + /** * sql task */ @@ -228,7 +233,15 @@ public class SqlTask extends AbstractTask { List createFuncs){ Connection connection = null; try { - + if (CommonUtils.getKerberosStartupState()) { + System.setProperty(cn.escheduler.common.Constants.JAVA_SECURITY_KRB5_CONF, + getString(cn.escheduler.common.Constants.JAVA_SECURITY_KRB5_CONF_PATH)); + Configuration configuration = new Configuration(); + configuration.set(cn.escheduler.common.Constants.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + UserGroupInformation.setConfiguration(configuration); + UserGroupInformation.loginUserFromKeytab(getString(cn.escheduler.common.Constants.LOGIN_USER_KEY_TAB_USERNAME), + getString(cn.escheduler.common.Constants.LOGIN_USER_KEY_TAB_PATH)); + } if (DbType.HIVE.name().equals(sqlParameters.getType())) { Properties paramProp = new Properties(); paramProp.setProperty("user", baseDataSource.getUser()); @@ -278,7 +291,7 @@ public class SqlTask extends AbstractTask { array.add(mapOfColValues); } - logger.info("execute sql : {}", JSONObject.toJSONString(array, SerializerFeature.WriteMapNullValue)); + logger.debug("execute sql : {}", JSONObject.toJSONString(array, SerializerFeature.WriteMapNullValue)); // send as an attachment if (StringUtils.isEmpty(sqlParameters.getShowType())) { From 671c296da6b3469b23685723738476e7170f8102 Mon Sep 17 00:00:00 2001 From: lenboo Date: Tue, 2 Jul 2019 11:48:17 +0800 Subject: [PATCH 053/149] add tenant id.. --- .../dag/_source/udp/_source/selectTenant.vue | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 escheduler-ui/src/js/conf/home/pages/dag/_source/udp/_source/selectTenant.vue diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/udp/_source/selectTenant.vue b/escheduler-ui/src/js/conf/home/pages/dag/_source/udp/_source/selectTenant.vue new file mode 100644 index 0000000000..307c2fdc27 --- /dev/null +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/udp/_source/selectTenant.vue @@ -0,0 +1,56 @@ + + From 68e822e4dde96e174526612ea4300ba3195e8c77 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 2 Jul 2019 12:07:44 +0800 Subject: [PATCH 054/149] modify previewTimes --- .../pages/definition/pages/list/_source/timing.vue | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue index 277060f11a..b8785c487b 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue @@ -46,9 +46,12 @@
-
{{$t('previewTime')}}
- +
未来五次执行时间
+
    +
  • {{time}}
    • +
+
{{$t('Failure Strategy')}} @@ -169,7 +172,7 @@ i18n: i18n.globalScope.LOCALE, processInstancePriority: 'MEDIUM', workerGroupId: -1, - previewTime: 12345 + previewTimes: [] } }, props: { @@ -245,8 +248,8 @@ let msg = '' this.store.dispatch(api, searchParams).then(res => { - this.previewTime = res - if (this.previewTime.length) { + this.previewTimes = res + if (this.previewTimes.length) { resolve() } else { reject(new Error(0)) From 5d1e8eb871681d71dcd9b51c25aafadffc486a44 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 2 Jul 2019 14:07:29 +0800 Subject: [PATCH 055/149] update install.sh from upstream/dev-1.1.0 --- install.sh | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/install.sh b/install.sh index 2245409986..a80c3198a9 100644 --- a/install.sh +++ b/install.sh @@ -110,14 +110,17 @@ xlsFilePath="/tmp/xls" #是否启动监控自启动脚本 monitorServerState="false" -# hadoop 配置 -# 是否启动hdfs,如果启动则为true,需要配置以下hadoop相关参数; -# 不启动设置为false,如果为false,以下配置不需要修改 -# 特别注意：如果启动hdfs，需要自行创建hdfs根路径，也就是install.sh中的 hdfsPath -hdfsStartupSate="false" +# 资源中心上传选择存储方式：HDFS,S3,NONE +resUploadStartupType="NONE" -# namenode地址，支持HA,需要将core-site.xml和hdfs-site.xml放到conf目录下 -namenodeFs="hdfs://mycluster:8020" +# 如果resUploadStartupType为HDFS，defaultFS写namenode地址，支持HA,需要将core-site.xml和hdfs-site.xml放到conf目录下 +# 如果是S3，则写S3地址，比如说：s3a://escheduler，注意，一定要创建根目录/escheduler +defaultFS="hdfs://mycluster:8020" + +# 如果配置了S3，则需要有以下配置 +s3Endpoint="http://192.168.199.91:9010" +s3AccessKey="A3DXS30FO22544RE" +s3SecretKey="OloCLq3n+8+sdPHUhJ21XrSxTC+JK" # resourcemanager HA配置，如果是单resourcemanager,这里为空即可 yarnHaIps="192.168.xx.xx,192.168.xx.xx" @@ -186,7 +189,7 @@ workersLock="/escheduler/lock/workers" mastersFailover="/escheduler/lock/failover/masters" # zk worker容错分布式锁 -workersFailover="/escheduler/lock/failover/masters" +workersFailover="/escheduler/lock/failover/workers" # zk master启动容错分布式锁 mastersStartupFailover="/escheduler/lock/failover/startup-masters" @@ -273,7 +276,10 @@ sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.us sed -i ${txt} "s#org.quartz.dataSource.myDs.password.*#org.quartz.dataSource.myDs.password=${mysqlPassword}#g" conf/quartz.properties -sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${namenodeFs}#g" conf/common/hadoop/hadoop.properties +sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g" conf/common/hadoop/hadoop.properties +sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g" conf/common/hadoop/hadoop.properties +sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g" conf/common/hadoop/hadoop.properties +sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g" conf/common/hadoop/hadoop.properties sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common/hadoop/hadoop.properties sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common/hadoop/hadoop.properties @@ -283,7 +289,7 @@ sed -i ${txt} "s#data.download.basedir.path.*#data.download.basedir.path=${downl sed -i ${txt} "s#process.exec.basepath.*#process.exec.basepath=${execPath}#g" conf/common/common.properties sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common/common.properties sed -i ${txt} "s#data.store2hdfs.basepath.*#data.store2hdfs.basepath=${hdfsPath}#g" conf/common/common.properties -sed -i ${txt} "s#hdfs.startup.state.*#hdfs.startup.state=${hdfsStartupSate}#g" conf/common/common.properties +sed -i ${txt} "s#res.upload.startup.type.*#res.upload.startup.type=${resUploadStartupType}#g" conf/common/common.properties sed -i ${txt} "s#escheduler.env.path.*#escheduler.env.path=${shellEnvPath}#g" conf/common/common.properties sed -i ${txt} "s#resource.view.suffixs.*#resource.view.suffixs=${resSuffixs}#g" conf/common/common.properties sed -i ${txt} "s#development.state.*#development.state=${devState}#g" conf/common/common.properties From 6d26e7ac0d480e4692a37cd180b4d327e2b8fa47 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 2 Jul 2019 14:10:03 +0800 Subject: [PATCH 056/149] update from upstream/dev-1.1.0 --- pom.xml | 2 +- sql/soft_version | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index e993636be4..e4186d9b81 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 cn.analysys escheduler - 1.0.3-SNAPSHOT + 1.0.4-SNAPSHOT pom escheduler http://maven.apache.org diff --git a/sql/soft_version b/sql/soft_version index e6d5cb833c..a6a3a43c3a 100644 --- a/sql/soft_version +++ b/sql/soft_version @@ -1 +1 @@ -1.0.2 \ No newline at end of file +1.0.4 \ No newline at end of file From 7d2b9e5925e7a2dd3893409c4f08cfba1428f2f7 Mon Sep 17 00:00:00 2001 From: lenboo Date: Tue, 2 Jul 2019 14:58:34 +0800 Subject: [PATCH 057/149] remove the auth for query list. --- .../cn/escheduler/api/service/TenantService.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java index 694266d4ad..555275273f 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java @@ -125,9 +125,9 @@ public class TenantService extends BaseService{ public Map queryTenantList(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { Map result = new HashMap<>(5); -// if (checkAdmin(loginUser, result)) { -// return result; -// } + if (checkAdmin(loginUser, result)) { + return result; + } Integer count = tenantMapper.countTenantPaging(searchVal); @@ -269,9 +269,9 @@ public class TenantService extends BaseService{ public Map queryTenantList(User loginUser) { Map result = new HashMap<>(5); - if (checkAdmin(loginUser, result)) { - return result; - } +// if (checkAdmin(loginUser, result)) { +// return result; +// } List resourceList = tenantMapper.queryAllTenant(); result.put(Constants.DATA_LIST, resourceList); From f55fd51ff2c3aae5b0845d5037023de3c4c6ebb9 Mon Sep 17 00:00:00 2001 From: lenboo Date: Tue, 2 Jul 2019 15:07:48 +0800 Subject: [PATCH 058/149] update tenant in ui --- .../js/conf/home/pages/dag/_source/udp/_source/selectTenant.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/udp/_source/selectTenant.vue b/escheduler-ui/src/js/conf/home/pages/dag/_source/udp/_source/selectTenant.vue index 307c2fdc27..9c39e8ec4a 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/udp/_source/selectTenant.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/udp/_source/selectTenant.vue @@ -45,7 +45,7 @@ if (stateTenantAllList.length) { this.itemList = stateTenantAllList } else { - this.store.dispatch('actions/getTenantList').then(res => { + this.store.dispatch('security/getTenantList').then(res => { this.$nextTick(() => { this.itemList = res }) From c90a2af9024ed6b45e2ccf87175011d6e799c154 Mon Sep 17 00:00:00 2001 From: huyuanming Date: Tue, 2 Jul 2019 15:29:53 +0800 Subject: [PATCH 059/149] the start time must not be the same as the end --- .../projects/pages/definition/pages/list/_source/timing.vue | 5 +++++ escheduler-ui/src/js/module/i18n/locale/en_US.js | 3 ++- escheduler-ui/src/js/module/i18n/locale/zh_CN.js | 3 ++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue index b8785c487b..46cfaa0c86 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue @@ -190,6 +190,11 @@ return false } + if (this.scheduleTime[0] === this.scheduleTime[1]) { + this.$message.warning(`${i18n.$t('The start time must not be the same as the end')}`) + return false + } + if (!this.crontab) { this.$message.warning(`${i18n.$t('Please enter crontab')}`) return false diff --git a/escheduler-ui/src/js/module/i18n/locale/en_US.js b/escheduler-ui/src/js/module/i18n/locale/en_US.js index 57526d5b8d..0a9e1126ec 100644 --- a/escheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/escheduler-ui/src/js/module/i18n/locale/en_US.js @@ -467,5 +467,6 @@ export default { 'statistics': 'statistics', 'select tenant':'select tenant', 'Process Instance Running Count': 'Process Instance Running Count', - 'Please enter Principal':'Please enter Principal' + 'Please enter Principal':'Please enter Principal', + 'The start time must not be the same as the end': 'The start time must not be the same as the end' } diff --git a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js index a81b9574f6..0acb1975a4 100644 --- a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -467,5 +467,6 @@ export default { 'Statistics manage': '统计管理', 'statistics': '统计', 'select tenant':'选择租户', - 'Please enter Principal':'请输入Principal' + 'Please enter Principal':'请输入Principal', + 'The start time must not be the same as the end': '开始时间和结束时间不能相同' } From 9429e24a122547eb38be18b147e70f45389cfcdb Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 2 Jul 2019 15:57:03 +0800 Subject: [PATCH 060/149] Added the ability for administrators to view all data sources --- .../api/service/DataSourceService.java | 9 ++++++++- .../dao/mapper/DataSourceMapper.java | 13 +++++++++++++ .../dao/mapper/DataSourceMapperProvider.java | 18 ++++++++++++++++-- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java index 02164f971b..80491f8f32 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java @@ -324,7 +324,14 @@ public class DataSourceService extends BaseService{ */ public Map queryDataSourceList(User loginUser, Integer type) { Map result = new HashMap<>(5); - List datasourceList = dataSourceMapper.queryDataSourceByType(loginUser.getId(), type); + + List datasourceList; + + if (isAdmin(loginUser)) { + datasourceList = dataSourceMapper.listAllDataSourceByType(); + }else{ + datasourceList = dataSourceMapper.queryDataSourceByType(loginUser.getId(), type); + } result.put(Constants.DATA_LIST, datasourceList); putMsg(result, Status.SUCCESS); diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/DataSourceMapper.java b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/DataSourceMapper.java index 149d15e662..acedde653d 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/DataSourceMapper.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/DataSourceMapper.java @@ -216,4 +216,17 @@ public interface DataSourceMapper { @SelectProvider(type = DataSourceMapperProvider.class, method = "queryDatasourceExceptUserId") List queryDatasourceExceptUserId(@Param("userId") int userId); + @Results(value = { + @Result(property = "id", column = "id", id = true, javaType = Integer.class, jdbcType = JdbcType.INTEGER), + @Result(property = "name", column = "name", javaType = String.class, jdbcType = JdbcType.VARCHAR), + @Result(property = "note", column = "note", javaType = String.class, jdbcType = JdbcType.VARCHAR), + @Result(property = "type", column = "type", typeHandler = EnumOrdinalTypeHandler.class, javaType = DbType.class, jdbcType = JdbcType.INTEGER), + @Result(property = "userId", column = "user_id", javaType = Integer.class, jdbcType = JdbcType.INTEGER), + @Result(property = "connectionParams", column = "connection_params", javaType = String.class, jdbcType = JdbcType.VARCHAR), + @Result(property = "createTime", column = "create_time", javaType = Timestamp.class, jdbcType = JdbcType.DATE), + @Result(property = "updateTime", column = "update_time", javaType = Timestamp.class, jdbcType = JdbcType.DATE) + }) + @SelectProvider(type = DataSourceMapperProvider.class, method = "listAllDataSourceByType") + List listAllDataSourceByType(); + } diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/DataSourceMapperProvider.java b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/DataSourceMapperProvider.java index 73228057c2..61461ff1c1 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/DataSourceMapperProvider.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/DataSourceMapperProvider.java @@ -175,8 +175,7 @@ public class DataSourceMapperProvider { } /** - * 查询总的数据源数目 - * + * Query the total number of data sources * @param parameter * @return */ @@ -228,4 +227,19 @@ public class DataSourceMapperProvider { WHERE("user_id <> #{userId}"); }}.toString(); } + + + /** + * list all data source by type + * + * @param parameter + * @return + */ + public String listAllDataSourceByType(Map parameter) { + return new SQL() {{ + SELECT("*"); + FROM(TABLE_NAME); + }}.toString(); + } + } From f21dd49ded83b002ba62c86a3159d081d106db7a Mon Sep 17 00:00:00 2001 From: lenboo Date: Tue, 2 Jul 2019 16:01:53 +0800 Subject: [PATCH 061/149] update tenant id --- .../java/cn/escheduler/dao/ProcessDao.java | 18 ++- .../mapper/ProcessInstanceMapperProvider.java | 6 +- .../server/worker/runner/FetchTaskThread.java | 6 - .../worker/runner/TaskScheduleThread.java | 144 ++++++++++-------- 4 files changed, 94 insertions(+), 80 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java index 2aa59c7013..17ca727340 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java @@ -642,6 +642,9 @@ public class ProcessDao extends AbstractBaseDao { // find pause tasks and init task's state cmdParam.remove(Constants.CMDPARAM_RECOVERY_START_NODE_STRING); List suspendedNodeList = this.findTaskIdByInstanceState(processInstance.getId(), ExecutionStatus.PAUSE); + List stopNodeList = findTaskIdByInstanceState(processInstance.getId(), + ExecutionStatus.KILL); + suspendedNodeList.addAll(stopNodeList); for(Integer taskId : suspendedNodeList){ // 把暂停状态初始化 initTaskInstance(this.findTaskInstanceById(taskId)); @@ -789,13 +792,16 @@ public class ProcessDao extends AbstractBaseDao { * @param taskInstance */ private void initTaskInstance(TaskInstance taskInstance){ - if(taskInstance.getState().typeIsFailure() && !taskInstance.isSubProcess()){ - taskInstance.setFlag(Flag.NO); - updateTaskInstance(taskInstance); - }else{ - taskInstance.setState(ExecutionStatus.SUBMITTED_SUCCESS); - updateTaskInstance(taskInstance); + + if(!taskInstance.isSubProcess()){ + if(taskInstance.getState().typeIsCancel() || taskInstance.getState().typeIsFailure()){ + taskInstance.setFlag(Flag.NO); + updateTaskInstance(taskInstance); + return; + } } + taskInstance.setState(ExecutionStatus.SUBMITTED_SUCCESS); + updateTaskInstance(taskInstance); } /** diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ProcessInstanceMapperProvider.java b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ProcessInstanceMapperProvider.java index 1dc3bbc99b..78165e3f9a 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ProcessInstanceMapperProvider.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ProcessInstanceMapperProvider.java @@ -222,11 +222,11 @@ public class ProcessInstanceMapperProvider { public String queryDetailById(Map parameter) { return new SQL() { { - SELECT("inst.*,q.queue_name as queue,UNIX_TIMESTAMP(inst.end_time)-UNIX_TIMESTAMP(inst.start_time) as duration"); + SELECT("inst.*,UNIX_TIMESTAMP(inst.end_time)-UNIX_TIMESTAMP(inst.start_time) as duration"); - FROM(TABLE_NAME + " inst, t_escheduler_user u,t_escheduler_queue q"); + FROM(TABLE_NAME + " inst"); - WHERE("inst.executor_id = u.id AND t.queue_id = q.id AND inst.id = #{processId}"); + WHERE("inst.id = #{processId}"); } }.toString(); } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java index e741966f06..1c328b6cba 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java @@ -195,12 +195,6 @@ public class FetchTaskThread implements Runnable{ // get process define ProcessDefinition processDefine = processDao.findProcessDefineById(taskInstance.getProcessDefinitionId()); - Tenant tenant = processDao.getTenantForProcess(processInstance.getTenantId(), - processDefine.getUserId()); - - if(tenant != null){ - processInstance.setTenantCode(tenant.getTenantCode()); - } taskInstance.setProcessInstance(processInstance); taskInstance.setProcessDefine(processDefine); diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java index c21a3a2f1b..591712ba7d 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java @@ -34,8 +34,10 @@ import cn.escheduler.common.task.sql.SqlParameters; import cn.escheduler.common.utils.*; import cn.escheduler.dao.ProcessDao; import cn.escheduler.dao.TaskRecordDao; +import cn.escheduler.dao.model.ProcessDefinition; import cn.escheduler.dao.model.ProcessInstance; import cn.escheduler.dao.model.TaskInstance; +import cn.escheduler.dao.model.Tenant; import cn.escheduler.server.utils.LoggerUtils; import cn.escheduler.server.utils.ParamUtils; import cn.escheduler.server.worker.log.TaskLogger; @@ -160,82 +162,94 @@ public class TaskScheduleThread implements Callable { // set task params taskProps.setTaskParams(taskNode.getParams()); // set tenant code , execute task linux user - taskProps.setTenantCode(taskInstance.getProcessInstance().getTenantCode()); ProcessInstance processInstance = processDao.findProcessInstanceByTaskId(taskInstance.getId()); - String queue = processDao.queryQueueByProcessInstanceId(processInstance.getId()); taskProps.setScheduleTime(processInstance.getScheduleTime()); taskProps.setNodeName(taskInstance.getName()); taskProps.setTaskInstId(taskInstance.getId()); taskProps.setEnvFile(CommonUtils.getSystemEnvPath()); - // set queue - if (StringUtils.isEmpty(queue)){ - taskProps.setQueue(taskInstance.getProcessInstance().getQueue()); - }else { - taskProps.setQueue(queue); - } - taskProps.setTaskStartTime(taskInstance.getStartTime()); - taskProps.setDefinedParams(allParamMap); - - // set task timeout - setTaskTimeout(taskProps, taskNode); - - taskProps.setDependence(taskInstance.getDependency()); - - taskProps.setTaskAppId(String.format("%s_%s_%s", - taskInstance.getProcessDefine().getId(), - taskInstance.getProcessInstance().getId(), - taskInstance.getId())); - - // custom logger - TaskLogger taskLogger = new TaskLogger(LoggerUtils.buildTaskId(TASK_PREFIX, - taskInstance.getProcessDefine().getId(), - taskInstance.getProcessInstance().getId(), - taskInstance.getId())); - - task = TaskManager.newTask(taskInstance.getTaskType(), taskProps, taskLogger); - - // job init - task.init(); - - // job handle - task.handle(); - - - logger.info("task : {} exit status code : {}", taskProps.getTaskAppId(),task.getExitStatusCode()); - - if (task.getExitStatusCode() == Constants.EXIT_CODE_SUCCESS){ - status = ExecutionStatus.SUCCESS; - // task recor flat : if true , start up qianfan - if (TaskRecordDao.getTaskRecordFlag() - && TaskType.typeIsNormalTask(taskInstance.getTaskType())){ - - AbstractParameters params = (AbstractParameters) JSONUtils.parseObject(taskProps.getTaskParams(), getCurTaskParamsClass()); - - // replace placeholder - Map paramsMap = ParamUtils.convert(taskProps.getUserDefParamsMap(), - taskProps.getDefinedParams(), - params.getLocalParametersMap(), - processInstance.getCmdTypeIfComplement(), - processInstance.getScheduleTime()); - if (paramsMap != null && !paramsMap.isEmpty() - && paramsMap.containsKey("v_proc_date")){ - String vProcDate = paramsMap.get("v_proc_date").getValue(); - if (!StringUtils.isEmpty(vProcDate)){ - TaskRecordStatus taskRecordState = TaskRecordDao.getTaskRecordState(taskInstance.getName(), vProcDate); - logger.info("task record status : {}",taskRecordState); - if (taskRecordState == TaskRecordStatus.FAILURE){ - status = ExecutionStatus.FAILURE; + + ProcessDefinition processDefine = processDao.findProcessDefineById(processInstance.getProcessDefinitionId()); + + Tenant tenant = processDao.getTenantForProcess(processInstance.getTenantId(), + processDefine.getUserId()); + + if(tenant == null){ + processInstance.setTenantCode(tenant.getTenantCode()); + logger.error("cannot find the tenant, process definition id:{}, tenant id:{}, user id:{}", + processDefine.getId(), processDefine.getTenantId(), processDefine.getUserId() + ); + status = ExecutionStatus.FAILURE; + }else{ + taskProps.setTenantCode(taskInstance.getProcessInstance().getTenantCode()); + String queue = processDao.queryQueueByProcessInstanceId(processInstance.getId()); + // set queue + if (StringUtils.isEmpty(queue)){ + taskProps.setQueue(taskInstance.getProcessInstance().getQueue()); + }else { + taskProps.setQueue(tenant.getQueueName()); + } + taskProps.setTaskStartTime(taskInstance.getStartTime()); + taskProps.setDefinedParams(allParamMap); + + // set task timeout + setTaskTimeout(taskProps, taskNode); + + taskProps.setDependence(taskInstance.getDependency()); + + taskProps.setTaskAppId(String.format("%s_%s_%s", + taskInstance.getProcessDefine().getId(), + taskInstance.getProcessInstance().getId(), + taskInstance.getId())); + + // custom logger + TaskLogger taskLogger = new TaskLogger(LoggerUtils.buildTaskId(TASK_PREFIX, + taskInstance.getProcessDefine().getId(), + taskInstance.getProcessInstance().getId(), + taskInstance.getId())); + + task = TaskManager.newTask(taskInstance.getTaskType(), taskProps, taskLogger); + + // job init + task.init(); + + // job handle + task.handle(); + logger.info("task : {} exit status code : {}", taskProps.getTaskAppId(),task.getExitStatusCode()); + + if (task.getExitStatusCode() == Constants.EXIT_CODE_SUCCESS){ + status = ExecutionStatus.SUCCESS; + // task recor flat : if true , start up qianfan + if (TaskRecordDao.getTaskRecordFlag() + && TaskType.typeIsNormalTask(taskInstance.getTaskType())){ + + AbstractParameters params = (AbstractParameters) JSONUtils.parseObject(taskProps.getTaskParams(), getCurTaskParamsClass()); + + // replace placeholder + Map paramsMap = ParamUtils.convert(taskProps.getUserDefParamsMap(), + taskProps.getDefinedParams(), + params.getLocalParametersMap(), + processInstance.getCmdTypeIfComplement(), + processInstance.getScheduleTime()); + if (paramsMap != null && !paramsMap.isEmpty() + && paramsMap.containsKey("v_proc_date")){ + String vProcDate = paramsMap.get("v_proc_date").getValue(); + if (!StringUtils.isEmpty(vProcDate)){ + TaskRecordStatus taskRecordState = TaskRecordDao.getTaskRecordState(taskInstance.getName(), vProcDate); + logger.info("task record status : {}",taskRecordState); + if (taskRecordState == TaskRecordStatus.FAILURE){ + status = ExecutionStatus.FAILURE; + } } } } - } - }else if (task.getExitStatusCode() == Constants.EXIT_CODE_KILL){ - status = ExecutionStatus.KILL; - }else { - status = ExecutionStatus.FAILURE; + }else if (task.getExitStatusCode() == Constants.EXIT_CODE_KILL){ + status = ExecutionStatus.KILL; + }else { + status = ExecutionStatus.FAILURE; + } } }catch (Exception e){ logger.error("task escheduler failure : " + e.getMessage(),e); From b8a9ef55de874ff8a2897cef6c0069fa6b2fd588 Mon Sep 17 00:00:00 2001 From: lenboo Date: Tue, 2 Jul 2019 16:13:02 +0800 Subject: [PATCH 062/149] update tenant in ui --- .../cn/escheduler/server/worker/runner/TaskScheduleThread.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java index 591712ba7d..89226bf8b5 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java @@ -182,7 +182,7 @@ public class TaskScheduleThread implements Callable { ); status = ExecutionStatus.FAILURE; }else{ - taskProps.setTenantCode(taskInstance.getProcessInstance().getTenantCode()); + taskProps.setTenantCode(tenant.getTenantCode()); String queue = processDao.queryQueueByProcessInstanceId(processInstance.getId()); // set queue if (StringUtils.isEmpty(queue)){ From 49d0690a7d8fe283eb9456f771c2ef26120e94be Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 2 Jul 2019 16:29:02 +0800 Subject: [PATCH 063/149] Added the ability for administrators to view all data sources by type --- .../main/java/cn/escheduler/api/service/DataSourceService.java | 2 +- .../main/java/cn/escheduler/dao/mapper/DataSourceMapper.java | 2 +- .../java/cn/escheduler/dao/mapper/DataSourceMapperProvider.java | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java index 80491f8f32..cb44c5751a 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java @@ -328,7 +328,7 @@ public class DataSourceService extends BaseService{ List datasourceList; if (isAdmin(loginUser)) { - datasourceList = dataSourceMapper.listAllDataSourceByType(); + datasourceList = dataSourceMapper.listAllDataSourceByType(type); }else{ datasourceList = dataSourceMapper.queryDataSourceByType(loginUser.getId(), type); } diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/DataSourceMapper.java b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/DataSourceMapper.java index acedde653d..66a7b61bbc 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/DataSourceMapper.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/DataSourceMapper.java @@ -227,6 +227,6 @@ public interface DataSourceMapper { @Result(property = "updateTime", column = "update_time", javaType = Timestamp.class, jdbcType = JdbcType.DATE) }) @SelectProvider(type = DataSourceMapperProvider.class, method = "listAllDataSourceByType") - List listAllDataSourceByType(); + List listAllDataSourceByType(@Param("type") Integer type); } diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/DataSourceMapperProvider.java b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/DataSourceMapperProvider.java index 61461ff1c1..7613e555c1 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/DataSourceMapperProvider.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/DataSourceMapperProvider.java @@ -239,6 +239,7 @@ public class DataSourceMapperProvider { return new SQL() {{ SELECT("*"); FROM(TABLE_NAME); + WHERE("type = #{type}"); }}.toString(); } From 46bf965f8191b4eb6557c4c4e1d1c88653ea33cf Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Tue, 2 Jul 2019 18:26:14 +0800 Subject: [PATCH 064/149] update --- .../cn/escheduler/server/worker/runner/FetchTaskThread.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java index 1c328b6cba..1c6232bc9a 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java @@ -211,9 +211,12 @@ public class FetchTaskThread implements Runnable{ // set task execute path taskInstance.setExecutePath(execLocalPath); + Tenant tenant = processDao.getTenantForProcess(processInstance.getTenantId(), + processDefine.getUserId()); + // check and create Linux users FileUtils.createWorkDirAndUserIfAbsent(execLocalPath, - processInstance.getTenantCode(), logger); + tenant.getTenantCode(), logger); logger.info("task : {} ready to submit to task scheduler thread",taskId); // submit task From 9a4cb31451b386c077dcdf0b394a7fa76c684ca7 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 2 Jul 2019 19:58:27 +0800 Subject: [PATCH 065/149] Determine when the user's tenant does not exist --- .../java/cn/escheduler/api/service/UsersService.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java index c0dab22b69..512369ea8a 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java @@ -307,11 +307,13 @@ public class UsersService extends BaseService { // delete user User user = userMapper.queryTenantCodeByUserId(id); - - if (PropertyUtils.getResUploadStartupState()){ - String userPath = HadoopUtils.getHdfsDataBasePath() + "/" + user.getTenantCode() + "/home/" + id; - - HadoopUtils.getInstance().delete(userPath, true); + if (user != null) { + if (PropertyUtils.getResUploadStartupState()) { + String userPath = HadoopUtils.getHdfsDataBasePath() + "/" + user.getTenantCode() + "/home/" + id; + if (HadoopUtils.getInstance().exists(userPath)) { + HadoopUtils.getInstance().delete(userPath, true); + } + } } userMapper.delete(id); From c9c91a45e35d2a932ff9d46c29215abfc7da1ef3 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Tue, 2 Jul 2019 20:02:34 +0800 Subject: [PATCH 066/149] remove double quotes --- escheduler-common/src/main/resources/common/common.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/escheduler-common/src/main/resources/common/common.properties b/escheduler-common/src/main/resources/common/common.properties index 874d3d0f0b..27d525f8f5 100644 --- a/escheduler-common/src/main/resources/common/common.properties +++ b/escheduler-common/src/main/resources/common/common.properties @@ -26,10 +26,10 @@ hadoop.security.authentication.startup.state=false java.security.krb5.conf.path=/opt/krb5.conf # loginUserFromKeytab user -login.user.keytab.username="hdfs-mycluster@ESZ.COM" +login.user.keytab.username=hdfs-mycluster@ESZ.COM # loginUserFromKeytab path -login.user.keytab.path="/opt/hdfs.headless.keytab" +login.user.keytab.path=/opt/hdfs.headless.keytab # system env path. self configuration, please make sure the directory and file exists and have read write execute permissions escheduler.env.path=/opt/.escheduler_env.sh From b495bf22d02a3c137f37d448aa4333f1d90fda84 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Wed, 3 Jul 2019 10:38:56 +0800 Subject: [PATCH 067/149] remove logback.xml --- escheduler-api/src/main/resources/logback.xml | 42 ------------------- 1 file changed, 42 deletions(-) delete mode 100644 escheduler-api/src/main/resources/logback.xml diff --git a/escheduler-api/src/main/resources/logback.xml b/escheduler-api/src/main/resources/logback.xml deleted file mode 100644 index 2e27d70ef3..0000000000 --- a/escheduler-api/src/main/resources/logback.xml +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - - - - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - - - - - - INFO - - ${log.base}/escheduler-api-server.log - - ${log.base}/escheduler-api-server.%d{yyyy-MM-dd_HH}.%i.log - 168 - 64MB - - - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - - - - - - - \ No newline at end of file From 2a1283437c2e01e4b33a924d0afb5c76c5eb844f Mon Sep 17 00:00:00 2001 From: lenboo Date: Wed, 3 Jul 2019 15:45:40 +0800 Subject: [PATCH 068/149] add recovery from stop --- .../api/service/ExecutorService.java | 12 ++++++++- .../api/service/ProcessInstanceService.java | 14 ++++++++--- .../escheduler/dao/model/ProcessInstance.java | 14 +++++++++++ .../instance/pages/list/_source/list.vue | 25 +++++++++++++------ .../src/js/module/i18n/locale/en_US.js | 1 + .../src/js/module/i18n/locale/zh_CN.js | 1 + 6 files changed, 54 insertions(+), 13 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ExecutorService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ExecutorService.java index 740fbc961c..9602ac6cef 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ExecutorService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ExecutorService.java @@ -191,6 +191,16 @@ public class ExecutorService extends BaseService{ return checkResult; } + // checkTenantExists(); + Tenant tenant = processDao.getTenantForProcess(processDefinition.getTenantId(), + processDefinition.getUserId()); + if(tenant == null){ + logger.error("there is not any vaild tenant for the process definition: id:{},name:{}, ", + processDefinition.getId(), processDefinition.getName()); + putMsg(result, Status.PROCESS_INSTANCE_NOT_EXIST, processInstanceId); + return result; + } + switch (executeType) { case REPEAT_RUNNING: result = insertCommand(loginUser, processInstanceId, processDefinition.getId(), CommandType.REPEAT_RUNNING); @@ -260,7 +270,7 @@ public class ExecutorService extends BaseService{ } break; case RECOVER_SUSPENDED_PROCESS: - if (executionStatus.typeIsPause()) { + if (executionStatus.typeIsPause()|| executionStatus.typeIsCancel()) { checkResult = true; } default: diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessInstanceService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessInstanceService.java index 3cbf5f1414..88ff3f1019 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessInstanceService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessInstanceService.java @@ -38,10 +38,7 @@ import cn.escheduler.common.utils.JSONUtils; import cn.escheduler.common.utils.ParameterUtils; import cn.escheduler.common.utils.placeholder.BusinessTimeUtils; import cn.escheduler.dao.ProcessDao; -import cn.escheduler.dao.mapper.ProcessDefinitionMapper; -import cn.escheduler.dao.mapper.ProcessInstanceMapper; -import cn.escheduler.dao.mapper.ProjectMapper; -import cn.escheduler.dao.mapper.TaskInstanceMapper; +import cn.escheduler.dao.mapper.*; import cn.escheduler.dao.model.*; import com.alibaba.fastjson.JSON; import org.apache.commons.lang3.StringUtils; @@ -97,6 +94,9 @@ public class ProcessInstanceService extends BaseDAGService { @Autowired LoggerService loggerService; + @Autowired + WorkerGroupMapper workerGroupMapper; + /** * query process instance by id * @@ -115,6 +115,12 @@ public class ProcessInstanceService extends BaseDAGService { return checkResult; } ProcessInstance processInstance = processDao.findProcessInstanceDetailById(processId); + if(processInstance.getWorkerGroupId() == -1){ + processInstance.setWorkerGroupName("Default"); + }else{ + WorkerGroup workerGroup = workerGroupMapper.queryById(processInstance.getWorkerGroupId()); + processInstance.setWorkerGroupName(workerGroup.getName()); + } result.put(Constants.DATA_LIST, processInstance); putMsg(result, Status.SUCCESS); diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/model/ProcessInstance.java b/escheduler-dao/src/main/java/cn/escheduler/dao/model/ProcessInstance.java index 24902c0121..158bf3f847 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/model/ProcessInstance.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/model/ProcessInstance.java @@ -194,6 +194,11 @@ public class ProcessInstance { */ private int tenantId; + /** + * worker group name. for api. + */ + private String workerGroupName; + public ProcessInstance(){ } @@ -549,6 +554,7 @@ public class ProcessInstance { ", dependenceScheduleTimes='" + dependenceScheduleTimes + '\'' + ", duration=" + duration + ", timeout=" + timeout + + ", workerGroupName=" + workerGroupName + ", processInstancePriority=" + processInstancePriority + '}'; } @@ -560,4 +566,12 @@ public class ProcessInstance { public int getTenantId() { return this.tenantId ; } + + public String getWorkerGroupName() { + return workerGroupName; + } + + public void setWorkerGroupName(String workerGroupName) { + this.workerGroupName = workerGroupName; + } } diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue index 2bc1cad066..b867c1751b 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue @@ -95,10 +95,10 @@ shape="circle" size="xsmall" data-toggle="tooltip" - :title="$t('Stop')" - @click="_stop(item)" + :title="item.state === 'STOP' ? $t('Recovery Stop') : $t('Stop')" + @click="_stop(item,$index)" icon="iconfont icon-zanting1" - :disabled="item.state !== 'RUNNING_EXEUTION'"> + :disabled="item.state !== 'RUNNING_EXEUTION' && item.state != 'STOP'"> Date: Tue, 14 May 2019 17:25:37 +0800 Subject: [PATCH 069/149] cherry-pick 1d6241725603dc761d44ec126203d6fc417b6aea --- .../cn/escheduler/alert/utils/Constants.java | 12 ++ .../alert/utils/EnterpriseWeChatUtils.java | 167 ++++++++++++++++++ .../src/main/resources/alert.properties | 7 + .../utils/EnterpriseWeChatUtilsTest.java | 110 ++++++++++++ 4 files changed, 296 insertions(+) create mode 100644 escheduler-alert/src/main/java/cn/escheduler/alert/utils/EnterpriseWeChatUtils.java create mode 100644 escheduler-alert/src/test/java/cn/escheduler/alert/utils/EnterpriseWeChatUtilsTest.java diff --git a/escheduler-alert/src/main/java/cn/escheduler/alert/utils/Constants.java b/escheduler-alert/src/main/java/cn/escheduler/alert/utils/Constants.java index 1e1a7671e8..70136f3972 100644 --- a/escheduler-alert/src/main/java/cn/escheduler/alert/utils/Constants.java +++ b/escheduler-alert/src/main/java/cn/escheduler/alert/utils/Constants.java @@ -128,4 +128,16 @@ public class Constants { public static final String TH_END = ""; public static final int ALERT_SCAN_INTERVEL = 5000; + + public static final String ENTERPRISE_WECHAT_CORP_ID = "enterprise.wechat.corp.id"; + + public static final String ENTERPRISE_WECHAT_SECRET = "enterprise.wechat.secret"; + + public static final String ENTERPRISE_WECHAT_TOKEN_URL = "enterprise.wechat.token.url"; + + public static final String ENTERPRISE_WECHAT_PUSH_URL = "enterprise.wechat.push.url"; + + public static final String ENTERPRISE_WECHAT_TEAM_SEND_MSG = "enterprise.wechat.team.send.msg"; + + public static final String ENTERPRISE_WECHAT_USER_SEND_MSG = "enterprise.wechat.user.send.msg"; } diff --git a/escheduler-alert/src/main/java/cn/escheduler/alert/utils/EnterpriseWeChatUtils.java b/escheduler-alert/src/main/java/cn/escheduler/alert/utils/EnterpriseWeChatUtils.java new file mode 100644 index 0000000000..169977b01e --- /dev/null +++ b/escheduler-alert/src/main/java/cn/escheduler/alert/utils/EnterpriseWeChatUtils.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package cn.escheduler.alert.utils; + +import com.alibaba.fastjson.JSON; + +import com.google.common.reflect.TypeToken; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Collection; +import java.util.Map; + +import static cn.escheduler.alert.utils.PropertyUtils.getString; + +/** + * qiye weixin utils + */ +public class EnterpriseWeChatUtils { + + public static final Logger logger = LoggerFactory.getLogger(EnterpriseWeChatUtils.class); + + private static final String enterpriseWeChatCorpId = getString(Constants.ENTERPRISE_WECHAT_CORP_ID); + + private static final String enterpriseWeChatSecret = getString(Constants.ENTERPRISE_WECHAT_SECRET); + + private static final String enterpriseWeChatTokenUrl = getString(Constants.ENTERPRISE_WECHAT_TOKEN_URL); + private String enterpriseWeChatTokenUrlReplace = enterpriseWeChatTokenUrl + .replaceAll("\\$corpId", enterpriseWeChatCorpId) + .replaceAll("\\$secret", enterpriseWeChatSecret); + + private static final String enterpriseWeChatPushUrl = getString(Constants.ENTERPRISE_WECHAT_PUSH_URL); + + private static final String enterpriseWeChatTeamSendMsg = getString(Constants.ENTERPRISE_WECHAT_TEAM_SEND_MSG); + + private static final String enterpriseWeChatUserSendMsg = getString(Constants.ENTERPRISE_WECHAT_USER_SEND_MSG); + + /** + * get winxin token info + * @return token string info + * @throws IOException + */ + public String getToken() throws IOException { + String resp; + + CloseableHttpClient httpClient = HttpClients.createDefault(); + HttpGet httpGet = new HttpGet(enterpriseWeChatTokenUrlReplace); + CloseableHttpResponse response = httpClient.execute(httpGet); + try { + HttpEntity entity = response.getEntity(); + resp = EntityUtils.toString(entity, "utf-8"); + EntityUtils.consume(entity); + } finally { + response.close(); + } + + Map map = JSON.parseObject(resp, + new TypeToken>() { + }.getType()); + return map.get("access_token").toString(); + } + + /** + * make team single weixin message + * @param toParty + * @param agentId + * @param msg + * @return weixin send message + */ + public String makeTeamSendMsg(String toParty, String agentId, String msg) { + return enterpriseWeChatTeamSendMsg.replaceAll("\\$toParty", toParty) + .replaceAll("\\$agentId", agentId) + .replaceAll("\\$msg", msg); + } + + /** + * make team multi weixin message + * @param toParty + * @param agentId + * @param msg + * @return weixin send message + */ + public String makeTeamSendMsg(Collection toParty, String agentId, String msg) { + String listParty = FuncUtils.mkString(toParty, "|"); + return enterpriseWeChatTeamSendMsg.replaceAll("\\$toParty", listParty) + .replaceAll("\\$agentId", agentId) + .replaceAll("\\$msg", msg); + } + + /** + * make team single user message + * @param toUser + * @param agentId + * @param msg + * @return weixin send message + */ + public String makeUserSendMsg(String toUser, String agentId, String msg) { + return enterpriseWeChatUserSendMsg.replaceAll("\\$toUser", toUser) + .replaceAll("\\$agentId", agentId) + .replaceAll("\\$msg", msg); + } + + /** + * make team multi user message + * @param toUser + * @param agentId + * @param msg + * @return weixin send message + */ + public String makeUserSendMsg(Collection toUser, String agentId, String msg) { + String listUser = FuncUtils.mkString(toUser, "|"); + return enterpriseWeChatUserSendMsg.replaceAll("\\$toUser", listUser) + .replaceAll("\\$agentId", agentId) + .replaceAll("\\$msg", msg); + } + + /** + * send weixin + * @param charset + * @param data + * @param token + * @return weixin resp, demo: {"errcode":0,"errmsg":"ok","invaliduser":""} + * @throws IOException + */ + public String sendQiyeWeixin(String charset, String data, String token) throws IOException { + String enterpriseWeChatPushUrlReplace = enterpriseWeChatPushUrl.replaceAll("\\$token", token); + + CloseableHttpClient httpclient = HttpClients.createDefault(); + HttpPost httpPost = new HttpPost(enterpriseWeChatPushUrlReplace); + httpPost.setEntity(new StringEntity(data, charset)); + CloseableHttpResponse response = httpclient.execute(httpPost); + String resp; + try { + HttpEntity entity = response.getEntity(); + resp = EntityUtils.toString(entity, charset); + EntityUtils.consume(entity); + } finally { + response.close(); + } + logger.info("qiye weixin send [{}], param:{}, resp:{}", enterpriseWeChatPushUrl, data, resp); + return resp; + } + +} diff --git a/escheduler-alert/src/main/resources/alert.properties b/escheduler-alert/src/main/resources/alert.properties index 602384a818..8a8b8a5365 100644 --- a/escheduler-alert/src/main/resources/alert.properties +++ b/escheduler-alert/src/main/resources/alert.properties @@ -16,6 +16,13 @@ mail.smtp.ssl.enable=true #xls file path,need create if not exist xls.file.path=/tmp/xls +# Enterprise WeChat configuration +enterprise.wechat.corp.id=xxxxxxx +enterprise.wechat.secret=xxxxxxx +enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret +enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token +enterprise.wechat.team.send.msg={\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"} +enterprise.wechat.user.send.msg={\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"} diff --git a/escheduler-alert/src/test/java/cn/escheduler/alert/utils/EnterpriseWeChatUtilsTest.java b/escheduler-alert/src/test/java/cn/escheduler/alert/utils/EnterpriseWeChatUtilsTest.java new file mode 100644 index 0000000000..2a45196aed --- /dev/null +++ b/escheduler-alert/src/test/java/cn/escheduler/alert/utils/EnterpriseWeChatUtilsTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package cn.escheduler.alert.utils; + +import org.junit.Assert; +import org.junit.Test; + +import com.alibaba.fastjson.JSON; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; + +/** + * Please manually modify the configuration file before testing. + * file: alert.properties + * enterprise.wechat.corp.id + * enterprise.wechat.secret + * enterprise.wechat.token.url + * enterprise.wechat.push.url + * enterprise.wechat.send.msg + */ +public class EnterpriseWeChatUtilsTest { + + // Please change + private String agentId = "1000002"; // app id + private String partyId = "2"; + private Collection listPartyId = Arrays.asList("2","4"); + private String userId = "test1"; + private Collection listUserId = Arrays.asList("test1","test2"); + + @Test + public void testSendSingleTeamWeChat() { + EnterpriseWeChatUtils wx = new EnterpriseWeChatUtils(); + + try { + String token = wx.getToken(); + String msg = wx.makeTeamSendMsg(partyId, agentId, "hello world"); + String resp = wx.sendQiyeWeixin("utf-8", msg, token); + + String errmsg = JSON.parseObject(resp).getString("errmsg"); + Assert.assertEquals(errmsg, "ok"); + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void testSendMultiTeamWeChat() { + EnterpriseWeChatUtils wx = new EnterpriseWeChatUtils(); + + try { + String token = wx.getToken(); + String msg = wx.makeTeamSendMsg(listPartyId, agentId, "hello world"); + String resp = wx.sendQiyeWeixin("utf-8", msg, token); + + String errmsg = JSON.parseObject(resp).getString("errmsg"); + Assert.assertEquals(errmsg, "ok"); + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void testSendSingleUserWeChat() { + EnterpriseWeChatUtils wx = new EnterpriseWeChatUtils(); + + try { + String token = wx.getToken(); + String msg = wx.makeUserSendMsg(userId, agentId, "hello world"); + String resp = wx.sendQiyeWeixin("utf-8", msg, token); + + String errmsg = JSON.parseObject(resp).getString("errmsg"); + Assert.assertEquals(errmsg, "ok"); + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void testSendMultiUserWeChat() { + EnterpriseWeChatUtils wx = new EnterpriseWeChatUtils(); + + try { + String token = wx.getToken(); + String msg = wx.makeUserSendMsg(listUserId, agentId, "hello world"); + String resp = wx.sendQiyeWeixin("utf-8", msg, token); + + String errmsg = JSON.parseObject(resp).getString("errmsg"); + Assert.assertEquals(errmsg, "ok"); + } catch (IOException e) { + e.printStackTrace(); + } + } + +} From 1a1db6715cfb179de0963ffba88fa66db8f9dfc3 Mon Sep 17 00:00:00 2001 From: chendapao Date: Tue, 14 May 2019 16:57:31 +0800 Subject: [PATCH 070/149] cherry-pick e76af95910112ffcbebce457ad0812fc0ca472fd --- .../cn/escheduler/alert/utils/FuncUtils.java | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 escheduler-alert/src/main/java/cn/escheduler/alert/utils/FuncUtils.java diff --git a/escheduler-alert/src/main/java/cn/escheduler/alert/utils/FuncUtils.java b/escheduler-alert/src/main/java/cn/escheduler/alert/utils/FuncUtils.java new file mode 100644 index 0000000000..b4238beef0 --- /dev/null +++ b/escheduler-alert/src/main/java/cn/escheduler/alert/utils/FuncUtils.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package cn.escheduler.alert.utils; + +public class FuncUtils { + + static public String mkString(Iterable list, String split) { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (String item : list) { + if (first) + first = false; + else + sb.append(split); + sb.append(item); + } + return sb.toString(); + } + +} From 27c33609ddb8d903b9ad90e669ba9524d0536c07 Mon Sep 17 00:00:00 2001 From: lenboo Date: Thu, 4 Jul 2019 11:33:29 +0800 Subject: [PATCH 071/149] change stop and pause at ui add receivers and cc in process instance for api --- .../api/service/ProcessInstanceService.java | 3 ++ .../escheduler/dao/model/ProcessInstance.java | 27 +++++++++++- .../instance/pages/list/_source/list.vue | 43 +++++++++++-------- .../src/js/module/i18n/locale/en_US.js | 1 - .../src/js/module/i18n/locale/zh_CN.js | 5 +-- 5 files changed, 57 insertions(+), 22 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessInstanceService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessInstanceService.java index 88ff3f1019..cf2e5e494b 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessInstanceService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessInstanceService.java @@ -121,6 +121,9 @@ public class ProcessInstanceService extends BaseDAGService { WorkerGroup workerGroup = workerGroupMapper.queryById(processInstance.getWorkerGroupId()); processInstance.setWorkerGroupName(workerGroup.getName()); } + ProcessDefinition processDefinition = processDao.findProcessDefineById(processInstance.getProcessDefinitionId()); + processInstance.setReceivers(processDefinition.getReceivers()); + processInstance.setReceiversCc(processDefinition.getReceiversCc()); result.put(Constants.DATA_LIST, processInstance); putMsg(result, Status.SUCCESS); diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/model/ProcessInstance.java b/escheduler-dao/src/main/java/cn/escheduler/dao/model/ProcessInstance.java index 158bf3f847..5c9418ca72 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/model/ProcessInstance.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/model/ProcessInstance.java @@ -199,6 +199,16 @@ public class ProcessInstance { */ private String workerGroupName; + /** + * receivers for api + */ + private String receivers; + + /** + * receivers cc for api + */ + private String receiversCc; + public ProcessInstance(){ } @@ -554,7 +564,6 @@ public class ProcessInstance { ", dependenceScheduleTimes='" + dependenceScheduleTimes + '\'' + ", duration=" + duration + ", timeout=" + timeout + - ", workerGroupName=" + workerGroupName + ", processInstancePriority=" + processInstancePriority + '}'; } @@ -574,4 +583,20 @@ public class ProcessInstance { public void setWorkerGroupName(String workerGroupName) { this.workerGroupName = workerGroupName; } + + public String getReceivers() { + return receivers; + } + + public void setReceivers(String receivers) { + this.receivers = receivers; + } + + public String getReceiversCc() { + return receiversCc; + } + + public void setReceiversCc(String receiversCc) { + this.receiversCc = receiversCc; + } } diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue index b867c1751b..619407a61a 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue @@ -95,9 +95,9 @@ shape="circle" size="xsmall" data-toggle="tooltip" - :title="item.state === 'STOP' ? $t('Recovery Stop') : $t('Stop')" + :title="item.state === 'STOP' ? $t('Recovery Suspend') : $t('Stop')" @click="_stop(item,$index)" - icon="iconfont icon-zanting1" + :icon="item.state === 'STOP' ? 'iconfont icon-ai06' : 'iconfont icon-zanting'" :disabled="item.state !== 'RUNNING_EXEUTION' && item.state != 'STOP'"> - {{item.count}}s + {{item.count}} - {{item.count}}s + {{item.count}} - - + + + + + + + - {{item.count}}s + {{item.count}} + + + @@ -392,7 +401,7 @@ } else { this._upExecutorsState({ processInstanceId: item.id, - executeType: item.state === 'PAUSE' ? 'RECOVER_SUSPENDED_PROCESS' : 'PAUSE' + executeType: 'PAUSE' }) } }, @@ -444,7 +453,7 @@ if (data.length) { _.map(data, v => { v.disabled = true - v.count = 10 + v.count = 9 }) } return data diff --git a/escheduler-ui/src/js/module/i18n/locale/en_US.js b/escheduler-ui/src/js/module/i18n/locale/en_US.js index 8933f10c4a..57526d5b8d 100644 --- a/escheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/escheduler-ui/src/js/module/i18n/locale/en_US.js @@ -238,7 +238,6 @@ export default { 'Stop': 'Stop', 'Pause': 'Pause', 'Recovery Suspend': 'Recovery Suspend', - 'Recovery Stop': 'Recovery Stop', 'Gantt': 'Gantt', 'Name': 'Name', 'Node Type': 'Node Type', diff --git a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js index 4c9c3a1cbe..9353ea8866 100644 --- a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -237,8 +237,7 @@ export default { 'Recovery Failed': '恢复失败', 'Stop': '停止', 'Pause': '暂停', - 'Recovery Suspend': '恢复暂停', - 'Recovery Stop': '恢复停止', + 'Recovery Suspend': '恢复运行', 'Gantt': '甘特图', 'Name': '名称', 'Node Type': '节点类型', @@ -283,7 +282,7 @@ export default { 'Start Process': '启动工作流', 'Execute from the current node': '从当前节点开始执行', 'Recover tolerance fault process': '恢复被容错的工作流', - 'Resume the suspension process': '恢复暂停流程', + 'Resume the suspension process': '恢复运行流程', 'Execute from the failed nodes': '从失败节点开始执行', 'Complement Data': '补数', 'Scheduling execution': '调度执行', From e678d68d9982a4b69143a3a33c33d641c2158505 Mon Sep 17 00:00:00 2001 From: lidongdai Date: Wed, 3 Jul 2019 20:47:18 +0800 Subject: [PATCH 072/149] update api i18n and docs --- docs/zh_CN/系统架构设计.md | 6 +++--- .../src/main/resources/i18n/messages.properties | 12 ++++++++++++ .../main/resources/i18n/messages_en_US.properties | 12 ++++++++++++ .../main/resources/i18n/messages_zh_CN.properties | 10 ++++++++++ 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/docs/zh_CN/系统架构设计.md b/docs/zh_CN/系统架构设计.md index a6e1645a4c..134684155d 100644 --- a/docs/zh_CN/系统架构设计.md +++ b/docs/zh_CN/系统架构设计.md @@ -13,13 +13,13 @@ **流程定义**：通过拖拽任务节点并建立任务节点的关联所形成的可视化**DAG** -**流程实例**：流程实例是流程定义的实例化，可以通过手动启动或定时调度生成 +**流程实例**：流程实例是流程定义的实例化，可以通过手动启动或定时调度生成,流程定义每运行一次，产生一个流程实例 **任务实例**：任务实例是流程定义中任务节点的实例化，标识着具体的任务执行状态 -**任务类型**： 目前支持有SHELL、SQL、SUB_PROCESS、PROCEDURE、MR、SPARK、PYTHON、DEPENDENT，同时计划支持动态插件扩展，注意：其中子 **SUB_PROCESS** 也是一个单独的流程定义，是可以单独启动执行的 +**任务类型**： 目前支持有SHELL、SQL、SUB_PROCESS(子流程)、PROCEDURE、MR、SPARK、PYTHON、DEPENDENT(依赖)，同时计划支持动态插件扩展，注意：其中子 **SUB_PROCESS** 也是一个单独的流程定义，是可以单独启动执行的 -**调度方式：** 系统支持基于cron表达式的定时调度和手动调度。命令类型支持：启动工作流、从当前节点开始执行、恢复被容错的工作流、恢复暂停流程、从失败节点开始执行、补数、调度、重跑、暂停、停止、恢复等待线程。其中 **恢复被容错的工作流** 和 **恢复等待线程** 两种命令类型是由调度内部控制使用，外部无法调用 +**调度方式：** 系统支持基于cron表达式的定时调度和手动调度。命令类型支持：启动工作流、从当前节点开始执行、恢复被容错的工作流、恢复暂停流程、从失败节点开始执行、补数、定时、重跑、暂停、停止、恢复等待线程。其中 **恢复被容错的工作流** 和 **恢复等待线程** 两种命令类型是由调度内部控制使用，外部无法调用 **定时调度**：系统采用 **quartz** 分布式调度器，并同时支持cron表达式可视化的生成 diff --git a/escheduler-api/src/main/resources/i18n/messages.properties b/escheduler-api/src/main/resources/i18n/messages.properties index ea29b7d329..a663c71013 100644 --- a/escheduler-api/src/main/resources/i18n/messages.properties +++ b/escheduler-api/src/main/resources/i18n/messages.properties @@ -1,4 +1,16 @@ QUERY_SCHEDULE_LIST_NOTES=query schedule list +EXECUTE_PROCESS_TAG=execute process related operation +PROCESS_INSTANCE_EXECUTOR_TAG=process instance executor related operation +RUN_PROCESS_INSTANCE_NOTES=run process instance +START_NODE_LIST=start node list（node name） +TASK_DEPEND_TYPE=task depend type +COMMAND_TYPE=command type +RUN_MODE=run mode +TIMEOUT=timeout +EXECUTE_ACTION_TO_PROCESS_INSTANCE_NOTES=execute action to process instance +EXECUTE_TYPE=execute type +START_CHECK_PROCESS_DEFINITION_NOTES=start check process definition +GET_RECEIVER_CC_NOTES=query receiver cc DESC=description GROUP_NAME=group name GROUP_TYPE=group type diff --git a/escheduler-api/src/main/resources/i18n/messages_en_US.properties b/escheduler-api/src/main/resources/i18n/messages_en_US.properties index ea29b7d329..a663c71013 100644 --- a/escheduler-api/src/main/resources/i18n/messages_en_US.properties +++ b/escheduler-api/src/main/resources/i18n/messages_en_US.properties @@ -1,4 +1,16 @@ QUERY_SCHEDULE_LIST_NOTES=query schedule list +EXECUTE_PROCESS_TAG=execute process related operation +PROCESS_INSTANCE_EXECUTOR_TAG=process instance executor related operation +RUN_PROCESS_INSTANCE_NOTES=run process instance +START_NODE_LIST=start node list（node name） +TASK_DEPEND_TYPE=task depend type +COMMAND_TYPE=command type +RUN_MODE=run mode +TIMEOUT=timeout +EXECUTE_ACTION_TO_PROCESS_INSTANCE_NOTES=execute action to process instance +EXECUTE_TYPE=execute type +START_CHECK_PROCESS_DEFINITION_NOTES=start check process definition +GET_RECEIVER_CC_NOTES=query receiver cc DESC=description GROUP_NAME=group name GROUP_TYPE=group type diff --git a/escheduler-api/src/main/resources/i18n/messages_zh_CN.properties b/escheduler-api/src/main/resources/i18n/messages_zh_CN.properties index 7ee8e8b778..b0d6694d2b 100644 --- a/escheduler-api/src/main/resources/i18n/messages_zh_CN.properties +++ b/escheduler-api/src/main/resources/i18n/messages_zh_CN.properties @@ -1,4 +1,14 @@ QUERY_SCHEDULE_LIST_NOTES=查询定时列表 +PROCESS_INSTANCE_EXECUTOR_TAG=流程实例执行相关操作 +RUN_PROCESS_INSTANCE_NOTES=运行流程实例 +START_NODE_LIST=开始节点列表(节点name) +TASK_DEPEND_TYPE=任务依赖类型 +COMMAND_TYPE=指令类型 +RUN_MODE=运行模式 +TIMEOUT=超时时间 +EXECUTE_ACTION_TO_PROCESS_INSTANCE_NOTES=执行流程实例的各种操作(暂停、停止、重跑、恢复等) +EXECUTE_TYPE=执行类型 +START_CHECK_PROCESS_DEFINITION_NOTES=检查流程定义 DESC=备注(描述) GROUP_NAME=组名称 GROUP_TYPE=组类型 From 65015f98d403f8deba0823819b71dd3386fbec70 Mon Sep 17 00:00:00 2001 From: lenboo Date: Thu, 4 Jul 2019 15:43:16 +0800 Subject: [PATCH 073/149] update --- .../java/cn/escheduler/api/service/ProcessInstanceService.java | 2 +- .../src/main/java/cn/escheduler/common/Constants.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessInstanceService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessInstanceService.java index cf2e5e494b..9ba29e0555 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessInstanceService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessInstanceService.java @@ -116,7 +116,7 @@ public class ProcessInstanceService extends BaseDAGService { } ProcessInstance processInstance = processDao.findProcessInstanceDetailById(processId); if(processInstance.getWorkerGroupId() == -1){ - processInstance.setWorkerGroupName("Default"); + processInstance.setWorkerGroupName(DEFAULT); }else{ WorkerGroup workerGroup = workerGroupMapper.queryById(processInstance.getWorkerGroupId()); processInstance.setWorkerGroupName(workerGroup.getName()); diff --git a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java index 62332cbb75..de63843bfb 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java @@ -483,6 +483,8 @@ public final class Constants { public static final String TASK_RECORD_PWD = "task.record.datasource.password"; + public static final String DEFAULT = "Default"; + public static String TASK_RECORD_TABLE_HIVE_LOG = "eamp_hive_log_hd"; public static String TASK_RECORD_TABLE_HISTORY_HIVE_LOG = "eamp_hive_hist_log_hd"; From c3a0dcc4a8173e1193db8a5c8e0d25707ee4c806 Mon Sep 17 00:00:00 2001 From: lidongdai Date: Wed, 3 Jul 2019 20:47:18 +0800 Subject: [PATCH 074/149] update api i18n and docs --- docs/zh_CN/系统架构设计.md | 6 +++--- .../src/main/resources/i18n/messages.properties | 12 ++++++++++++ .../main/resources/i18n/messages_en_US.properties | 12 ++++++++++++ .../main/resources/i18n/messages_zh_CN.properties | 10 ++++++++++ 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/docs/zh_CN/系统架构设计.md b/docs/zh_CN/系统架构设计.md index a6e1645a4c..134684155d 100644 --- a/docs/zh_CN/系统架构设计.md +++ b/docs/zh_CN/系统架构设计.md @@ -13,13 +13,13 @@ **流程定义**：通过拖拽任务节点并建立任务节点的关联所形成的可视化**DAG** -**流程实例**：流程实例是流程定义的实例化，可以通过手动启动或定时调度生成 +**流程实例**：流程实例是流程定义的实例化，可以通过手动启动或定时调度生成,流程定义每运行一次，产生一个流程实例 **任务实例**：任务实例是流程定义中任务节点的实例化，标识着具体的任务执行状态 -**任务类型**： 目前支持有SHELL、SQL、SUB_PROCESS、PROCEDURE、MR、SPARK、PYTHON、DEPENDENT，同时计划支持动态插件扩展，注意：其中子 **SUB_PROCESS** 也是一个单独的流程定义，是可以单独启动执行的 +**任务类型**： 目前支持有SHELL、SQL、SUB_PROCESS(子流程)、PROCEDURE、MR、SPARK、PYTHON、DEPENDENT(依赖)，同时计划支持动态插件扩展，注意：其中子 **SUB_PROCESS** 也是一个单独的流程定义，是可以单独启动执行的 -**调度方式：** 系统支持基于cron表达式的定时调度和手动调度。命令类型支持：启动工作流、从当前节点开始执行、恢复被容错的工作流、恢复暂停流程、从失败节点开始执行、补数、调度、重跑、暂停、停止、恢复等待线程。其中 **恢复被容错的工作流** 和 **恢复等待线程** 两种命令类型是由调度内部控制使用，外部无法调用 +**调度方式：** 系统支持基于cron表达式的定时调度和手动调度。命令类型支持：启动工作流、从当前节点开始执行、恢复被容错的工作流、恢复暂停流程、从失败节点开始执行、补数、定时、重跑、暂停、停止、恢复等待线程。其中 **恢复被容错的工作流** 和 **恢复等待线程** 两种命令类型是由调度内部控制使用，外部无法调用 **定时调度**：系统采用 **quartz** 分布式调度器，并同时支持cron表达式可视化的生成 diff --git a/escheduler-api/src/main/resources/i18n/messages.properties b/escheduler-api/src/main/resources/i18n/messages.properties index ea29b7d329..a663c71013 100644 --- a/escheduler-api/src/main/resources/i18n/messages.properties +++ b/escheduler-api/src/main/resources/i18n/messages.properties @@ -1,4 +1,16 @@ QUERY_SCHEDULE_LIST_NOTES=query schedule list +EXECUTE_PROCESS_TAG=execute process related operation +PROCESS_INSTANCE_EXECUTOR_TAG=process instance executor related operation +RUN_PROCESS_INSTANCE_NOTES=run process instance +START_NODE_LIST=start node list（node name） +TASK_DEPEND_TYPE=task depend type +COMMAND_TYPE=command type +RUN_MODE=run mode +TIMEOUT=timeout +EXECUTE_ACTION_TO_PROCESS_INSTANCE_NOTES=execute action to process instance +EXECUTE_TYPE=execute type +START_CHECK_PROCESS_DEFINITION_NOTES=start check process definition +GET_RECEIVER_CC_NOTES=query receiver cc DESC=description GROUP_NAME=group name GROUP_TYPE=group type diff --git a/escheduler-api/src/main/resources/i18n/messages_en_US.properties b/escheduler-api/src/main/resources/i18n/messages_en_US.properties index ea29b7d329..a663c71013 100644 --- a/escheduler-api/src/main/resources/i18n/messages_en_US.properties +++ b/escheduler-api/src/main/resources/i18n/messages_en_US.properties @@ -1,4 +1,16 @@ QUERY_SCHEDULE_LIST_NOTES=query schedule list +EXECUTE_PROCESS_TAG=execute process related operation +PROCESS_INSTANCE_EXECUTOR_TAG=process instance executor related operation +RUN_PROCESS_INSTANCE_NOTES=run process instance +START_NODE_LIST=start node list（node name） +TASK_DEPEND_TYPE=task depend type +COMMAND_TYPE=command type +RUN_MODE=run mode +TIMEOUT=timeout +EXECUTE_ACTION_TO_PROCESS_INSTANCE_NOTES=execute action to process instance +EXECUTE_TYPE=execute type +START_CHECK_PROCESS_DEFINITION_NOTES=start check process definition +GET_RECEIVER_CC_NOTES=query receiver cc DESC=description GROUP_NAME=group name GROUP_TYPE=group type diff --git a/escheduler-api/src/main/resources/i18n/messages_zh_CN.properties b/escheduler-api/src/main/resources/i18n/messages_zh_CN.properties index 7ee8e8b778..b0d6694d2b 100644 --- a/escheduler-api/src/main/resources/i18n/messages_zh_CN.properties +++ b/escheduler-api/src/main/resources/i18n/messages_zh_CN.properties @@ -1,4 +1,14 @@ QUERY_SCHEDULE_LIST_NOTES=查询定时列表 +PROCESS_INSTANCE_EXECUTOR_TAG=流程实例执行相关操作 +RUN_PROCESS_INSTANCE_NOTES=运行流程实例 +START_NODE_LIST=开始节点列表(节点name) +TASK_DEPEND_TYPE=任务依赖类型 +COMMAND_TYPE=指令类型 +RUN_MODE=运行模式 +TIMEOUT=超时时间 +EXECUTE_ACTION_TO_PROCESS_INSTANCE_NOTES=执行流程实例的各种操作(暂停、停止、重跑、恢复等) +EXECUTE_TYPE=执行类型 +START_CHECK_PROCESS_DEFINITION_NOTES=检查流程定义 DESC=备注(描述) GROUP_NAME=组名称 GROUP_TYPE=组类型 From 9cefd2e898ddf3bae3d49981fe366a1e01061194 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Thu, 4 Jul 2019 15:47:13 +0800 Subject: [PATCH 075/149] add markdown style utils --- .../cn/escheduler/alert/utils/Constants.java | 4 + .../alert/utils/EnterpriseWeChatUtils.java | 117 +++++++++++++++--- 2 files changed, 101 insertions(+), 20 deletions(-) diff --git a/escheduler-alert/src/main/java/cn/escheduler/alert/utils/Constants.java b/escheduler-alert/src/main/java/cn/escheduler/alert/utils/Constants.java index 70136f3972..b9a81b3922 100644 --- a/escheduler-alert/src/main/java/cn/escheduler/alert/utils/Constants.java +++ b/escheduler-alert/src/main/java/cn/escheduler/alert/utils/Constants.java @@ -129,6 +129,10 @@ public class Constants { public static final int ALERT_SCAN_INTERVEL = 5000; + public static final String MARKDOWN_QUOTE = ">"; + + public static final String MARKDOWN_ENTER = "\n"; + public static final String ENTERPRISE_WECHAT_CORP_ID = "enterprise.wechat.corp.id"; public static final String ENTERPRISE_WECHAT_SECRET = "enterprise.wechat.secret"; diff --git a/escheduler-alert/src/main/java/cn/escheduler/alert/utils/EnterpriseWeChatUtils.java b/escheduler-alert/src/main/java/cn/escheduler/alert/utils/EnterpriseWeChatUtils.java index 169977b01e..697412ea23 100644 --- a/escheduler-alert/src/main/java/cn/escheduler/alert/utils/EnterpriseWeChatUtils.java +++ b/escheduler-alert/src/main/java/cn/escheduler/alert/utils/EnterpriseWeChatUtils.java @@ -16,9 +16,12 @@ */ package cn.escheduler.alert.utils; +import cn.escheduler.common.enums.ShowType; +import cn.escheduler.dao.model.Alert; import com.alibaba.fastjson.JSON; import com.google.common.reflect.TypeToken; +import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; @@ -31,13 +34,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Collection; -import java.util.Map; +import java.util.*; import static cn.escheduler.alert.utils.PropertyUtils.getString; /** - * qiye weixin utils + * Enterprise WeChat utils */ public class EnterpriseWeChatUtils { @@ -48,7 +50,7 @@ public class EnterpriseWeChatUtils { private static final String enterpriseWeChatSecret = getString(Constants.ENTERPRISE_WECHAT_SECRET); private static final String enterpriseWeChatTokenUrl = getString(Constants.ENTERPRISE_WECHAT_TOKEN_URL); - private String enterpriseWeChatTokenUrlReplace = enterpriseWeChatTokenUrl + private static String enterpriseWeChatTokenUrlReplace = enterpriseWeChatTokenUrl .replaceAll("\\$corpId", enterpriseWeChatCorpId) .replaceAll("\\$secret", enterpriseWeChatSecret); @@ -59,11 +61,11 @@ public class EnterpriseWeChatUtils { private static final String enterpriseWeChatUserSendMsg = getString(Constants.ENTERPRISE_WECHAT_USER_SEND_MSG); /** - * get winxin token info + * get Enterprise WeChat token info * @return token string info * @throws IOException */ - public String getToken() throws IOException { + public static String getToken() throws IOException { String resp; CloseableHttpClient httpClient = HttpClients.createDefault(); @@ -71,7 +73,7 @@ public class EnterpriseWeChatUtils { CloseableHttpResponse response = httpClient.execute(httpGet); try { HttpEntity entity = response.getEntity(); - resp = EntityUtils.toString(entity, "utf-8"); + resp = EntityUtils.toString(entity, Constants.UTF_8); EntityUtils.consume(entity); } finally { response.close(); @@ -84,26 +86,26 @@ public class EnterpriseWeChatUtils { } /** - * make team single weixin message + * make team single Enterprise WeChat message * @param toParty * @param agentId * @param msg - * @return weixin send message + * @return Enterprise WeChat send message */ - public String makeTeamSendMsg(String toParty, String agentId, String msg) { + public static String makeTeamSendMsg(String toParty, String agentId, String msg) { return enterpriseWeChatTeamSendMsg.replaceAll("\\$toParty", toParty) .replaceAll("\\$agentId", agentId) .replaceAll("\\$msg", msg); } /** - * make team multi weixin message + * make team multi Enterprise WeChat message * @param toParty * @param agentId * @param msg - * @return weixin send message + * @return Enterprise WeChat send message */ - public String makeTeamSendMsg(Collection toParty, String agentId, String msg) { + public static String makeTeamSendMsg(Collection toParty, String agentId, String msg) { String listParty = FuncUtils.mkString(toParty, "|"); return enterpriseWeChatTeamSendMsg.replaceAll("\\$toParty", listParty) .replaceAll("\\$agentId", agentId) @@ -115,9 +117,9 @@ public class EnterpriseWeChatUtils { * @param toUser * @param agentId * @param msg - * @return weixin send message + * @return Enterprise WeChat send message */ - public String makeUserSendMsg(String toUser, String agentId, String msg) { + public static String makeUserSendMsg(String toUser, String agentId, String msg) { return enterpriseWeChatUserSendMsg.replaceAll("\\$toUser", toUser) .replaceAll("\\$agentId", agentId) .replaceAll("\\$msg", msg); @@ -128,9 +130,9 @@ public class EnterpriseWeChatUtils { * @param toUser * @param agentId * @param msg - * @return weixin send message + * @return Enterprise WeChat send message */ - public String makeUserSendMsg(Collection toUser, String agentId, String msg) { + public static String makeUserSendMsg(Collection toUser, String agentId, String msg) { String listUser = FuncUtils.mkString(toUser, "|"); return enterpriseWeChatUserSendMsg.replaceAll("\\$toUser", listUser) .replaceAll("\\$agentId", agentId) @@ -142,10 +144,10 @@ public class EnterpriseWeChatUtils { * @param charset * @param data * @param token - * @return weixin resp, demo: {"errcode":0,"errmsg":"ok","invaliduser":""} + * @return Enterprise WeChat resp, demo: {"errcode":0,"errmsg":"ok","invaliduser":""} * @throws IOException */ - public String sendQiyeWeixin(String charset, String data, String token) throws IOException { + public static String sendQiyeWeixin(String charset, String data, String token) throws IOException { String enterpriseWeChatPushUrlReplace = enterpriseWeChatPushUrl.replaceAll("\\$token", token); CloseableHttpClient httpclient = HttpClients.createDefault(); @@ -160,8 +162,83 @@ public class EnterpriseWeChatUtils { } finally { response.close(); } - logger.info("qiye weixin send [{}], param:{}, resp:{}", enterpriseWeChatPushUrl, data, resp); + logger.info("Enterprise WeChat send [{}], param:{}, resp:{}", enterpriseWeChatPushUrl, data, resp); return resp; } + /** + * convert table to markdown style + * @param title + * @param content + * @return + */ + public static String markdownTable(String title,String content){ + List mapItemsList = JSONUtils.toList(content, LinkedHashMap.class); + StringBuilder contents = new StringBuilder(200); + for (LinkedHashMap mapItems : mapItemsList){ + + Set> entries = mapItems.entrySet(); + + Iterator> iterator = entries.iterator(); + + StringBuilder t = new StringBuilder(String.format("`%s`%s",title,Constants.MARKDOWN_ENTER)); + while (iterator.hasNext()){ + + Map.Entry entry = iterator.next(); + t.append(Constants.MARKDOWN_QUOTE); + t.append(entry.getKey()).append(":").append(entry.getValue()); + t.append(Constants.MARKDOWN_ENTER); + } + + contents.append(t); + } + return contents.toString(); + } + + /** + * convert text to markdown style + * @param title + * @param content + * @return + */ + public static String markdownText(String title,String content){ + if (StringUtils.isNotEmpty(content)){ + List list; + try { + list = JSONUtils.toList(content,String.class); + }catch (Exception e){ + logger.error("json format exception",e); + return null; + } + + StringBuilder contents = new StringBuilder(100); + contents.append(String.format("`%s`\n",title)); + for (String str : list){ + contents.append(Constants.MARKDOWN_QUOTE); + contents.append(str); + contents.append(Constants.MARKDOWN_ENTER); + } + + return contents.toString(); + + } + return null; + } + + /** + * Determine the mardown style based on the show type of the alert + * @param alert + * @return + */ + public static String markdownByAlert(Alert alert){ + String result = ""; + if (alert.getShowType() == ShowType.TABLE) { + result = markdownTable(alert.getTitle(),alert.getContent()); + }else if(alert.getShowType() == ShowType.TEXT){ + result = markdownText(alert.getTitle(),alert.getContent()); + } + return result; + + } + } From 6b477f495caad1acb7964ebd8d15f4086edf580c Mon Sep 17 00:00:00 2001 From: huyuanming Date: Thu, 4 Jul 2019 15:51:44 +0800 Subject: [PATCH 076/149] startup parameters --- .../js/conf/home/pages/dag/_source/dag.vue | 18 +++ .../pages/dag/_source/startingParam/index.vue | 116 ++++++++++++++++++ .../conf/home/pages/dag/instanceDetails.vue | 4 +- .../src/js/conf/home/store/dag/actions.js | 4 + .../src/js/conf/home/store/dag/state.js | 5 +- .../src/js/module/i18n/locale/en_US.js | 6 +- .../src/js/module/i18n/locale/zh_CN.js | 5 +- 7 files changed, 153 insertions(+), 5 deletions(-) create mode 100644 escheduler-ui/src/js/conf/home/pages/dag/_source/startingParam/index.vue diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue b/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue index 2c0422c701..b2062b7769 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue @@ -28,6 +28,17 @@ @click="_toggleView" icon="fa fa-code"> + + {{name}}   @@ -383,6 +394,13 @@ _toggleView () { findComponentDownward(this.$root, `assist-dag-index`)._toggleView() }, + + /** + * Starting parameters + */ + _toggleParam () { + findComponentDownward(this.$root, `starting-params-dag-index`)._toggleParam() + }, /** * Create a node popup layer * @param Object id diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/startingParam/index.vue b/escheduler-ui/src/js/conf/home/pages/dag/_source/startingParam/index.vue new file mode 100644 index 0000000000..ceb019ce90 --- /dev/null +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/startingParam/index.vue @@ -0,0 +1,116 @@ + + + diff --git a/escheduler-ui/src/js/conf/home/pages/dag/instanceDetails.vue b/escheduler-ui/src/js/conf/home/pages/dag/instanceDetails.vue index cbe220ad36..705a151b78 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/instanceDetails.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/instanceDetails.vue @@ -1,6 +1,7 @@ From 5c7515cf8c74450a885e39c28fb6853f60a8cb5b Mon Sep 17 00:00:00 2001 From: lenboo Date: Fri, 5 Jul 2019 11:11:43 +0800 Subject: [PATCH 086/149] fix sub process worker group id is 0 --- .../escheduler/api/service/ProcessInstanceService.java | 10 ++++++++-- .../src/main/java/cn/escheduler/dao/ProcessDao.java | 3 ++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessInstanceService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessInstanceService.java index 9ba29e0555..f5f05a74d6 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessInstanceService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessInstanceService.java @@ -115,12 +115,18 @@ public class ProcessInstanceService extends BaseDAGService { return checkResult; } ProcessInstance processInstance = processDao.findProcessInstanceDetailById(processId); + String workerGroupName = ""; if(processInstance.getWorkerGroupId() == -1){ - processInstance.setWorkerGroupName(DEFAULT); + workerGroupName = DEFAULT; }else{ WorkerGroup workerGroup = workerGroupMapper.queryById(processInstance.getWorkerGroupId()); - processInstance.setWorkerGroupName(workerGroup.getName()); + if(workerGroup != null){ + workerGroupName = DEFAULT; + }else{ + workerGroupName = workerGroup.getName(); + } } + processInstance.setWorkerGroupName(workerGroupName); ProcessDefinition processDefinition = processDao.findProcessDefineById(processInstance.getProcessDefinitionId()); processInstance.setReceivers(processDefinition.getReceivers()); processInstance.setReceiversCc(processDefinition.getReceiversCc()); diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java index 8ba0f47960..d393339a5e 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java @@ -493,7 +493,8 @@ public class ProcessDao extends AbstractBaseDao { processInstance.setProcessInstanceJson(processDefinition.getProcessDefinitionJson()); // set process instance priority processInstance.setProcessInstancePriority(command.getProcessInstancePriority()); - processInstance.setWorkerGroupId(command.getWorkerGroupId()); + int workerGroupId = command.getWorkerGroupId() == 0 ? -1 : command.getWorkerGroupId(); + processInstance.setWorkerGroupId(workerGroupId); processInstance.setTimeout(processDefinition.getTimeout()); processInstance.setTenantId(processDefinition.getTenantId()); return processInstance; From d0a0b67c8b55fd4c538f3fc3218a675b713dd38a Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Fri, 5 Jul 2019 11:46:11 +0800 Subject: [PATCH 087/149] change 1.0.4 to 1.1.0 --- sql/soft_version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/soft_version b/sql/soft_version index a6a3a43c3a..1cc5f657e0 100644 --- a/sql/soft_version +++ b/sql/soft_version @@ -1 +1 @@ -1.0.4 \ No newline at end of file +1.1.0 \ No newline at end of file From d720108e2fd406c39856289e4c1c9667b60a3fc8 Mon Sep 17 00:00:00 2001 From: lenboo Date: Fri, 5 Jul 2019 15:25:15 +0800 Subject: [PATCH 088/149] update worker get task from queue --- .../common/queue/TaskQueueZkImpl.java | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java b/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java index e116cd3271..df75a3d525 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java @@ -22,6 +22,7 @@ import cn.escheduler.common.utils.Bytes; import cn.escheduler.common.utils.IpUtils; import cn.escheduler.common.utils.OSUtils; import cn.escheduler.common.zk.AbstractZKClient; +import org.apache.commons.lang3.StringUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.data.Stat; @@ -157,7 +158,7 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue { String taskDetail = list.get(i); String[] taskDetailArrs = taskDetail.split(Constants.UNDERLINE); - //向前版本兼容 + //向前版本兼ProcessInstanceService容 if(taskDetailArrs.length >= 4){ //format ${processInstancePriority}_${processInstanceId}_${taskInstancePriority}_${taskId} @@ -209,17 +210,33 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue { while(iterator.hasNext()){ if(j++ < tasksNum){ String task = iterator.next(); - String[] taskArray = task.split(Constants.UNDERLINE); - int processInstanceId = Integer.parseInt(taskArray[1]); - int taskId = Integer.parseInt(taskArray[3]); - String destTask = taskArray[0]+Constants.UNDERLINE + processInstanceId + Constants.UNDERLINE - + taskArray[2] + Constants.UNDERLINE + taskId; - taskslist.add(destTask); + + taskslist.add(getOriginTaskFormat(task)); } } return taskslist; } + /** + * format ${processInstancePriority}_${processInstanceId}_${taskInstancePriority}_${taskId} + * processInstanceId and task id need to be convert to int. + * @param formatTask + * @return + */ + private String getOriginTaskFormat(String formatTask){ + String[] taskArray = formatTask.split(Constants.UNDERLINE); + int processInstanceId = Integer.parseInt(taskArray[1]); + int taskId = Integer.parseInt(taskArray[3]); + String suffix = ""; + for(int index =4; index < taskArray.length; index++){ + suffix += taskArray[index] + Constants.UNDERLINE; + } + String destTask = String.format("%s_%s_%s_%s", taskArray[0], processInstanceId, taskArray[3], taskId); + if(StringUtils.isNotEmpty(suffix)){ + destTask += Constants.UNDERLINE + suffix; + } + return destTask; + } @Override public void removeNode(String key, String nodeValue){ From 67f82d71a8dc9a1d02aa311392af4ca512f17a79 Mon Sep 17 00:00:00 2001 From: lenboo Date: Fri, 5 Jul 2019 15:26:03 +0800 Subject: [PATCH 089/149] update worker get task from queue --- .../cn/escheduler/server/worker/runner/FetchTaskThread.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java index a960570ea5..2d88fdb843 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java @@ -153,7 +153,7 @@ public class FetchTaskThread implements Runnable{ } String[] taskStringArray = taskQueueStr.split(Constants.UNDERLINE); - String taskInstIdStr = taskStringArray[taskStringArray.length - 1]; + String taskInstIdStr = taskStringArray[3]; Date now = new Date(); Integer taskId = Integer.parseInt(taskInstIdStr); From e39e4768655d361d86982fbffcaeb76c5bd76240 Mon Sep 17 00:00:00 2001 From: lidongdai Date: Fri, 5 Jul 2019 16:08:09 +0800 Subject: [PATCH 090/149] optimize poll task from zookeeper --- .../common/queue/TaskQueueZkImpl.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java b/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java index df75a3d525..e6c9df2833 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java @@ -158,7 +158,7 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue { String taskDetail = list.get(i); String[] taskDetailArrs = taskDetail.split(Constants.UNDERLINE); - //向前版本兼ProcessInstanceService容 + //forward compatibility 向前版本兼容 if(taskDetailArrs.length >= 4){ //format ${processInstancePriority}_${processInstanceId}_${taskInstancePriority}_${taskId} @@ -227,15 +227,18 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue { String[] taskArray = formatTask.split(Constants.UNDERLINE); int processInstanceId = Integer.parseInt(taskArray[1]); int taskId = Integer.parseInt(taskArray[3]); - String suffix = ""; - for(int index =4; index < taskArray.length; index++){ - suffix += taskArray[index] + Constants.UNDERLINE; - } + + StringBuilder sb = new StringBuilder(50); String destTask = String.format("%s_%s_%s_%s", taskArray[0], processInstanceId, taskArray[3], taskId); - if(StringUtils.isNotEmpty(suffix)){ - destTask += Constants.UNDERLINE + suffix; + + sb.append(destTask); + + if(taskArray.length > 4){ + for(int index = 4; index < taskArray.length; index++){ + sb.append(Constants.UNDERLINE).append(taskArray[index]); + } } - return destTask; + return sb.toString(); } @Override From 050162a3da5f1a810e58f2ada1cd7b98bcf36f6b Mon Sep 17 00:00:00 2001 From: ligang Date: Mon, 8 Jul 2019 11:05:17 +0800 Subject: [PATCH 091/149] [maven-release-plugin] prepare release 1.1.0-preview --- escheduler-alert/pom.xml | 2 +- escheduler-api/pom.xml | 5 ++--- escheduler-common/pom.xml | 2 +- escheduler-dao/pom.xml | 2 +- escheduler-rpc/pom.xml | 2 +- escheduler-server/pom.xml | 2 +- pom.xml | 4 ++-- 7 files changed, 9 insertions(+), 10 deletions(-) diff --git a/escheduler-alert/pom.xml b/escheduler-alert/pom.xml index ed3baa40f0..cda2488c36 100644 --- a/escheduler-alert/pom.xml +++ b/escheduler-alert/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.0.4-SNAPSHOT + 1.1.0 escheduler-alert jar diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index 0c2a3019cd..6daff5e25f 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -1,10 +1,9 @@ - + 4.0.0 cn.analysys escheduler - 1.0.4-SNAPSHOT + 1.1.0 escheduler-api jar diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index 8e337a3e61..cc2bb36717 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.0.4-SNAPSHOT + 1.1.0 escheduler-common escheduler-common diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index 2f2a5ad74e..57d1ad30c2 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.0.4-SNAPSHOT + 1.1.0 escheduler-dao escheduler-dao diff --git a/escheduler-rpc/pom.xml b/escheduler-rpc/pom.xml index 4ec4ea2260..4687d1e462 100644 --- a/escheduler-rpc/pom.xml +++ b/escheduler-rpc/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.0.4-SNAPSHOT + 1.1.0 4.0.0 diff --git a/escheduler-server/pom.xml b/escheduler-server/pom.xml index 6341539e4c..ac9a312734 100644 --- a/escheduler-server/pom.xml +++ b/escheduler-server/pom.xml @@ -3,7 +3,7 @@ escheduler cn.analysys - 1.0.4-SNAPSHOT + 1.1.0 escheduler-server escheduler-server diff --git a/pom.xml b/pom.xml index e4186d9b81..93ac6f8ee9 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 cn.analysys escheduler - 1.0.4-SNAPSHOT + 1.1.0 pom escheduler http://maven.apache.org @@ -382,7 +382,7 @@ scm:git:https://github.com/analysys/EasyScheduler.git scm:git:https://github.com/analysys/EasyScheduler.git https://github.com/analysys/EasyScheduler.git - HEAD + 1.1.0-preview From 6a1e0b3fbadbbde9636b9cc7878cda2cbfd50c05 Mon Sep 17 00:00:00 2001 From: ligang Date: Mon, 8 Jul 2019 11:05:44 +0800 Subject: [PATCH 092/149] [maven-release-plugin] prepare for next development iteration --- escheduler-alert/pom.xml | 2 +- escheduler-api/pom.xml | 2 +- escheduler-common/pom.xml | 2 +- escheduler-dao/pom.xml | 2 +- escheduler-rpc/pom.xml | 2 +- escheduler-server/pom.xml | 2 +- pom.xml | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/escheduler-alert/pom.xml b/escheduler-alert/pom.xml index cda2488c36..be8b5fbb6d 100644 --- a/escheduler-alert/pom.xml +++ b/escheduler-alert/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT escheduler-alert jar diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index 6daff5e25f..56688f4024 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -3,7 +3,7 @@ cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT escheduler-api jar diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index cc2bb36717..35b025b1d8 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0 + 1.1.0-SNAPSHOT escheduler-common escheduler-common diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index 57d1ad30c2..52d9f7646d 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT escheduler-dao escheduler-dao diff --git a/escheduler-rpc/pom.xml b/escheduler-rpc/pom.xml index 4687d1e462..e3d7c91a92 100644 --- a/escheduler-rpc/pom.xml +++ b/escheduler-rpc/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0 + 1.1.0-SNAPSHOT 4.0.0 diff --git a/escheduler-server/pom.xml b/escheduler-server/pom.xml index ac9a312734..7f149278c8 100644 --- a/escheduler-server/pom.xml +++ b/escheduler-server/pom.xml @@ -3,7 +3,7 @@ escheduler cn.analysys - 1.1.0 + 1.1.0-SNAPSHOT escheduler-server escheduler-server diff --git a/pom.xml b/pom.xml index 93ac6f8ee9..a97650aafb 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT pom escheduler http://maven.apache.org @@ -382,7 +382,7 @@ scm:git:https://github.com/analysys/EasyScheduler.git scm:git:https://github.com/analysys/EasyScheduler.git https://github.com/analysys/EasyScheduler.git - 1.1.0-preview + HEAD From eadb230c63694ec52b48c83af9db022b5888771e Mon Sep 17 00:00:00 2001 From: ligang Date: Mon, 8 Jul 2019 14:00:08 +0800 Subject: [PATCH 093/149] [maven-release-plugin] prepare release 1.1.0-preview --- escheduler-alert/pom.xml | 2 +- escheduler-api/pom.xml | 2 +- escheduler-common/pom.xml | 2 +- escheduler-dao/pom.xml | 2 +- escheduler-rpc/pom.xml | 2 +- escheduler-server/pom.xml | 2 +- pom.xml | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/escheduler-alert/pom.xml b/escheduler-alert/pom.xml index be8b5fbb6d..a1ed6d7e07 100644 --- a/escheduler-alert/pom.xml +++ b/escheduler-alert/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0-preview escheduler-alert jar diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index 56688f4024..2c3c8733c9 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -3,7 +3,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0-preview escheduler-api jar diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index 35b025b1d8..95943ed098 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0-preview escheduler-common escheduler-common diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index 52d9f7646d..c0ccc4cc0f 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0-preview escheduler-dao escheduler-dao diff --git a/escheduler-rpc/pom.xml b/escheduler-rpc/pom.xml index e3d7c91a92..4b6257aa35 100644 --- a/escheduler-rpc/pom.xml +++ b/escheduler-rpc/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0-preview 4.0.0 diff --git a/escheduler-server/pom.xml b/escheduler-server/pom.xml index 7f149278c8..8ca47304a9 100644 --- a/escheduler-server/pom.xml +++ b/escheduler-server/pom.xml @@ -3,7 +3,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0-preview escheduler-server escheduler-server diff --git a/pom.xml b/pom.xml index a97650aafb..b4c610f4df 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0-preview pom escheduler http://maven.apache.org @@ -382,7 +382,7 @@ scm:git:https://github.com/analysys/EasyScheduler.git scm:git:https://github.com/analysys/EasyScheduler.git https://github.com/analysys/EasyScheduler.git - HEAD + 1.1.0-preview From 2519d5a594b38048a7df3d467de575fde3ddcf93 Mon Sep 17 00:00:00 2001 From: ligang Date: Mon, 8 Jul 2019 14:05:22 +0800 Subject: [PATCH 094/149] [maven-release-plugin] rollback the release of 1.1.0-preview --- escheduler-alert/pom.xml | 2 +- escheduler-api/pom.xml | 2 +- escheduler-common/pom.xml | 2 +- escheduler-dao/pom.xml | 2 +- escheduler-rpc/pom.xml | 2 +- escheduler-server/pom.xml | 2 +- pom.xml | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/escheduler-alert/pom.xml b/escheduler-alert/pom.xml index a1ed6d7e07..be8b5fbb6d 100644 --- a/escheduler-alert/pom.xml +++ b/escheduler-alert/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0-preview + 1.1.0-SNAPSHOT escheduler-alert jar diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index 2c3c8733c9..56688f4024 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -3,7 +3,7 @@ cn.analysys escheduler - 1.1.0-preview + 1.1.0-SNAPSHOT escheduler-api jar diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index 95943ed098..35b025b1d8 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0-preview + 1.1.0-SNAPSHOT escheduler-common escheduler-common diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index c0ccc4cc0f..52d9f7646d 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0-preview + 1.1.0-SNAPSHOT escheduler-dao escheduler-dao diff --git a/escheduler-rpc/pom.xml b/escheduler-rpc/pom.xml index 4b6257aa35..e3d7c91a92 100644 --- a/escheduler-rpc/pom.xml +++ b/escheduler-rpc/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0-preview + 1.1.0-SNAPSHOT 4.0.0 diff --git a/escheduler-server/pom.xml b/escheduler-server/pom.xml index 8ca47304a9..7f149278c8 100644 --- a/escheduler-server/pom.xml +++ b/escheduler-server/pom.xml @@ -3,7 +3,7 @@ escheduler cn.analysys - 1.1.0-preview + 1.1.0-SNAPSHOT escheduler-server escheduler-server diff --git a/pom.xml b/pom.xml index b4c610f4df..a97650aafb 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 cn.analysys escheduler - 1.1.0-preview + 1.1.0-SNAPSHOT pom escheduler http://maven.apache.org @@ -382,7 +382,7 @@ scm:git:https://github.com/analysys/EasyScheduler.git scm:git:https://github.com/analysys/EasyScheduler.git https://github.com/analysys/EasyScheduler.git - 1.1.0-preview + HEAD From 5b01837d500538ad1a02ad5443f1b7d88a54110f Mon Sep 17 00:00:00 2001 From: ligang Date: Mon, 8 Jul 2019 14:10:54 +0800 Subject: [PATCH 095/149] [maven-release-plugin] prepare release 1.1.0-preview --- escheduler-alert/pom.xml | 2 +- escheduler-api/pom.xml | 2 +- escheduler-common/pom.xml | 2 +- escheduler-dao/pom.xml | 2 +- escheduler-rpc/pom.xml | 2 +- escheduler-server/pom.xml | 2 +- pom.xml | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/escheduler-alert/pom.xml b/escheduler-alert/pom.xml index be8b5fbb6d..cda2488c36 100644 --- a/escheduler-alert/pom.xml +++ b/escheduler-alert/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 escheduler-alert jar diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index 56688f4024..6daff5e25f 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -3,7 +3,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 escheduler-api jar diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index 35b025b1d8..cc2bb36717 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0 escheduler-common escheduler-common diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index 52d9f7646d..57d1ad30c2 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 escheduler-dao escheduler-dao diff --git a/escheduler-rpc/pom.xml b/escheduler-rpc/pom.xml index e3d7c91a92..4687d1e462 100644 --- a/escheduler-rpc/pom.xml +++ b/escheduler-rpc/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0 4.0.0 diff --git a/escheduler-server/pom.xml b/escheduler-server/pom.xml index 7f149278c8..ac9a312734 100644 --- a/escheduler-server/pom.xml +++ b/escheduler-server/pom.xml @@ -3,7 +3,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0 escheduler-server escheduler-server diff --git a/pom.xml b/pom.xml index a97650aafb..93ac6f8ee9 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 pom escheduler http://maven.apache.org @@ -382,7 +382,7 @@ scm:git:https://github.com/analysys/EasyScheduler.git scm:git:https://github.com/analysys/EasyScheduler.git https://github.com/analysys/EasyScheduler.git - HEAD + 1.1.0-preview From f958bb32b2b6d7784e9c788fc5f26bf85be74457 Mon Sep 17 00:00:00 2001 From: ligang Date: Mon, 8 Jul 2019 14:16:05 +0800 Subject: [PATCH 096/149] [maven-release-plugin] rollback the release of 1.1.0-preview --- escheduler-alert/pom.xml | 2 +- escheduler-api/pom.xml | 2 +- escheduler-common/pom.xml | 2 +- escheduler-dao/pom.xml | 2 +- escheduler-rpc/pom.xml | 2 +- escheduler-server/pom.xml | 2 +- pom.xml | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/escheduler-alert/pom.xml b/escheduler-alert/pom.xml index cda2488c36..be8b5fbb6d 100644 --- a/escheduler-alert/pom.xml +++ b/escheduler-alert/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT escheduler-alert jar diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index 6daff5e25f..56688f4024 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -3,7 +3,7 @@ cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT escheduler-api jar diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index cc2bb36717..35b025b1d8 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0 + 1.1.0-SNAPSHOT escheduler-common escheduler-common diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index 57d1ad30c2..52d9f7646d 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT escheduler-dao escheduler-dao diff --git a/escheduler-rpc/pom.xml b/escheduler-rpc/pom.xml index 4687d1e462..e3d7c91a92 100644 --- a/escheduler-rpc/pom.xml +++ b/escheduler-rpc/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0 + 1.1.0-SNAPSHOT 4.0.0 diff --git a/escheduler-server/pom.xml b/escheduler-server/pom.xml index ac9a312734..7f149278c8 100644 --- a/escheduler-server/pom.xml +++ b/escheduler-server/pom.xml @@ -3,7 +3,7 @@ escheduler cn.analysys - 1.1.0 + 1.1.0-SNAPSHOT escheduler-server escheduler-server diff --git a/pom.xml b/pom.xml index 93ac6f8ee9..a97650aafb 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT pom escheduler http://maven.apache.org @@ -382,7 +382,7 @@ scm:git:https://github.com/analysys/EasyScheduler.git scm:git:https://github.com/analysys/EasyScheduler.git https://github.com/analysys/EasyScheduler.git - 1.1.0-preview + HEAD From 8ef2cf99d1e23f855b6e253afe558302fddb7e85 Mon Sep 17 00:00:00 2001 From: ligang Date: Mon, 8 Jul 2019 14:29:02 +0800 Subject: [PATCH 097/149] [maven-release-plugin] prepare release 1.1.0-preview --- escheduler-alert/pom.xml | 2 +- escheduler-api/pom.xml | 2 +- escheduler-common/pom.xml | 2 +- escheduler-dao/pom.xml | 2 +- escheduler-rpc/pom.xml | 2 +- escheduler-server/pom.xml | 2 +- pom.xml | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/escheduler-alert/pom.xml b/escheduler-alert/pom.xml index be8b5fbb6d..a1ed6d7e07 100644 --- a/escheduler-alert/pom.xml +++ b/escheduler-alert/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0-preview escheduler-alert jar diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index 56688f4024..2c3c8733c9 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -3,7 +3,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0-preview escheduler-api jar diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index 35b025b1d8..95943ed098 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0-preview escheduler-common escheduler-common diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index 52d9f7646d..c0ccc4cc0f 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0-preview escheduler-dao escheduler-dao diff --git a/escheduler-rpc/pom.xml b/escheduler-rpc/pom.xml index e3d7c91a92..4b6257aa35 100644 --- a/escheduler-rpc/pom.xml +++ b/escheduler-rpc/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0-preview 4.0.0 diff --git a/escheduler-server/pom.xml b/escheduler-server/pom.xml index 7f149278c8..8ca47304a9 100644 --- a/escheduler-server/pom.xml +++ b/escheduler-server/pom.xml @@ -3,7 +3,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0-preview escheduler-server escheduler-server diff --git a/pom.xml b/pom.xml index a97650aafb..b4c610f4df 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0-preview pom escheduler http://maven.apache.org @@ -382,7 +382,7 @@ scm:git:https://github.com/analysys/EasyScheduler.git scm:git:https://github.com/analysys/EasyScheduler.git https://github.com/analysys/EasyScheduler.git - HEAD + 1.1.0-preview From 57383b9771e36e1214b1d5a47237c5c3a9177a82 Mon Sep 17 00:00:00 2001 From: ligang Date: Mon, 8 Jul 2019 14:31:04 +0800 Subject: [PATCH 098/149] [maven-release-plugin] rollback the release of 1.1.0-preview --- escheduler-alert/pom.xml | 2 +- escheduler-api/pom.xml | 2 +- escheduler-common/pom.xml | 2 +- escheduler-dao/pom.xml | 2 +- escheduler-rpc/pom.xml | 2 +- escheduler-server/pom.xml | 2 +- pom.xml | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/escheduler-alert/pom.xml b/escheduler-alert/pom.xml index a1ed6d7e07..be8b5fbb6d 100644 --- a/escheduler-alert/pom.xml +++ b/escheduler-alert/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0-preview + 1.1.0-SNAPSHOT escheduler-alert jar diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index 2c3c8733c9..56688f4024 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -3,7 +3,7 @@ cn.analysys escheduler - 1.1.0-preview + 1.1.0-SNAPSHOT escheduler-api jar diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index 95943ed098..35b025b1d8 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0-preview + 1.1.0-SNAPSHOT escheduler-common escheduler-common diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index c0ccc4cc0f..52d9f7646d 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0-preview + 1.1.0-SNAPSHOT escheduler-dao escheduler-dao diff --git a/escheduler-rpc/pom.xml b/escheduler-rpc/pom.xml index 4b6257aa35..e3d7c91a92 100644 --- a/escheduler-rpc/pom.xml +++ b/escheduler-rpc/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0-preview + 1.1.0-SNAPSHOT 4.0.0 diff --git a/escheduler-server/pom.xml b/escheduler-server/pom.xml index 8ca47304a9..7f149278c8 100644 --- a/escheduler-server/pom.xml +++ b/escheduler-server/pom.xml @@ -3,7 +3,7 @@ escheduler cn.analysys - 1.1.0-preview + 1.1.0-SNAPSHOT escheduler-server escheduler-server diff --git a/pom.xml b/pom.xml index b4c610f4df..a97650aafb 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 cn.analysys escheduler - 1.1.0-preview + 1.1.0-SNAPSHOT pom escheduler http://maven.apache.org @@ -382,7 +382,7 @@ scm:git:https://github.com/analysys/EasyScheduler.git scm:git:https://github.com/analysys/EasyScheduler.git https://github.com/analysys/EasyScheduler.git - 1.1.0-preview + HEAD From 18f3eabb0ca15b4df2ab7c3b36020333d6e6d535 Mon Sep 17 00:00:00 2001 From: ligang Date: Mon, 8 Jul 2019 14:37:46 +0800 Subject: [PATCH 099/149] [maven-release-plugin] prepare release 1.1.0-preview --- escheduler-alert/pom.xml | 2 +- escheduler-api/pom.xml | 2 +- escheduler-common/pom.xml | 2 +- escheduler-dao/pom.xml | 2 +- escheduler-rpc/pom.xml | 2 +- escheduler-server/pom.xml | 2 +- pom.xml | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/escheduler-alert/pom.xml b/escheduler-alert/pom.xml index be8b5fbb6d..cda2488c36 100644 --- a/escheduler-alert/pom.xml +++ b/escheduler-alert/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 escheduler-alert jar diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index 56688f4024..6daff5e25f 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -3,7 +3,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 escheduler-api jar diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index 35b025b1d8..cc2bb36717 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0 escheduler-common escheduler-common diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index 52d9f7646d..57d1ad30c2 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 escheduler-dao escheduler-dao diff --git a/escheduler-rpc/pom.xml b/escheduler-rpc/pom.xml index e3d7c91a92..4687d1e462 100644 --- a/escheduler-rpc/pom.xml +++ b/escheduler-rpc/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0 4.0.0 diff --git a/escheduler-server/pom.xml b/escheduler-server/pom.xml index 7f149278c8..ac9a312734 100644 --- a/escheduler-server/pom.xml +++ b/escheduler-server/pom.xml @@ -3,7 +3,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0 escheduler-server escheduler-server diff --git a/pom.xml b/pom.xml index a97650aafb..93ac6f8ee9 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 pom escheduler http://maven.apache.org @@ -382,7 +382,7 @@ scm:git:https://github.com/analysys/EasyScheduler.git scm:git:https://github.com/analysys/EasyScheduler.git https://github.com/analysys/EasyScheduler.git - HEAD + 1.1.0-preview From 3916f463828f979653250cbb3a0cdc2ea583e3a4 Mon Sep 17 00:00:00 2001 From: ligang Date: Mon, 8 Jul 2019 14:38:19 +0800 Subject: [PATCH 100/149] [maven-release-plugin] prepare for next development iteration --- escheduler-alert/pom.xml | 2 +- escheduler-api/pom.xml | 2 +- escheduler-common/pom.xml | 2 +- escheduler-dao/pom.xml | 2 +- escheduler-rpc/pom.xml | 2 +- escheduler-server/pom.xml | 2 +- pom.xml | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/escheduler-alert/pom.xml b/escheduler-alert/pom.xml index cda2488c36..be8b5fbb6d 100644 --- a/escheduler-alert/pom.xml +++ b/escheduler-alert/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT escheduler-alert jar diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index 6daff5e25f..56688f4024 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -3,7 +3,7 @@ cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT escheduler-api jar diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index cc2bb36717..35b025b1d8 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0 + 1.1.0-SNAPSHOT escheduler-common escheduler-common diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index 57d1ad30c2..52d9f7646d 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT escheduler-dao escheduler-dao diff --git a/escheduler-rpc/pom.xml b/escheduler-rpc/pom.xml index 4687d1e462..e3d7c91a92 100644 --- a/escheduler-rpc/pom.xml +++ b/escheduler-rpc/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0 + 1.1.0-SNAPSHOT 4.0.0 diff --git a/escheduler-server/pom.xml b/escheduler-server/pom.xml index ac9a312734..7f149278c8 100644 --- a/escheduler-server/pom.xml +++ b/escheduler-server/pom.xml @@ -3,7 +3,7 @@ escheduler cn.analysys - 1.1.0 + 1.1.0-SNAPSHOT escheduler-server escheduler-server diff --git a/pom.xml b/pom.xml index 93ac6f8ee9..a97650aafb 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT pom escheduler http://maven.apache.org @@ -382,7 +382,7 @@ scm:git:https://github.com/analysys/EasyScheduler.git scm:git:https://github.com/analysys/EasyScheduler.git https://github.com/analysys/EasyScheduler.git - 1.1.0-preview + HEAD From fb37c9bf7b019900e28d2a21fa8bb5878d883d3c Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Mon, 8 Jul 2019 19:04:03 +0800 Subject: [PATCH 101/149] python task env variables and task submit update --- .../worker/task/PythonCommandExecutor.java | 11 +++---- .../server/worker/task/python/PythonTask.java | 29 ++++++++++--------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java index 94caffe596..559f827cf5 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java @@ -18,6 +18,7 @@ package cn.escheduler.server.worker.task; import cn.escheduler.common.Constants; import cn.escheduler.common.utils.FileUtils; +import cn.escheduler.common.utils.PropertyUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,11 +72,11 @@ public class PythonCommandExecutor extends AbstractCommandExecutor { if (!Files.exists(Paths.get(commandFile))) { logger.info("generate command file:{}", commandFile); - StringBuilder sb = new StringBuilder(200); + StringBuilder sb = new StringBuilder(); sb.append("#-*- encoding=utf8 -*-\n"); sb.append("\n\n"); - sb.append(String.format("import py_%s_node\n",taskAppId)); + sb.append(execCommand); logger.info(sb.toString()); // write data to file @@ -86,13 +87,13 @@ public class PythonCommandExecutor extends AbstractCommandExecutor { @Override protected String commandType() { - String envPath = System.getProperty("user.dir") + Constants.SINGLE_SLASH + "conf"+ - Constants.SINGLE_SLASH +"env" + Constants.SINGLE_SLASH + Constants.ESCHEDULER_ENV_SH; + String envPath = PropertyUtils.getString(Constants.ESCHEDULER_ENV_PATH); + String pythonHome = getPythonHome(envPath); if (StringUtils.isEmpty(pythonHome)){ return PYTHON; } - return pythonHome; + return pythonHome + Constants.SINGLE_SLASH +PYTHON; } @Override diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/python/PythonTask.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/python/PythonTask.java index 49d754404a..c446215a38 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/python/PythonTask.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/python/PythonTask.java @@ -112,14 +112,14 @@ public class PythonTask extends AbstractTask { */ private String buildCommand() throws Exception { // generate scripts - String fileName = String.format("%s/py_%s_node.py", taskDir, taskProps.getTaskAppId()); - Path path = new File(fileName).toPath(); +// String fileName = String.format("%s/py_%s_node.py", taskDir, taskProps.getTaskAppId()); +// Path path = new File(fileName).toPath(); - if (Files.exists(path)) { - return fileName; - } +// if (Files.exists(path)) { +// return fileName; +// } String rawScript = pythonParameters.getRawScript().replaceAll("\\r\\n", "\n"); @@ -140,19 +140,20 @@ public class PythonTask extends AbstractTask { } - pythonParameters.setRawScript(rawScript); +// pythonParameters.setRawScript(rawScript); logger.info("raw script : {}", pythonParameters.getRawScript()); logger.info("task dir : {}", taskDir); - Set perms = PosixFilePermissions.fromString("rwxr-xr-x"); - FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); - - Files.createFile(path, attr); - - Files.write(path, pythonParameters.getRawScript().getBytes(), StandardOpenOption.APPEND); - - return fileName; +// Set perms = PosixFilePermissions.fromString("rwxr-xr-x"); +// FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); +// +// Files.createFile(path, attr); +// +// Files.write(path, pythonParameters.getRawScript().getBytes(), StandardOpenOption.APPEND); +// +// return fileName; + return rawScript; } @Override From 92c21362750cf7d65714b45a2d5fb1e95ca56d63 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Mon, 8 Jul 2019 19:34:21 +0800 Subject: [PATCH 102/149] python home update --- .../cn/escheduler/server/worker/task/PythonCommandExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java index 559f827cf5..e1df0b71a8 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/PythonCommandExecutor.java @@ -93,7 +93,7 @@ public class PythonCommandExecutor extends AbstractCommandExecutor { if (StringUtils.isEmpty(pythonHome)){ return PYTHON; } - return pythonHome + Constants.SINGLE_SLASH +PYTHON; + return pythonHome; } @Override From 64b173f47b7276c785a7bbb5ad990645319667f7 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Mon, 8 Jul 2019 20:33:40 +0800 Subject: [PATCH 103/149] The version will be updated automatically, so there is no need to add it here --- sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql b/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql index 4ddd9ddcdc..e69de29bb2 100644 --- a/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql +++ b/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql @@ -1 +0,0 @@ -INSERT INTO `t_escheduler_version` (`version`) VALUES ('1.1.0'); \ No newline at end of file From 1ab0a02a818fa3146786c2e54481c1ad4484d38a Mon Sep 17 00:00:00 2001 From: lenboo Date: Mon, 8 Jul 2019 21:01:25 +0800 Subject: [PATCH 104/149] update worker task queue --- .../common/queue/TaskQueueZkImpl.java | 26 ++++++++++++++++--- .../common/queue/TaskQueueImplTest.java | 14 +++++----- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java b/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java index e6c9df2833..eed7fb1ac5 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java @@ -151,7 +151,27 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue { int size = list.size(); - Set taskTreeSet = new TreeSet<>(); + Set taskTreeSet = new TreeSet<>(new Comparator() { + @Override + public int compare(String o1, String o2) { + + String s1 = o1; + String s2 = o2; + String[] s1Array = s1.split(Constants.UNDERLINE); + if(s1Array.length>4){ + // warning: if this length > 5, need to be changed + s1 = s1.substring(0, s1.lastIndexOf(Constants.UNDERLINE) ); + } + + String[] s2Array = s2.split(Constants.UNDERLINE); + if(s2Array.length>4){ + // warning: if this length > 5, need to be changed + s2 = s2.substring(0, s2.lastIndexOf(Constants.UNDERLINE) ); + } + + return s1.compareTo(s2); + } + }); for (int i = 0; i < size; i++) { @@ -173,8 +193,8 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue { continue; } } + formatTask += Constants.UNDERLINE + taskDetailArrs[4]; } - taskTreeSet.add(formatTask); } @@ -229,7 +249,7 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue { int taskId = Integer.parseInt(taskArray[3]); StringBuilder sb = new StringBuilder(50); - String destTask = String.format("%s_%s_%s_%s", taskArray[0], processInstanceId, taskArray[3], taskId); + String destTask = String.format("%s_%s_%s_%s", taskArray[0], processInstanceId, taskArray[2], taskId); sb.append(destTask); diff --git a/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java b/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java index 72a6e46200..21d2f5858e 100644 --- a/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java +++ b/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java @@ -17,6 +17,8 @@ package cn.escheduler.common.queue; import cn.escheduler.common.Constants; +import cn.escheduler.common.utils.IpUtils; +import cn.escheduler.common.utils.OSUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -58,31 +60,31 @@ public class TaskQueueImplTest { @Test public void testAdd(){ + //add - tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1_1_1_1_2130706433,3232236775"); + tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1_0_1_1_-1"); tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"0_1_1_1_2130706433,3232236775"); - tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1_1_0_1_2130706433,3232236775"); + tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1_1_0_1_2130706433,3232236775,"+IpUtils.ipToLong(OSUtils.getHost())); tasksQueue.add(Constants.SCHEDULER_TASKS_QUEUE,"1_2_1_1_2130706433,3232236775"); List tasks = tasksQueue.poll(Constants.SCHEDULER_TASKS_QUEUE, 1); - if(tasks.size() < 0){ + if(tasks.size() <= 0){ return; } //pop String node1 = tasks.get(0); - assertEquals(node1,"0_0000000001_1_0000000001"); + assertEquals(node1,"1_0_1_1_-1"); tasks = tasksQueue.poll(Constants.SCHEDULER_TASKS_QUEUE, 1); - if(tasks.size() < 0){ + if(tasks.size() <= 0){ return; } String node2 = tasks.get(0); - assertEquals(node2,"0_0000000001_1_0000000001"); } From 401467d85a7c0176fd28d89667ccbab109745102 Mon Sep 17 00:00:00 2001 From: huyuanming Date: Tue, 9 Jul 2019 10:21:26 +0800 Subject: [PATCH 105/149] remove default tenant --- escheduler-ui/src/js/conf/home/store/security/actions.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/escheduler-ui/src/js/conf/home/store/security/actions.js b/escheduler-ui/src/js/conf/home/store/security/actions.js index ff96adccf9..ac3f82d81d 100644 --- a/escheduler-ui/src/js/conf/home/store/security/actions.js +++ b/escheduler-ui/src/js/conf/home/store/security/actions.js @@ -241,10 +241,10 @@ export default { return new Promise((resolve, reject) => { io.get(`tenant/list`, payload, res => { let list=res.data - list.unshift({ - id: -1, - tenantName: 'Default' - }) + // list.unshift({ + // id: -1, + // tenantName: 'Default' + // }) state.tenantAllList = list resolve(list) }).catch(e => { From 9d1cc24e0520811b4bc1bdbf2b2ed1aff0c05475 Mon Sep 17 00:00:00 2001 From: huyuanming Date: Tue, 9 Jul 2019 10:38:33 +0800 Subject: [PATCH 106/149] remove default tenant --- .../pages/security/pages/users/_source/createUser.vue | 5 ++++- escheduler-ui/src/js/conf/home/store/security/actions.js | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue b/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue index 574a995ec4..f8f728a0df 100644 --- a/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue +++ b/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue @@ -184,7 +184,10 @@ _getTenantList () { return new Promise((resolve, reject) => { this.store.dispatch('security/getTenantList').then(res => { - this.tenantList = _.map(res, v => { + let arr = _.filter(res, (o) => { + return o.id !== -1 + }) + this.tenantList = _.map(arr, v => { return { id: v.id, code: v.tenantName diff --git a/escheduler-ui/src/js/conf/home/store/security/actions.js b/escheduler-ui/src/js/conf/home/store/security/actions.js index ac3f82d81d..ff96adccf9 100644 --- a/escheduler-ui/src/js/conf/home/store/security/actions.js +++ b/escheduler-ui/src/js/conf/home/store/security/actions.js @@ -241,10 +241,10 @@ export default { return new Promise((resolve, reject) => { io.get(`tenant/list`, payload, res => { let list=res.data - // list.unshift({ - // id: -1, - // tenantName: 'Default' - // }) + list.unshift({ + id: -1, + tenantName: 'Default' + }) state.tenantAllList = list resolve(list) }).catch(e => { From c6a151fdf0233d01d70d00e3863ec96cce4c7680 Mon Sep 17 00:00:00 2001 From: huyuanming Date: Tue, 9 Jul 2019 14:02:57 +0800 Subject: [PATCH 107/149] regEmail --- .../home/pages/security/pages/users/_source/createUser.vue | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue b/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue index f8f728a0df..378f410d38 100644 --- a/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue +++ b/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue @@ -131,7 +131,8 @@ } }, _verification () { - let regEmail = /^([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+@([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+\.[a-zA-Z]{2,3}$/ // eslint-disable-line + let regEmail = /^([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+@([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+\.[a-zA-Z]{2,}$/ // eslint-disable-line + // Mobile phone number regular let regPhone = /^1(3|4|5|6|7|8)\d{9}$/; // eslint-disable-line From 68f53bdfa4078534685614c2e9b65fdb525cc5ac Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 9 Jul 2019 14:19:50 +0800 Subject: [PATCH 108/149] remove resolve because resolve is not defined --- .../projects/pages/definition/pages/list/_source/timing.vue | 5 ----- 1 file changed, 5 deletions(-) diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue index a61f0634d9..42bb7905a0 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue @@ -254,11 +254,6 @@ this.store.dispatch(api, searchParams).then(res => { this.previewTimes = res - if (this.previewTimes.length) { - resolve() - } else { - reject(new Error(0)) - } }) } }, From 758da8b9a4dc11de50521463ca4b4c6517a92f79 Mon Sep 17 00:00:00 2001 From: lenboo Date: Tue, 9 Jul 2019 14:28:01 +0800 Subject: [PATCH 109/149] update --- .../api/controller/MonitorController.java | 4 +- .../api/service/MonitorService.java | 41 ++++++++- ...onitorUtils.java => ZookeeperMonitor.java} | 40 ++++++++- .../api/utils/ZookeeperMonitorUtilsTest.java | 34 ++++++++ .../common/queue/TaskQueueZkImpl.java | 10 --- .../common/zk/AbstractZKClient.java | 87 +++++++++++++++++-- .../common/queue/TaskQueueImplTest.java | 2 - .../java/cn/escheduler/server/ResInfo.java | 25 ++++++ .../escheduler/server/zk/ZKMasterClient.java | 39 --------- .../escheduler/server/zk/ZKWorkerClient.java | 10 --- .../server/zk/ZKWorkerClientTest.java | 5 +- 11 files changed, 223 insertions(+), 74 deletions(-) rename escheduler-api/src/main/java/cn/escheduler/api/utils/{ZookeeperMonitorUtils.java => ZookeeperMonitor.java} (64%) create mode 100644 escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java diff --git a/escheduler-api/src/main/java/cn/escheduler/api/controller/MonitorController.java b/escheduler-api/src/main/java/cn/escheduler/api/controller/MonitorController.java index 666126c0c9..cba39d5403 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/controller/MonitorController.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/controller/MonitorController.java @@ -66,7 +66,7 @@ public class MonitorController extends BaseController{ logger.info("login user: {}, query all master", loginUser.getUserName()); try{ logger.info("list master, user:{}", loginUser.getUserName()); - Map result = serverService.queryMaster(loginUser); + Map result = monitorService.queryMaster(loginUser); return returnDataList(result); }catch (Exception e){ logger.error(LIST_MASTERS_ERROR.getMsg(),e); @@ -86,7 +86,7 @@ public class MonitorController extends BaseController{ public Result listWorker(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser) { logger.info("login user: {}, query all workers", loginUser.getUserName()); try{ - Map result = serverService.queryWorker(loginUser); + Map result = monitorService.queryWorker(loginUser); return returnDataList(result); }catch (Exception e){ logger.error(LIST_WORKERS_ERROR.getMsg(),e); diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/MonitorService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/MonitorService.java index 4708b00175..08e8bf576e 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/MonitorService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/MonitorService.java @@ -18,13 +18,16 @@ package cn.escheduler.api.service; import cn.escheduler.api.enums.Status; import cn.escheduler.api.utils.Constants; -import cn.escheduler.api.utils.ZookeeperMonitorUtils; +import cn.escheduler.api.utils.ZookeeperMonitor; import cn.escheduler.dao.MonitorDBDao; +import cn.escheduler.dao.model.MasterServer; import cn.escheduler.dao.model.MonitorRecord; import cn.escheduler.dao.model.User; import cn.escheduler.dao.model.ZookeeperRecord; +import org.apache.hadoop.mapred.Master; import org.springframework.stereotype.Service; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -52,6 +55,22 @@ public class MonitorService extends BaseService{ } + /** + * query master list + * + * @param loginUser + * @return + */ + public Map queryMaster(User loginUser) { + + Map result = new HashMap<>(5); + + List masterServers = new ZookeeperMonitor().getMasterServers(); + result.put(Constants.DATA_LIST, masterServers); + putMsg(result,Status.SUCCESS); + + return result; + } /** * query zookeeper state @@ -61,7 +80,7 @@ public class MonitorService extends BaseService{ public Map queryZookeeperState(User loginUser) { Map result = new HashMap<>(5); - List zookeeperRecordList = ZookeeperMonitorUtils.zookeeperInfoList(); + List zookeeperRecordList = ZookeeperMonitor.zookeeperInfoList(); result.put(Constants.DATA_LIST, zookeeperRecordList); putMsg(result, Status.SUCCESS); @@ -69,4 +88,22 @@ public class MonitorService extends BaseService{ return result; } + + + /** + * query master list + * + * @param loginUser + * @return + */ + public Map queryWorker(User loginUser) { + + Map result = new HashMap<>(5); + + List workerServers = new ZookeeperMonitor().getWorkerServers(); + result.put(Constants.DATA_LIST, workerServers); + putMsg(result,Status.SUCCESS); + + return result; + } } diff --git a/escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitorUtils.java b/escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitor.java similarity index 64% rename from escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitorUtils.java rename to escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitor.java index 6612a355fd..a70e9dc4a8 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitorUtils.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitor.java @@ -1,22 +1,26 @@ package cn.escheduler.api.utils; import cn.escheduler.common.zk.AbstractZKClient; +import cn.escheduler.dao.model.MasterServer; import cn.escheduler.dao.model.ZookeeperRecord; +import cn.escheduler.server.ResInfo; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import sun.jvm.hotspot.opto.MachSafePointNode; import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.Map; /** * monitor zookeeper info */ -public class ZookeeperMonitorUtils { +public class ZookeeperMonitor extends AbstractZKClient{ - private static final Logger LOG = LoggerFactory.getLogger(ZookeeperMonitorUtils.class); + private static final Logger LOG = LoggerFactory.getLogger(ZookeeperMonitor.class); private static final String zookeeperList = AbstractZKClient.getZookeeperQuorum(); /** @@ -33,6 +37,38 @@ public class ZookeeperMonitorUtils { return null; } + /** + * get server list. + * @param isMaster + * @return + */ + public List getServers(boolean isMaster){ + List masterServers = new ArrayList<>(); + Map masterMap = getServerList(isMaster); + String parentPath = isMaster ? getMasterZNodeParentPath() : getWorkerZNodeParentPath(); + for(String path : masterMap.keySet()){ + MasterServer masterServer = ResInfo.parseHeartbeatForZKInfo(masterMap.get(path)); + masterServer.setZkDirectory( parentPath + "/"+ path); + masterServers.add(masterServer); + } + return masterServers; + } + + /** + * get master servers + * @return + */ + public List getMasterServers(){ + return getServers(true); + } + + /** + * master construct is the same with worker, use the master instead + * @return + */ + public List getWorkerServers(){ + return getServers(false); + } private static List zookeeperInfoList(String zookeeperServers) { diff --git a/escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java b/escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java new file mode 100644 index 0000000000..05c2ba9fe0 --- /dev/null +++ b/escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java @@ -0,0 +1,34 @@ +package cn.escheduler.api.utils; + +import cn.escheduler.dao.model.MasterServer; +import org.junit.Assert; +import org.junit.Test; + +import javax.crypto.MacSpi; + +import java.util.List; + +import static org.junit.Assert.*; + +public class ZookeeperMonitorUtilsTest { + + + @Test + public void testGetMasterLsit(){ + + ZookeeperMonitor zookeeperMonitor = new ZookeeperMonitor(); + + + List masterServerList = zookeeperMonitor.getMasterServers(); + + List workerServerList = zookeeperMonitor.getWorkerServers(); + + System.out.println("master:" + masterServerList); + System.out.println("worker:" + workerServerList); + Assert.assertEquals(masterServerList.size(), 1); + Assert.assertEquals(workerServerList.size(), 1); + + + } + +} \ No newline at end of file diff --git a/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java b/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java index eed7fb1ac5..73f7499fa7 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java @@ -417,16 +417,6 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue { } } - - /** - * get zookeeper client of CuratorFramework - * @return - */ - public CuratorFramework getZkClient() { - return zkClient; - } - - /** * Get the task queue path * @param key task queue name diff --git a/escheduler-common/src/main/java/cn/escheduler/common/zk/AbstractZKClient.java b/escheduler-common/src/main/java/cn/escheduler/common/zk/AbstractZKClient.java index e2f064be13..f7134f5e7a 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/zk/AbstractZKClient.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/zk/AbstractZKClient.java @@ -30,13 +30,12 @@ import org.apache.curator.framework.imps.CuratorFrameworkState; import org.apache.curator.framework.state.ConnectionState; import org.apache.curator.framework.state.ConnectionStateListener; import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos; import org.apache.zookeeper.CreateMode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; +import java.util.*; import static cn.escheduler.common.Constants.*; @@ -213,9 +212,9 @@ public abstract class AbstractZKClient { protected void initSystemZNode(){ try { // read master node parent path from conf - masterZNodeParentPath = conf.getString(Constants.ZOOKEEPER_ESCHEDULER_MASTERS); + masterZNodeParentPath = getMasterZNodeParentPath(); // read worker node parent path from conf - workerZNodeParentPath = conf.getString(Constants.ZOOKEEPER_ESCHEDULER_WORKERS); + workerZNodeParentPath = getWorkerZNodeParentPath(); // read server node parent path from conf deadServerZNodeParentPath = conf.getString(ZOOKEEPER_ESCHEDULER_DEAD_SERVERS); @@ -243,6 +242,7 @@ public abstract class AbstractZKClient { } } + public void removeDeadServerByHost(String host, String serverType) throws Exception { List deadServers = zkClient.getChildren().forPath(deadServerZNodeParentPath); for(String serverPath : deadServers){ @@ -291,6 +291,8 @@ public abstract class AbstractZKClient { } + + /** * for stop server * @param serverStoppable @@ -340,6 +342,81 @@ public abstract class AbstractZKClient { return sb.toString(); } + /** + * get master server list map. + * result : {host : resource info} + * @return + */ + public Map getServerList(boolean isMaster ){ + + Map masterMap = new HashMap<>(); + try { + String path = isMaster ? getMasterZNodeParentPath() : getWorkerZNodeParentPath(); + List serverList = getZkClient().getChildren().forPath(path); + for(String server : serverList){ + byte[] bytes = getZkClient().getData().forPath(path + "/" + server); + masterMap.putIfAbsent(server, new String(bytes)); + } + } catch (Exception e) { + e.printStackTrace(); + } + + return masterMap; + } + + /** + * get zkclient + * @return + */ + public CuratorFramework getZkClient() { + return zkClient; + } + + /** + * get worker node parent path + * @return + */ + protected String getWorkerZNodeParentPath(){return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_WORKERS);}; + + /** + * get master node parent path + * @return + */ + protected String getMasterZNodeParentPath(){return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_MASTERS);} + + /** + * get master lock path + * @return + */ + public String getMasterLockPath(){ + return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_MASTERS); + } + + /** + * get master start up lock path + * @return + */ + public String getMasterStartUpLockPath(){ + return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_STARTUP_MASTERS); + } + + /** + * get master failover lock path + * @return + */ + public String getMasterFailoverLockPath(){ + return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_MASTERS); + } + + /** + * get worker failover lock path + * @return + */ + public String getWorkerFailoverLockPath(){ + return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_WORKERS); + } + + @Override public String toString() { return "AbstractZKClient{" + diff --git a/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java b/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java index 21d2f5858e..16562d54aa 100644 --- a/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java +++ b/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java @@ -84,8 +84,6 @@ public class TaskQueueImplTest { return; } - String node2 = tasks.get(0); - } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/ResInfo.java b/escheduler-server/src/main/java/cn/escheduler/server/ResInfo.java index 81f126998c..581a0af482 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/ResInfo.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/ResInfo.java @@ -17,8 +17,12 @@ package cn.escheduler.server; import cn.escheduler.common.Constants; +import cn.escheduler.common.utils.DateUtils; import cn.escheduler.common.utils.JSONUtils; import cn.escheduler.common.utils.OSUtils; +import cn.escheduler.dao.model.MasterServer; + +import java.util.Date; /** * heartbeat for ZK reigster res info @@ -119,4 +123,25 @@ public class ResInfo { + lastHeartbeatTime; } + /** + * parse heartbeat info for zk + * @param heartBeatInfo + * @return + */ + public static MasterServer parseHeartbeatForZKInfo(String heartBeatInfo){ + MasterServer masterServer = null; + String[] masterArray = heartBeatInfo.split(Constants.COMMA); + if(masterArray.length != 6){ + return masterServer; + + } + masterServer = new MasterServer(); + masterServer.setHost(masterArray[0]); + masterServer.setPort(Integer.parseInt(masterArray[1])); + masterServer.setResInfo(getResInfoJson(Double.parseDouble(masterArray[2]), Double.parseDouble(masterArray[3]))); + masterServer.setCreateTime(DateUtils.stringToDate(masterArray[4])); + masterServer.setLastHeartbeatTime(DateUtils.stringToDate(masterArray[5])); + return masterServer; + } + } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java index 85c805e2fc..0bc3247bf4 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java @@ -448,45 +448,6 @@ public class ZKMasterClient extends AbstractZKClient { } - /** - * get master lock path - * @return - */ - public String getMasterLockPath(){ - return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_MASTERS); - } - - /** - * get master start up lock path - * @return - */ - public String getMasterStartUpLockPath(){ - return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_STARTUP_MASTERS); - } - - /** - * get master failover lock path - * @return - */ - public String getMasterFailoverLockPath(){ - return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_MASTERS); - } - - /** - * get worker failover lock path - * @return - */ - public String getWorkerFailoverLockPath(){ - return conf.getString(Constants.ZOOKEEPER_ESCHEDULER_LOCK_FAILOVER_WORKERS); - } - - /** - * get zkclient - * @return - */ - public CuratorFramework getZkClient() { - return zkClient; - } /** diff --git a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKWorkerClient.java b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKWorkerClient.java index 7cf899d832..4f03d0a41c 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKWorkerClient.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKWorkerClient.java @@ -264,16 +264,6 @@ public class ZKWorkerClient extends AbstractZKClient { return workerZNode; } - - /** - * get zkclient - * @return - */ - public CuratorFramework getZkClient() { - return zkClient; - } - - /** * get worker lock path * @return diff --git a/escheduler-server/src/test/java/cn/escheduler/server/zk/ZKWorkerClientTest.java b/escheduler-server/src/test/java/cn/escheduler/server/zk/ZKWorkerClientTest.java index c8aa0930a2..c8e4b93027 100644 --- a/escheduler-server/src/test/java/cn/escheduler/server/zk/ZKWorkerClientTest.java +++ b/escheduler-server/src/test/java/cn/escheduler/server/zk/ZKWorkerClientTest.java @@ -1,6 +1,7 @@ package cn.escheduler.server.zk; import cn.escheduler.common.Constants; +import cn.escheduler.common.zk.AbstractZKClient; import org.junit.Test; import java.util.Arrays; @@ -17,8 +18,8 @@ public class ZKWorkerClientTest { public void getZKWorkerClient() throws Exception { - ZKWorkerClient zkWorkerClient = ZKWorkerClient.getZKWorkerClient(); - zkWorkerClient.removeDeadServerByHost("127.0.0.1", Constants.WORKER_PREFIX); +// ZKWorkerClient zkWorkerClient = ZKWorkerClient.getZKWorkerClient(); +// zkWorkerClient.removeDeadServerByHost("127.0.0.1", Constants.WORKER_PREFIX); } From beaa6c6aa076b4414a1dd3fc6ab7e81907a80f79 Mon Sep 17 00:00:00 2001 From: lenboo Date: Tue, 9 Jul 2019 14:37:42 +0800 Subject: [PATCH 110/149] update --- .../main/java/cn/escheduler/api/utils/ZookeeperMonitor.java | 1 - .../cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java | 3 --- 2 files changed, 4 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitor.java b/escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitor.java index a70e9dc4a8..0f44b5f7db 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitor.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/utils/ZookeeperMonitor.java @@ -7,7 +7,6 @@ import cn.escheduler.server.ResInfo; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import sun.jvm.hotspot.opto.MachSafePointNode; import java.util.ArrayList; import java.util.Date; diff --git a/escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java b/escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java index 05c2ba9fe0..adaa78f7ba 100644 --- a/escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java +++ b/escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java @@ -4,12 +4,9 @@ import cn.escheduler.dao.model.MasterServer; import org.junit.Assert; import org.junit.Test; -import javax.crypto.MacSpi; import java.util.List; -import static org.junit.Assert.*; - public class ZookeeperMonitorUtilsTest { From 5a168d81d2c5d7c260dc2b9b031ea0bcf92fb792 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Tue, 9 Jul 2019 14:44:08 +0800 Subject: [PATCH 111/149] install.sh and log4j-to-slf4j-2.11.2.jar conflict solve --- escheduler-api/pom.xml | 4 ++++ escheduler-dao/pom.xml | 4 ++++ install.sh | 1 + 3 files changed, 9 insertions(+) diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index 56688f4024..b0d92d54a8 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -47,6 +47,10 @@ org.springframework.boot spring-boot-starter-tomcat + + log4j-to-slf4j + org.apache.logging.log4j + diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index 52d9f7646d..4d8fb6912e 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -37,6 +37,10 @@ org.apache.tomcat tomcat-jdbc + + log4j-to-slf4j + org.apache.logging.log4j + diff --git a/install.sh b/install.sh index d58be482fd..33acdf5eec 100644 --- a/install.sh +++ b/install.sh @@ -144,6 +144,7 @@ singleYarnIp="ark1" hdfsPath="/escheduler" # 拥有在hdfs根路径/下创建目录权限的用户 +# 注意：如果开启了kerberos，则直接hdfsRootUser=，就可以 hdfsRootUser="hdfs" # common 配置 From b3984265e54324010b29d164e433802043018705 Mon Sep 17 00:00:00 2001 From: huyuanming Date: Tue, 9 Jul 2019 15:00:44 +0800 Subject: [PATCH 112/149] text --- .../src/js/conf/home/pages/monitor/pages/servers/statistics.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/escheduler-ui/src/js/conf/home/pages/monitor/pages/servers/statistics.vue b/escheduler-ui/src/js/conf/home/pages/monitor/pages/servers/statistics.vue index e6302fdd1f..a552e4cc00 100644 --- a/escheduler-ui/src/js/conf/home/pages/monitor/pages/servers/statistics.vue +++ b/escheduler-ui/src/js/conf/home/pages/monitor/pages/servers/statistics.vue @@ -16,7 +16,7 @@
- {{$t('failure command number')}}} + {{$t('failure command number')}}
{{commandCountData.errorCount}} From 964ea73e98be21463e83baf1bdbfabfd81ae7991 Mon Sep 17 00:00:00 2001 From: ligang Date: Tue, 9 Jul 2019 15:14:00 +0800 Subject: [PATCH 113/149] [maven-release-plugin] prepare release 1.1.0-preview --- escheduler-alert/pom.xml | 2 +- escheduler-api/pom.xml | 2 +- escheduler-common/pom.xml | 2 +- escheduler-dao/pom.xml | 2 +- escheduler-rpc/pom.xml | 2 +- escheduler-server/pom.xml | 2 +- pom.xml | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/escheduler-alert/pom.xml b/escheduler-alert/pom.xml index be8b5fbb6d..cda2488c36 100644 --- a/escheduler-alert/pom.xml +++ b/escheduler-alert/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 escheduler-alert jar diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index b0d92d54a8..b3d9fc53fd 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -3,7 +3,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 escheduler-api jar diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index 35b025b1d8..cc2bb36717 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0 escheduler-common escheduler-common diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index 4d8fb6912e..3eb6dbf3e0 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 escheduler-dao escheduler-dao diff --git a/escheduler-rpc/pom.xml b/escheduler-rpc/pom.xml index e3d7c91a92..4687d1e462 100644 --- a/escheduler-rpc/pom.xml +++ b/escheduler-rpc/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0 4.0.0 diff --git a/escheduler-server/pom.xml b/escheduler-server/pom.xml index 7f149278c8..ac9a312734 100644 --- a/escheduler-server/pom.xml +++ b/escheduler-server/pom.xml @@ -3,7 +3,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0 escheduler-server escheduler-server diff --git a/pom.xml b/pom.xml index a97650aafb..93ac6f8ee9 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 pom escheduler http://maven.apache.org @@ -382,7 +382,7 @@ scm:git:https://github.com/analysys/EasyScheduler.git scm:git:https://github.com/analysys/EasyScheduler.git https://github.com/analysys/EasyScheduler.git - HEAD + 1.1.0-preview From e2f73b715bfd639546bfa585ee3386908f056392 Mon Sep 17 00:00:00 2001 From: ligang Date: Tue, 9 Jul 2019 15:14:27 +0800 Subject: [PATCH 114/149] [maven-release-plugin] prepare for next development iteration --- escheduler-alert/pom.xml | 2 +- escheduler-api/pom.xml | 2 +- escheduler-common/pom.xml | 2 +- escheduler-dao/pom.xml | 2 +- escheduler-rpc/pom.xml | 2 +- escheduler-server/pom.xml | 2 +- pom.xml | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/escheduler-alert/pom.xml b/escheduler-alert/pom.xml index cda2488c36..be8b5fbb6d 100644 --- a/escheduler-alert/pom.xml +++ b/escheduler-alert/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT escheduler-alert jar diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index b3d9fc53fd..b0d92d54a8 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -3,7 +3,7 @@ cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT escheduler-api jar diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index cc2bb36717..35b025b1d8 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0 + 1.1.0-SNAPSHOT escheduler-common escheduler-common diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index 3eb6dbf3e0..4d8fb6912e 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT escheduler-dao escheduler-dao diff --git a/escheduler-rpc/pom.xml b/escheduler-rpc/pom.xml index 4687d1e462..e3d7c91a92 100644 --- a/escheduler-rpc/pom.xml +++ b/escheduler-rpc/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0 + 1.1.0-SNAPSHOT 4.0.0 diff --git a/escheduler-server/pom.xml b/escheduler-server/pom.xml index ac9a312734..7f149278c8 100644 --- a/escheduler-server/pom.xml +++ b/escheduler-server/pom.xml @@ -3,7 +3,7 @@ escheduler cn.analysys - 1.1.0 + 1.1.0-SNAPSHOT escheduler-server escheduler-server diff --git a/pom.xml b/pom.xml index 93ac6f8ee9..a97650aafb 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT pom escheduler http://maven.apache.org @@ -382,7 +382,7 @@ scm:git:https://github.com/analysys/EasyScheduler.git scm:git:https://github.com/analysys/EasyScheduler.git https://github.com/analysys/EasyScheduler.git - 1.1.0-preview + HEAD From 92e182e6f825f8d16ed2a5db7a84c460628841c8 Mon Sep 17 00:00:00 2001 From: lidongdai Date: Tue, 9 Jul 2019 15:23:29 +0800 Subject: [PATCH 115/149] optimize datasource connection params safety --- .../api/service/DataSourceService.java | 47 ++++++++++++++----- .../java/cn/escheduler/common/Constants.java | 2 + 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java index e7f90c5ce5..5331e03418 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java @@ -17,16 +17,14 @@ package cn.escheduler.api.service; import cn.escheduler.api.enums.Status; -import cn.escheduler.api.utils.CheckUtils; import cn.escheduler.api.utils.Constants; import cn.escheduler.api.utils.PageInfo; import cn.escheduler.api.utils.Result; import cn.escheduler.common.enums.DbType; -import cn.escheduler.common.enums.ResUploadType; import cn.escheduler.common.enums.UserType; import cn.escheduler.common.job.db.*; import cn.escheduler.common.utils.CommonUtils; -import cn.escheduler.common.utils.PropertyUtils; +import cn.escheduler.common.utils.JSONUtils; import cn.escheduler.dao.mapper.DataSourceMapper; import cn.escheduler.dao.mapper.DatasourceUserMapper; import cn.escheduler.dao.mapper.ProjectMapper; @@ -48,7 +46,6 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.util.*; -import static cn.escheduler.common.utils.PropertyUtils.getBoolean; import static cn.escheduler.common.utils.PropertyUtils.getString; /** @@ -67,7 +64,7 @@ public class DataSourceService extends BaseService{ public static final String PRINCIPAL = "principal"; public static final String DATABASE = "database"; public static final String USER_NAME = "userName"; - public static final String PASSWORD = "password"; + public static final String PASSWORD = cn.escheduler.common.Constants.PASSWORD; public static final String OTHER = "other"; @Autowired @@ -296,13 +293,37 @@ public class DataSourceService extends BaseService{ * @return */ private List getDataSources(User loginUser, String searchVal, Integer pageSize, PageInfo pageInfo) { + List dataSourceList = null; if (isAdmin(loginUser)) { - return dataSourceMapper.queryAllDataSourcePaging(searchVal, pageInfo.getStart(), pageSize); + dataSourceList = dataSourceMapper.queryAllDataSourcePaging(searchVal, pageInfo.getStart(), pageSize); + }else{ + dataSourceList = dataSourceMapper.queryDataSourcePaging(loginUser.getId(), searchVal, + pageInfo.getStart(), pageSize); } - return dataSourceMapper.queryDataSourcePaging(loginUser.getId(), searchVal, - pageInfo.getStart(), pageSize); + + handlePasswd(dataSourceList); + + return dataSourceList; } + + /** + * handle datasource connection password for safety + * @param dataSourceList + */ + private void handlePasswd(List dataSourceList) { + + for (DataSource dataSource : dataSourceList) { + + String connectionParams = dataSource.getConnectionParams(); + JSONObject object = JSONObject.parseObject(connectionParams); + object.put(cn.escheduler.common.Constants.PASSWORD, cn.escheduler.common.Constants.XXXXXX); + dataSource.setConnectionParams(JSONUtils.toJson(object)); + + } + } + + /** * get datasource total num * @@ -660,13 +681,13 @@ public class DataSourceService extends BaseService{ */ private String[] getHostsAndPort(String address) { String[] result = new String[2]; - String[] tmpArray = address.split("//"); + String[] tmpArray = address.split(cn.escheduler.common.Constants.DOUBLE_SLASH); String hostsAndPorts = tmpArray[tmpArray.length - 1]; - StringBuilder hosts = new StringBuilder(""); - String[] hostPortArray = hostsAndPorts.split(","); - String port = hostPortArray[0].split(":")[1]; + StringBuilder hosts = new StringBuilder(); + String[] hostPortArray = hostsAndPorts.split(cn.escheduler.common.Constants.COMMA); + String port = hostPortArray[0].split(cn.escheduler.common.Constants.COLON)[1]; for (String hostPort : hostPortArray) { - hosts.append(hostPort.split(":")[0]).append(","); + hosts.append(hostPort.split(cn.escheduler.common.Constants.COLON)[0]).append(cn.escheduler.common.Constants.COMMA); } hosts.deleteCharAt(hosts.length() - 1); result[0] = hosts.toString(); diff --git a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java index 7cd077864c..d2ee3a34c0 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java @@ -489,6 +489,8 @@ public final class Constants { public static final String TASK_RECORD_PWD = "task.record.datasource.password"; public static final String DEFAULT = "Default"; + public static final String PASSWORD = "password"; + public static final String XXXXXX = "xxxxxx"; public static String TASK_RECORD_TABLE_HIVE_LOG = "eamp_hive_log_hd"; From fcd9b661b9af02eae58cd101141ac7a2061b8ba7 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 9 Jul 2019 15:28:36 +0800 Subject: [PATCH 116/149] The start time must not be the same as the end --- .../src/main/java/cn/escheduler/api/enums/Status.java | 1 + .../cn/escheduler/api/service/SchedulerService.java | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java b/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java index bd1d5e91b3..4867f3ca98 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java @@ -163,6 +163,7 @@ public enum Status { BATCH_DELETE_PROCESS_INSTANCE_BY_IDS_ERROR(10117,"batch delete process instance by ids {0} error"), PREVIEW_SCHEDULE_ERROR(10139,"preview schedule error"), PARSE_TO_CRON_EXPRESSION_ERROR(10140,"parse cron to cron expression error"), + SCHEDULE_START_TIME_END_TIME_SAME(10141,"The start time must not be the same as the end"), UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found"), diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java index ab2ef9018a..2766a101e1 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java @@ -119,6 +119,11 @@ public class SchedulerService extends BaseService { scheduleObj.setProcessDefinitionName(processDefinition.getName()); ScheduleParam scheduleParam = JSONUtils.parseObject(schedule, ScheduleParam.class); + if (DateUtils.differSec(scheduleParam.getStartTime(),scheduleParam.getEndTime()) == 0) { + logger.warn("The start time must not be the same as the end"); + putMsg(result,Status.SCHEDULE_START_TIME_END_TIME_SAME); + return result; + } scheduleObj.setStartTime(scheduleParam.getStartTime()); scheduleObj.setEndTime(scheduleParam.getEndTime()); if (!org.quartz.CronExpression.isValidExpression(scheduleParam.getCrontab())) { @@ -205,6 +210,11 @@ public class SchedulerService extends BaseService { // updateProcessInstance param if (StringUtils.isNotEmpty(scheduleExpression)) { ScheduleParam scheduleParam = JSONUtils.parseObject(scheduleExpression, ScheduleParam.class); + if (DateUtils.differSec(scheduleParam.getStartTime(),scheduleParam.getEndTime()) == 0) { + logger.warn("The start time must not be the same as the end"); + putMsg(result,Status.SCHEDULE_START_TIME_END_TIME_SAME); + return result; + } schedule.setStartTime(scheduleParam.getStartTime()); schedule.setEndTime(scheduleParam.getEndTime()); if (!org.quartz.CronExpression.isValidExpression(scheduleParam.getCrontab())) { From 5869a87aca92e23dc0365809e6134a70cc906192 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 9 Jul 2019 15:57:55 +0800 Subject: [PATCH 117/149] The start time must not be the same as the end --- .../java/cn/escheduler/api/service/SchedulerService.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java index 2766a101e1..500a831710 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java @@ -567,6 +567,11 @@ public class SchedulerService extends BaseService { Date startTime = now.after(scheduleParam.getStartTime()) ? now : scheduleParam.getStartTime(); Date endTime = scheduleParam.getEndTime(); + if (DateUtils.differSec(scheduleParam.getStartTime(),scheduleParam.getEndTime()) == 0) { + logger.warn("The start time must not be the same as the end"); + putMsg(result,Status.SCHEDULE_START_TIME_END_TIME_SAME); + return result; + } try { cronExpression = CronUtils.parse2CronExpression(scheduleParam.getCrontab()); } catch (ParseException e) { From 9dbf0265453095e474054954834f0dfdc972c891 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 9 Jul 2019 16:06:53 +0800 Subject: [PATCH 118/149] update preview --- .../java/cn/escheduler/api/service/SchedulerService.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java index 500a831710..2766a101e1 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java @@ -567,11 +567,6 @@ public class SchedulerService extends BaseService { Date startTime = now.after(scheduleParam.getStartTime()) ? now : scheduleParam.getStartTime(); Date endTime = scheduleParam.getEndTime(); - if (DateUtils.differSec(scheduleParam.getStartTime(),scheduleParam.getEndTime()) == 0) { - logger.warn("The start time must not be the same as the end"); - putMsg(result,Status.SCHEDULE_START_TIME_END_TIME_SAME); - return result; - } try { cronExpression = CronUtils.parse2CronExpression(scheduleParam.getCrontab()); } catch (ParseException e) { From b6eb2157797a475a043762377c517440ed124e33 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Tue, 9 Jul 2019 16:07:50 +0800 Subject: [PATCH 119/149] only kerberos startup and sql task type is hive or spark --- .../java/cn/escheduler/api/service/DataSourceService.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java index 5331e03418..3615ca65ad 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java @@ -522,7 +522,10 @@ public class DataSourceService extends BaseService{ parameterMap.put(Constants.JDBC_URL, jdbcUrl); parameterMap.put(Constants.USER, userName); parameterMap.put(Constants.PASSWORD, password); - parameterMap.put(Constants.PRINCIPAL,principal); + if (CommonUtils.getKerberosStartupState() && + (type == DbType.HIVE || type == DbType.SPARK)){ + parameterMap.put(Constants.PRINCIPAL,principal); + } if (other != null && !"".equals(other)) { Map map = JSONObject.parseObject(other, new TypeReference>() { }); From 603c478d2afacf3e1f2a720432b06233af64829f Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Tue, 9 Jul 2019 16:17:15 +0800 Subject: [PATCH 120/149] Constants XXXXXX update --- .../java/cn/escheduler/api/service/DataSourceService.java | 4 ---- .../src/main/java/cn/escheduler/common/Constants.java | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java index 3615ca65ad..2acaef0e84 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java @@ -67,14 +67,10 @@ public class DataSourceService extends BaseService{ public static final String PASSWORD = cn.escheduler.common.Constants.PASSWORD; public static final String OTHER = "other"; - @Autowired - private ProjectMapper projectMapper; @Autowired private DataSourceMapper dataSourceMapper; - @Autowired - private ProjectService projectService; @Autowired private DatasourceUserMapper datasourceUserMapper; diff --git a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java index d2ee3a34c0..d90ea032f4 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java @@ -490,7 +490,7 @@ public final class Constants { public static final String DEFAULT = "Default"; public static final String PASSWORD = "password"; - public static final String XXXXXX = "xxxxxx"; + public static final String XXXXXX = "******"; public static String TASK_RECORD_TABLE_HIVE_LOG = "eamp_hive_log_hd"; From f9d4cf46c0924140946a754f50e8185f9beb3b2a Mon Sep 17 00:00:00 2001 From: lidongdai Date: Tue, 9 Jul 2019 17:46:30 +0800 Subject: [PATCH 121/149] delete nouse code --- .../src/main/java/cn/escheduler/alert/utils/FuncUtils.java | 5 +++-- .../main/java/cn/escheduler/common/zk/AbstractZKClient.java | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/escheduler-alert/src/main/java/cn/escheduler/alert/utils/FuncUtils.java b/escheduler-alert/src/main/java/cn/escheduler/alert/utils/FuncUtils.java index b4238beef0..c9dbe1d676 100644 --- a/escheduler-alert/src/main/java/cn/escheduler/alert/utils/FuncUtils.java +++ b/escheduler-alert/src/main/java/cn/escheduler/alert/utils/FuncUtils.java @@ -22,10 +22,11 @@ public class FuncUtils { StringBuilder sb = new StringBuilder(); boolean first = true; for (String item : list) { - if (first) + if (first) { first = false; - else + } else { sb.append(split); + } sb.append(item); } return sb.toString(); diff --git a/escheduler-common/src/main/java/cn/escheduler/common/zk/AbstractZKClient.java b/escheduler-common/src/main/java/cn/escheduler/common/zk/AbstractZKClient.java index e2f064be13..b72524f411 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/zk/AbstractZKClient.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/zk/AbstractZKClient.java @@ -312,7 +312,6 @@ public abstract class AbstractZKClient { childrenList = zkClient.getChildren().forPath(masterZNodeParentPath); } } catch (Exception e) { -// logger.warn(e.getMessage()); if(!e.getMessage().contains("java.lang.IllegalStateException: instance must be started")){ logger.warn(e.getMessage(),e); } From fd89dff3ab2e860dbb4d386c537a5e99786b57b2 Mon Sep 17 00:00:00 2001 From: lenboo Date: Tue, 9 Jul 2019 18:26:54 +0800 Subject: [PATCH 122/149] refactor ResInfo --- .../api/utils/ZookeeperMonitorUtilsTest.java | 2 - .../cn/escheduler/common/utils/IpUtils.java | 6 --- .../common/queue/TaskQueueImplTest.java | 2 - .../escheduler/common/utils/IpUtilsTest.java | 41 +++++++++++++++++++ .../java/cn/escheduler/server/ResInfo.java | 10 +++++ .../escheduler/server/zk/ZKMasterClient.java | 28 +++---------- .../escheduler/server/zk/ZKWorkerClient.java | 21 +--------- 7 files changed, 59 insertions(+), 51 deletions(-) create mode 100644 escheduler-common/src/test/java/cn/escheduler/common/utils/IpUtilsTest.java diff --git a/escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java b/escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java index adaa78f7ba..87a26ba449 100644 --- a/escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java +++ b/escheduler-api/src/test/java/cn/escheduler/api/utils/ZookeeperMonitorUtilsTest.java @@ -20,8 +20,6 @@ public class ZookeeperMonitorUtilsTest { List workerServerList = zookeeperMonitor.getWorkerServers(); - System.out.println("master:" + masterServerList); - System.out.println("worker:" + workerServerList); Assert.assertEquals(masterServerList.size(), 1); Assert.assertEquals(workerServerList.size(), 1); diff --git a/escheduler-common/src/main/java/cn/escheduler/common/utils/IpUtils.java b/escheduler-common/src/main/java/cn/escheduler/common/utils/IpUtils.java index ddc520a876..972135ae89 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/utils/IpUtils.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/utils/IpUtils.java @@ -61,10 +61,4 @@ public class IpUtils { return sb.toString(); } - - - public static void main(String[] args){ - long ipLong = ipToLong("11.3.4.5"); - logger.info(longToIp(ipLong)); - } } diff --git a/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java b/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java index 16562d54aa..b0f9776b5f 100644 --- a/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java +++ b/escheduler-common/src/test/java/cn/escheduler/common/queue/TaskQueueImplTest.java @@ -20,13 +20,11 @@ import cn.escheduler.common.Constants; import cn.escheduler.common.utils.IpUtils; import cn.escheduler.common.utils.OSUtils; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Arrays; import java.util.List; import java.util.Random; diff --git a/escheduler-common/src/test/java/cn/escheduler/common/utils/IpUtilsTest.java b/escheduler-common/src/test/java/cn/escheduler/common/utils/IpUtilsTest.java new file mode 100644 index 0000000000..11a03a2334 --- /dev/null +++ b/escheduler-common/src/test/java/cn/escheduler/common/utils/IpUtilsTest.java @@ -0,0 +1,41 @@ +package cn.escheduler.common.utils; + +import org.junit.Assert; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class IpUtilsTest { + + @Test + public void ipToLong() { + + String ip = "192.168.110.1"; + String ip2 = "0.0.0.0"; + long longNumber = IpUtils.ipToLong(ip); + long longNumber2 = IpUtils.ipToLong(ip2); + System.out.println(longNumber); + Assert.assertEquals(longNumber, 3232263681L); + Assert.assertEquals(longNumber2, 0L); + + String ip3 = "255.255.255.255"; + long longNumber3 = IpUtils.ipToLong(ip3); + System.out.println(longNumber3); + Assert.assertEquals(longNumber3, 4294967295L); + + } + + @Test + public void longToIp() { + + String ip = "192.168.110.1"; + String ip2 = "0.0.0.0"; + long longNum = 3232263681L; + String i1 = IpUtils.longToIp(longNum); + + String i2 = IpUtils.longToIp(0); + + Assert.assertEquals(ip, i1); + Assert.assertEquals(ip2, i2); + } +} \ No newline at end of file diff --git a/escheduler-server/src/main/java/cn/escheduler/server/ResInfo.java b/escheduler-server/src/main/java/cn/escheduler/server/ResInfo.java index 581a0af482..844c7be8b0 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/ResInfo.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/ResInfo.java @@ -102,6 +102,16 @@ public class ResInfo { } + public static String getHeartBeatInfo(Date now){ + return buildHeartbeatForZKInfo(OSUtils.getHost(), + OSUtils.getProcessID(), + OSUtils.cpuUsage(), + OSUtils.memoryUsage(), + DateUtils.dateToString(now), + DateUtils.dateToString(now)); + + } + /** * build heartbeat info for zk * @param host diff --git a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java index 0bc3247bf4..3596155dd3 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java @@ -204,7 +204,7 @@ public class ZKMasterClient extends AbstractZKClient { } // specify the format of stored data in ZK nodes - String heartbeatZKInfo = getOsInfo(now); + String heartbeatZKInfo = ResInfo.getHeartBeatInfo(now); // create temporary sequence nodes for master znode masterZNode = zkClient.create().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath( masterZNodeParentPath + "/" + OSUtils.getHost() + "_", heartbeatZKInfo.getBytes()); @@ -259,10 +259,10 @@ public class ZKMasterClient extends AbstractZKClient { return false; } - List masterZNodeList = null; - masterZNodeList = zkClient.getChildren().forPath(path); - if (CollectionUtils.isNotEmpty(masterZNodeList)){ - for (String masterZNode : masterZNodeList){ + List serverList = null; + serverList = zkClient.getChildren().forPath(path); + if (CollectionUtils.isNotEmpty(serverList)){ + for (String masterZNode : serverList){ if (masterZNode.startsWith(host)){ return true; } @@ -423,22 +423,6 @@ public class ZKMasterClient extends AbstractZKClient { } - - /** - * get os info - * @param now - * @return - */ - private String getOsInfo(Date now) { - return ResInfo.buildHeartbeatForZKInfo(OSUtils.getHost(), - OSUtils.getProcessID(), - OSUtils.cpuUsage(), - OSUtils.memoryUsage(), - DateUtils.dateToString(now), - DateUtils.dateToString(now)); - } - - /** * get master znode * @return @@ -541,7 +525,7 @@ public class ZKMasterClient extends AbstractZKClient { } /** - * get host ip + * get host ip, string format: masterParentPath/ip_000001/value * @param path * @return */ diff --git a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKWorkerClient.java b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKWorkerClient.java index 4f03d0a41c..e00d72da24 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKWorkerClient.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKWorkerClient.java @@ -116,11 +116,10 @@ public class ZKWorkerClient extends AbstractZKClient { public String initWorkZNode() throws Exception { - Date now = new Date(); - String heartbeatZKInfo = getOsInfo(now); - + String heartbeatZKInfo = ResInfo.getHeartBeatInfo(new Date()); workerZNode = workerZNodeParentPath + "/" + OSUtils.getHost() + "_"; + workerZNode = zkClient.create().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(workerZNode, heartbeatZKInfo.getBytes()); logger.info("register worker node {} success", workerZNode); @@ -141,7 +140,6 @@ public class ZKWorkerClient extends AbstractZKClient { workerZNode = workerZNodeParentPath + "/" + OSUtils.getHost() + "_"; List workerZNodeList = zkClient.getChildren().forPath(workerZNodeParentPath); - if (CollectionUtils.isNotEmpty(workerZNodeList)){ boolean flag = false; for (String workerZNode : workerZNodeList){ @@ -241,21 +239,6 @@ public class ZKWorkerClient extends AbstractZKClient { } - /** - * get os info - * @param now - * @return - */ - private String getOsInfo(Date now) { - return ResInfo.buildHeartbeatForZKInfo(OSUtils.getHost(), - OSUtils.getProcessID(), - OSUtils.cpuUsage(), - OSUtils.memoryUsage(), - DateUtils.dateToString(now), - DateUtils.dateToString(now)); - } - - /** * get worker znode * @return From 5aef28ae66ecd1b07de6f3bd7328d33cd5fcb3f6 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 9 Jul 2019 18:48:01 +0800 Subject: [PATCH 123/149] update email regex --- .../src/main/java/cn/escheduler/common/Constants.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java index d90ea032f4..6c39e0a264 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java @@ -337,7 +337,7 @@ public final class Constants { /** * email regex */ - public static final Pattern REGEX_MAIL_NAME = Pattern.compile("^[a-zA-Z0-9_-]+@[a-zA-Z0-9_-]+(\\.[a-zA-Z0-9_-]+)+$"); + public static final Pattern REGEX_MAIL_NAME = Pattern.compile("^([a-z0-9A-Z]+[-|\\.]?)+[a-z0-9A-Z]@([a-z0-9A-Z]+(-[a-z0-9A-Z]+)?\\.)+[a-zA-Z]{2,}$"); /** * read permission From ffad909fae60b1cdc6589118b4dd28fc0435980c Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 9 Jul 2019 19:56:32 +0800 Subject: [PATCH 124/149] update upgrade and create log tips --- .../cn/escheduler/dao/upgrade/shell/CreateEscheduler.java | 8 +++++--- .../escheduler/dao/upgrade/shell/UpgradeEscheduler.java | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/shell/CreateEscheduler.java b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/shell/CreateEscheduler.java index 3726a70809..2f1e070e7b 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/shell/CreateEscheduler.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/shell/CreateEscheduler.java @@ -30,13 +30,15 @@ public class CreateEscheduler { public static void main(String[] args) { EschedulerManager eschedulerManager = new EschedulerManager(); - eschedulerManager.initEscheduler(); - logger.info("init escheduler finished"); + try { + eschedulerManager.initEscheduler(); + logger.info("init escheduler finished"); eschedulerManager.upgradeEscheduler(); logger.info("upgrade escheduler finished"); + logger.info("create escheduler success"); } catch (Exception e) { - logger.error("upgrade escheduler failed",e); + logger.error("create escheduler failed",e); } } diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/shell/UpgradeEscheduler.java b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/shell/UpgradeEscheduler.java index 822e020cad..7608d8ce6f 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/shell/UpgradeEscheduler.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/upgrade/shell/UpgradeEscheduler.java @@ -30,7 +30,7 @@ public class UpgradeEscheduler { EschedulerManager eschedulerManager = new EschedulerManager(); try { eschedulerManager.upgradeEscheduler(); - logger.info("upgrade escheduler finished"); + logger.info("upgrade escheduler success"); } catch (Exception e) { logger.error(e.getMessage(),e); logger.info("Upgrade escheduler failed"); From 28c67fa13edc609a8aeaa1ea05b0f36c97986f33 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 9 Jul 2019 20:01:27 +0800 Subject: [PATCH 125/149] remove escheduler.sql and quartz.sql --- sql/escheduler.sql | 436 --------------------------------------------- sql/quartz.sql | 179 ------------------- 2 files changed, 615 deletions(-) delete mode 100644 sql/escheduler.sql delete mode 100644 sql/quartz.sql diff --git a/sql/escheduler.sql b/sql/escheduler.sql deleted file mode 100644 index 774de10e42..0000000000 --- a/sql/escheduler.sql +++ /dev/null @@ -1,436 +0,0 @@ -/* -Navicat MySQL Data Transfer - -Source Server : xx.xx -Source Server Version : 50725 -Source Host : 192.168.xx.xx:3306 -Source Database : escheduler - -Target Server Type : MYSQL -Target Server Version : 50725 -File Encoding : 65001 - -Date: 2019-03-23 11:47:30 -*/ - -SET FOREIGN_KEY_CHECKS=0; - --- ---------------------------- --- Table structure for t_escheduler_alert --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_alert`; -CREATE TABLE `t_escheduler_alert` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `title` varchar(64) DEFAULT NULL COMMENT '消息标题', - `show_type` tinyint(4) DEFAULT NULL COMMENT '发送格式，0是TABLE,1是TEXT', - `content` text COMMENT '消息内容（可以是邮件，可以是短信。邮件是JSON Map存放，短信是字符串）', - `alert_type` tinyint(4) DEFAULT NULL COMMENT '0是邮件，1是短信', - `alert_status` tinyint(4) DEFAULT '0' COMMENT '0是待执行，1是执行成功，2执行失败', - `log` text COMMENT '执行日志', - `alertgroup_id` int(11) DEFAULT NULL COMMENT '发送组', - `receivers` text COMMENT '收件人', - `receivers_cc` text COMMENT '抄送人', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_alertgroup --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_alertgroup`; -CREATE TABLE `t_escheduler_alertgroup` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `group_name` varchar(255) DEFAULT NULL COMMENT '组名称', - `group_type` tinyint(4) DEFAULT NULL COMMENT '组类型(邮件0，短信1...)', - `desc` varchar(255) DEFAULT NULL COMMENT '备注', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_command --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_command`; -CREATE TABLE `t_escheduler_command` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `command_type` tinyint(4) DEFAULT NULL COMMENT '命令类型：0 启动工作流,1 从当前节点开始执行,2 恢复被容错的工作流,3 恢复暂停流程,4 从失败节点开始执行,5 补数,6 调度,7 重跑,8 暂停,9 停止,10 恢复等待线程', - `process_definition_id` int(11) DEFAULT NULL COMMENT '流程定义id', - `command_param` text COMMENT '命令的参数（json格式）', - `task_depend_type` tinyint(4) DEFAULT NULL COMMENT '节点依赖类型：0 当前节点,1 向前执行,2 向后执行', - `failure_strategy` tinyint(4) DEFAULT '0' COMMENT '失败策略：0结束，1继续', - `warning_type` tinyint(4) DEFAULT '0' COMMENT '告警类型：0 不发,1 流程成功发,2 流程失败发,3 成功失败都发', - `warning_group_id` int(11) DEFAULT NULL COMMENT '告警组', - `schedule_time` datetime DEFAULT NULL COMMENT '预期运行时间', - `start_time` datetime DEFAULT NULL COMMENT '开始时间', - `executor_id` int(11) DEFAULT NULL COMMENT '执行用户id', - `dependence` varchar(255) DEFAULT NULL COMMENT '依赖字段', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - `process_instance_priority` int(11) DEFAULT NULL COMMENT '流程实例优先级：0 Highest,1 High,2 Medium,3 Low,4 Lowest', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_datasource --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_datasource`; -CREATE TABLE `t_escheduler_datasource` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `name` varchar(64) NOT NULL COMMENT '数据源名称', - `note` varchar(256) DEFAULT NULL COMMENT '描述', - `type` tinyint(4) NOT NULL COMMENT '数据源类型：0 mysql,1 postgresql,2 hive,3 spark', - `user_id` int(11) NOT NULL COMMENT '创建用户id', - `connection_params` text NOT NULL COMMENT '连接参数(json格式)', - `create_time` datetime NOT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_master_server --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_master_server`; -CREATE TABLE `t_escheduler_master_server` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `host` varchar(45) DEFAULT NULL COMMENT 'ip', - `port` int(11) DEFAULT NULL COMMENT '进程号', - `zk_directory` varchar(64) DEFAULT NULL COMMENT 'zk注册目录', - `res_info` varchar(256) DEFAULT NULL COMMENT '集群资源信息：json格式{"cpu":xxx,"memroy":xxx}', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `last_heartbeat_time` datetime DEFAULT NULL COMMENT '最后心跳时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_process_definition --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_process_definition`; -CREATE TABLE `t_escheduler_process_definition` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `name` varchar(255) DEFAULT NULL COMMENT '流程定义名称', - `version` int(11) DEFAULT NULL COMMENT '流程定义版本', - `release_state` tinyint(4) DEFAULT NULL COMMENT '流程定义的发布状态：0 未上线 1已上线', - `project_id` int(11) DEFAULT NULL COMMENT '项目id', - `user_id` int(11) DEFAULT NULL COMMENT '流程定义所属用户id', - `process_definition_json` longtext COMMENT '流程定义json串', - `desc` text COMMENT '流程定义描述', - `global_params` text COMMENT '全局参数', - `flag` tinyint(4) DEFAULT NULL COMMENT '流程是否可用\r\n：0 不可用\r\n，1 可用', - `locations` text COMMENT '节点坐标信息', - `connects` text COMMENT '节点连线信息', - `receivers` text COMMENT '收件人', - `receivers_cc` text COMMENT '抄送人', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`), - KEY `process_definition_index` (`project_id`,`id`) USING BTREE -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_process_instance --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_process_instance`; -CREATE TABLE `t_escheduler_process_instance` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `name` varchar(255) DEFAULT NULL COMMENT '流程实例名称', - `process_definition_id` int(11) DEFAULT NULL COMMENT '流程定义id', - `state` tinyint(4) DEFAULT NULL COMMENT '流程实例状态：0 提交成功,1 正在运行,2 准备暂停,3 暂停,4 准备停止,5 停止,6 失败,7 成功,8 需要容错,9 kill,10 等待线程,11 等待依赖完成', - `recovery` tinyint(4) DEFAULT NULL COMMENT '流程实例容错标识：0 正常,1 需要被容错重启', - `start_time` datetime DEFAULT NULL COMMENT '流程实例开始时间', - `end_time` datetime DEFAULT NULL COMMENT '流程实例结束时间', - `run_times` int(11) DEFAULT NULL COMMENT '流程实例运行次数', - `host` varchar(45) DEFAULT NULL COMMENT '流程实例所在的机器', - `command_type` tinyint(4) DEFAULT NULL COMMENT '命令类型：0 启动工作流,1 从当前节点开始执行,2 恢复被容错的工作流,3 恢复暂停流程,4 从失败节点开始执行,5 补数,6 调度,7 重跑,8 暂停,9 停止,10 恢复等待线程', - `command_param` text COMMENT '命令的参数（json格式）', - `task_depend_type` tinyint(4) DEFAULT NULL COMMENT '节点依赖类型：0 当前节点,1 向前执行,2 向后执行', - `max_try_times` tinyint(4) DEFAULT '0' COMMENT '最大重试次数', - `failure_strategy` tinyint(4) DEFAULT '0' COMMENT '失败策略 0 失败后结束，1 失败后继续', - `warning_type` tinyint(4) DEFAULT '0' COMMENT '告警类型：0 不发,1 流程成功发,2 流程失败发,3 成功失败都发', - `warning_group_id` int(11) DEFAULT NULL COMMENT '告警组id', - `schedule_time` datetime DEFAULT NULL COMMENT '预期运行时间', - `command_start_time` datetime DEFAULT NULL COMMENT '开始命令时间', - `global_params` text COMMENT '全局参数（固化流程定义的参数）', - `process_instance_json` longtext COMMENT '流程实例json(copy的流程定义的json)', - `flag` tinyint(4) DEFAULT '1' COMMENT '是否可用，1 可用，0不可用', - `update_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `is_sub_process` int(11) DEFAULT '0' COMMENT '是否是子工作流 1 是，0 不是', - `executor_id` int(11) NOT NULL COMMENT '命令执行者', - `locations` text COMMENT '节点坐标信息', - `connects` text COMMENT '节点连线信息', - `history_cmd` text COMMENT '历史命令，记录所有对流程实例的操作', - `dependence_schedule_times` text COMMENT '依赖节点的预估时间', - `process_instance_priority` int(11) DEFAULT NULL COMMENT '流程实例优先级：0 Highest,1 High,2 Medium,3 Low,4 Lowest', - PRIMARY KEY (`id`), - KEY `process_instance_index` (`process_definition_id`,`id`) USING BTREE, - KEY `start_time_index` (`start_time`) USING BTREE -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_project --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_project`; -CREATE TABLE `t_escheduler_project` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `name` varchar(100) DEFAULT NULL COMMENT '项目名称', - `desc` varchar(200) DEFAULT NULL COMMENT '项目描述', - `user_id` int(11) DEFAULT NULL COMMENT '所属用户', - `flag` tinyint(4) DEFAULT '1' COMMENT '是否可用 1 可用,0 不可用', - `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', - `update_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间', - PRIMARY KEY (`id`), - KEY `user_id_index` (`user_id`) USING BTREE -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_queue --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_queue`; -CREATE TABLE `t_escheduler_queue` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `queue_name` varchar(64) DEFAULT NULL COMMENT '队列名称', - `queue` varchar(64) DEFAULT NULL COMMENT 'yarn队列名称', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_relation_datasource_user --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_relation_datasource_user`; -CREATE TABLE `t_escheduler_relation_datasource_user` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `user_id` int(11) NOT NULL COMMENT '用户id', - `datasource_id` int(11) DEFAULT NULL COMMENT '数据源id', - `perm` int(11) DEFAULT '1' COMMENT '权限', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_relation_process_instance --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_relation_process_instance`; -CREATE TABLE `t_escheduler_relation_process_instance` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `parent_process_instance_id` int(11) DEFAULT NULL COMMENT '父流程实例id', - `parent_task_instance_id` int(11) DEFAULT NULL COMMENT '父任务实例id', - `process_instance_id` int(11) DEFAULT NULL COMMENT '子流程实例id', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_relation_project_user --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_relation_project_user`; -CREATE TABLE `t_escheduler_relation_project_user` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `user_id` int(11) NOT NULL COMMENT '用户id', - `project_id` int(11) DEFAULT NULL COMMENT '项目id', - `perm` int(11) DEFAULT '1' COMMENT '权限', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`), - KEY `user_id_index` (`user_id`) USING BTREE -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_relation_resources_user --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_relation_resources_user`; -CREATE TABLE `t_escheduler_relation_resources_user` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `user_id` int(11) NOT NULL COMMENT '用户id', - `resources_id` int(11) DEFAULT NULL COMMENT '资源id', - `perm` int(11) DEFAULT '1' COMMENT '权限', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_relation_udfs_user --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_relation_udfs_user`; -CREATE TABLE `t_escheduler_relation_udfs_user` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `user_id` int(11) NOT NULL COMMENT '用户id', - `udf_id` int(11) DEFAULT NULL COMMENT 'udf id', - `perm` int(11) DEFAULT '1' COMMENT '权限', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_relation_user_alertgroup --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_relation_user_alertgroup`; -CREATE TABLE `t_escheduler_relation_user_alertgroup` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `alertgroup_id` int(11) DEFAULT NULL COMMENT '组消息id', - `user_id` int(11) DEFAULT NULL COMMENT '用户id', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_resources --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_resources`; -CREATE TABLE `t_escheduler_resources` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `alias` varchar(64) DEFAULT NULL COMMENT '别名', - `file_name` varchar(64) DEFAULT NULL COMMENT '文件名', - `desc` varchar(256) DEFAULT NULL COMMENT '描述', - `user_id` int(11) DEFAULT NULL COMMENT '用户id', - `type` tinyint(4) DEFAULT NULL COMMENT '资源类型，0 FILE，1 UDF', - `size` bigint(20) DEFAULT NULL COMMENT '资源大小', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_schedules --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_schedules`; -CREATE TABLE `t_escheduler_schedules` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `process_definition_id` int(11) NOT NULL COMMENT '流程定义id', - `start_time` datetime NOT NULL COMMENT '调度开始时间', - `end_time` datetime NOT NULL COMMENT '调度结束时间', - `crontab` varchar(256) NOT NULL COMMENT 'crontab 表达式', - `failure_strategy` tinyint(4) NOT NULL COMMENT '失败策略： 0 结束，1 继续', - `user_id` int(11) NOT NULL COMMENT '用户id', - `release_state` tinyint(4) NOT NULL COMMENT '状态：0 未上线，1 上线', - `warning_type` tinyint(4) NOT NULL COMMENT '告警类型：0 不发,1 流程成功发,2 流程失败发,3 成功失败都发', - `warning_group_id` int(11) DEFAULT NULL COMMENT '告警组id', - `process_instance_priority` int(11) DEFAULT NULL COMMENT '流程实例优先级：0 Highest,1 High,2 Medium,3 Low,4 Lowest', - `create_time` datetime NOT NULL COMMENT '创建时间', - `update_time` datetime NOT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_session --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_session`; -CREATE TABLE `t_escheduler_session` ( - `id` varchar(64) NOT NULL COMMENT '主键', - `user_id` int(11) DEFAULT NULL COMMENT '用户id', - `ip` varchar(45) DEFAULT NULL COMMENT '登录ip', - `last_login_time` datetime DEFAULT NULL COMMENT '最后登录时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_task_instance --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_task_instance`; -CREATE TABLE `t_escheduler_task_instance` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `name` varchar(255) DEFAULT NULL COMMENT '任务名称', - `task_type` varchar(64) DEFAULT NULL COMMENT '任务类型', - `process_definition_id` int(11) DEFAULT NULL COMMENT '流程定义id', - `process_instance_id` int(11) DEFAULT NULL COMMENT '流程实例id', - `task_json` longtext COMMENT '任务节点json', - `state` tinyint(4) DEFAULT NULL COMMENT '任务实例状态：0 提交成功,1 正在运行,2 准备暂停,3 暂停,4 准备停止,5 停止,6 失败,7 成功,8 需要容错,9 kill,10 等待线程,11 等待依赖完成', - `submit_time` datetime DEFAULT NULL COMMENT '任务提交时间', - `start_time` datetime DEFAULT NULL COMMENT '任务开始时间', - `end_time` datetime DEFAULT NULL COMMENT '任务结束时间', - `host` varchar(45) DEFAULT NULL COMMENT '执行任务的机器', - `execute_path` varchar(200) DEFAULT NULL COMMENT '任务执行路径', - `log_path` varchar(200) DEFAULT NULL COMMENT '任务日志路径', - `alert_flag` tinyint(4) DEFAULT NULL COMMENT '是否告警', - `retry_times` int(4) DEFAULT '0' COMMENT '重试次数', - `pid` int(4) DEFAULT NULL COMMENT '进程pid', - `app_link` varchar(255) DEFAULT NULL COMMENT 'yarn app id', - `flag` tinyint(4) DEFAULT '1' COMMENT '是否可用：0 不可用，1 可用', - `retry_interval` int(4) DEFAULT NULL COMMENT '重试间隔', - `max_retry_times` int(2) DEFAULT NULL COMMENT '最大重试次数', - `task_instance_priority` int(11) DEFAULT NULL COMMENT '任务实例优先级：0 Highest,1 High,2 Medium,3 Low,4 Lowest', - PRIMARY KEY (`id`), - KEY `process_instance_id` (`process_instance_id`) USING BTREE, - KEY `task_instance_index` (`process_definition_id`,`process_instance_id`) USING BTREE, - CONSTRAINT `foreign_key_instance_id` FOREIGN KEY (`process_instance_id`) REFERENCES `t_escheduler_process_instance` (`id`) ON DELETE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_tenant --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_tenant`; -CREATE TABLE `t_escheduler_tenant` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `tenant_code` varchar(64) DEFAULT NULL COMMENT '租户编码', - `tenant_name` varchar(64) DEFAULT NULL COMMENT '租户名称', - `desc` varchar(256) DEFAULT NULL COMMENT '描述', - `queue_id` int(11) DEFAULT NULL COMMENT '队列id', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_udfs --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_udfs`; -CREATE TABLE `t_escheduler_udfs` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `user_id` int(11) NOT NULL COMMENT '用户id', - `func_name` varchar(100) NOT NULL COMMENT 'UDF函数名', - `class_name` varchar(255) NOT NULL COMMENT '类名', - `type` tinyint(4) NOT NULL COMMENT 'Udf函数类型', - `arg_types` varchar(255) DEFAULT NULL COMMENT '参数', - `database` varchar(255) DEFAULT NULL COMMENT '库名', - `desc` varchar(255) DEFAULT NULL COMMENT '描述', - `resource_id` int(11) NOT NULL COMMENT '资源id', - `resource_name` varchar(255) NOT NULL COMMENT '资源名称', - `create_time` datetime NOT NULL COMMENT '创建时间', - `update_time` datetime NOT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_user --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_user`; -CREATE TABLE `t_escheduler_user` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '用户id', - `user_name` varchar(64) DEFAULT NULL COMMENT '用户名', - `user_password` varchar(64) DEFAULT NULL COMMENT '用户密码', - `user_type` tinyint(4) DEFAULT NULL COMMENT '用户类型：0 管理员，1 普通用户', - `email` varchar(64) DEFAULT NULL COMMENT '邮箱', - `phone` varchar(11) DEFAULT NULL COMMENT '手机', - `tenant_id` int(11) DEFAULT NULL COMMENT '管理员0,普通用户所属租户id', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `update_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`), - UNIQUE KEY `user_name_unique` (`user_name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- ---------------------------- --- Table structure for t_escheduler_worker_server --- ---------------------------- -DROP TABLE IF EXISTS `t_escheduler_worker_server`; -CREATE TABLE `t_escheduler_worker_server` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `host` varchar(45) DEFAULT NULL COMMENT 'ip', - `port` int(11) DEFAULT NULL COMMENT '进程号', - `zk_directory` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL COMMENT 'zk注册目录', - `res_info` varchar(255) DEFAULT NULL COMMENT '集群资源信息：json格式{"cpu":xxx,"memroy":xxx}', - `create_time` datetime DEFAULT NULL COMMENT '创建时间', - `last_heartbeat_time` datetime DEFAULT NULL COMMENT '更新时间', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- Records of t_escheduler_user,user : admin , password : escheduler123 -INSERT INTO `t_escheduler_user` VALUES ('1', 'admin', '055a97b5fcd6d120372ad1976518f371', '0', 'xxx@qq.com', 'xxxx', '0', '2018-03-27 15:48:50', '2018-10-24 17:40:22'); -INSERT INTO `t_escheduler_alertgroup` VALUES (1, 'escheduler管理员告警组', '0', 'escheduler管理员告警组','2018-11-29 10:20:39', '2018-11-29 10:20:39'); -INSERT INTO `t_escheduler_relation_user_alertgroup` VALUES ('1', '1', '1', '2018-11-29 10:22:33', '2018-11-29 10:22:33'); - --- Records of t_escheduler_queue,default queue name : default -INSERT INTO `t_escheduler_queue` VALUES ('1', 'default', 'default'); - - diff --git a/sql/quartz.sql b/sql/quartz.sql deleted file mode 100644 index 22754b39dc..0000000000 --- a/sql/quartz.sql +++ /dev/null @@ -1,179 +0,0 @@ - # - # In your Quartz properties file, you'll need to set - # org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.StdJDBCDelegate - # - # - # By: Ron Cordell - roncordell - # I didn't see this anywhere, so I thought I'd post it here. This is the script from Quartz to create the tables in a MySQL database, modified to use INNODB instead of MYISAM. - - DROP TABLE IF EXISTS QRTZ_FIRED_TRIGGERS; - DROP TABLE IF EXISTS QRTZ_PAUSED_TRIGGER_GRPS; - DROP TABLE IF EXISTS QRTZ_SCHEDULER_STATE; - DROP TABLE IF EXISTS QRTZ_LOCKS; - DROP TABLE IF EXISTS QRTZ_SIMPLE_TRIGGERS; - DROP TABLE IF EXISTS QRTZ_SIMPROP_TRIGGERS; - DROP TABLE IF EXISTS QRTZ_CRON_TRIGGERS; - DROP TABLE IF EXISTS QRTZ_BLOB_TRIGGERS; - DROP TABLE IF EXISTS QRTZ_TRIGGERS; - DROP TABLE IF EXISTS QRTZ_JOB_DETAILS; - DROP TABLE IF EXISTS QRTZ_CALENDARS; - - CREATE TABLE QRTZ_JOB_DETAILS( - SCHED_NAME VARCHAR(120) NOT NULL, - JOB_NAME VARCHAR(200) NOT NULL, - JOB_GROUP VARCHAR(200) NOT NULL, - DESCRIPTION VARCHAR(250) NULL, - JOB_CLASS_NAME VARCHAR(250) NOT NULL, - IS_DURABLE VARCHAR(1) NOT NULL, - IS_NONCONCURRENT VARCHAR(1) NOT NULL, - IS_UPDATE_DATA VARCHAR(1) NOT NULL, - REQUESTS_RECOVERY VARCHAR(1) NOT NULL, - JOB_DATA BLOB NULL, - PRIMARY KEY (SCHED_NAME,JOB_NAME,JOB_GROUP)) - ENGINE=InnoDB; - - CREATE TABLE QRTZ_TRIGGERS ( - SCHED_NAME VARCHAR(120) NOT NULL, - TRIGGER_NAME VARCHAR(200) NOT NULL, - TRIGGER_GROUP VARCHAR(200) NOT NULL, - JOB_NAME VARCHAR(200) NOT NULL, - JOB_GROUP VARCHAR(200) NOT NULL, - DESCRIPTION VARCHAR(250) NULL, - NEXT_FIRE_TIME BIGINT(13) NULL, - PREV_FIRE_TIME BIGINT(13) NULL, - PRIORITY INTEGER NULL, - TRIGGER_STATE VARCHAR(16) NOT NULL, - TRIGGER_TYPE VARCHAR(8) NOT NULL, - START_TIME BIGINT(13) NOT NULL, - END_TIME BIGINT(13) NULL, - CALENDAR_NAME VARCHAR(200) NULL, - MISFIRE_INSTR SMALLINT(2) NULL, - JOB_DATA BLOB NULL, - PRIMARY KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP), - FOREIGN KEY (SCHED_NAME,JOB_NAME,JOB_GROUP) - REFERENCES QRTZ_JOB_DETAILS(SCHED_NAME,JOB_NAME,JOB_GROUP)) - ENGINE=InnoDB; - - CREATE TABLE QRTZ_SIMPLE_TRIGGERS ( - SCHED_NAME VARCHAR(120) NOT NULL, - TRIGGER_NAME VARCHAR(200) NOT NULL, - TRIGGER_GROUP VARCHAR(200) NOT NULL, - REPEAT_COUNT BIGINT(7) NOT NULL, - REPEAT_INTERVAL BIGINT(12) NOT NULL, - TIMES_TRIGGERED BIGINT(10) NOT NULL, - PRIMARY KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP), - FOREIGN KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP) - REFERENCES QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP)) - ENGINE=InnoDB; - - CREATE TABLE QRTZ_CRON_TRIGGERS ( - SCHED_NAME VARCHAR(120) NOT NULL, - TRIGGER_NAME VARCHAR(200) NOT NULL, - TRIGGER_GROUP VARCHAR(200) NOT NULL, - CRON_EXPRESSION VARCHAR(120) NOT NULL, - TIME_ZONE_ID VARCHAR(80), - PRIMARY KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP), - FOREIGN KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP) - REFERENCES QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP)) - ENGINE=InnoDB; - - CREATE TABLE QRTZ_SIMPROP_TRIGGERS - ( - SCHED_NAME VARCHAR(120) NOT NULL, - TRIGGER_NAME VARCHAR(200) NOT NULL, - TRIGGER_GROUP VARCHAR(200) NOT NULL, - STR_PROP_1 VARCHAR(512) NULL, - STR_PROP_2 VARCHAR(512) NULL, - STR_PROP_3 VARCHAR(512) NULL, - INT_PROP_1 INT NULL, - INT_PROP_2 INT NULL, - LONG_PROP_1 BIGINT NULL, - LONG_PROP_2 BIGINT NULL, - DEC_PROP_1 NUMERIC(13,4) NULL, - DEC_PROP_2 NUMERIC(13,4) NULL, - BOOL_PROP_1 VARCHAR(1) NULL, - BOOL_PROP_2 VARCHAR(1) NULL, - PRIMARY KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP), - FOREIGN KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP) - REFERENCES QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP)) - ENGINE=InnoDB; - - CREATE TABLE QRTZ_BLOB_TRIGGERS ( - SCHED_NAME VARCHAR(120) NOT NULL, - TRIGGER_NAME VARCHAR(200) NOT NULL, - TRIGGER_GROUP VARCHAR(200) NOT NULL, - BLOB_DATA BLOB NULL, - PRIMARY KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP), - INDEX (SCHED_NAME,TRIGGER_NAME, TRIGGER_GROUP), - FOREIGN KEY (SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP) - REFERENCES QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP)) - ENGINE=InnoDB; - - CREATE TABLE QRTZ_CALENDARS ( - SCHED_NAME VARCHAR(120) NOT NULL, - CALENDAR_NAME VARCHAR(200) NOT NULL, - CALENDAR BLOB NOT NULL, - PRIMARY KEY (SCHED_NAME,CALENDAR_NAME)) - ENGINE=InnoDB; - - CREATE TABLE QRTZ_PAUSED_TRIGGER_GRPS ( - SCHED_NAME VARCHAR(120) NOT NULL, - TRIGGER_GROUP VARCHAR(200) NOT NULL, - PRIMARY KEY (SCHED_NAME,TRIGGER_GROUP)) - ENGINE=InnoDB; - - CREATE TABLE QRTZ_FIRED_TRIGGERS ( - SCHED_NAME VARCHAR(120) NOT NULL, - ENTRY_ID VARCHAR(95) NOT NULL, - TRIGGER_NAME VARCHAR(200) NOT NULL, - TRIGGER_GROUP VARCHAR(200) NOT NULL, - INSTANCE_NAME VARCHAR(200) NOT NULL, - FIRED_TIME BIGINT(13) NOT NULL, - SCHED_TIME BIGINT(13) NOT NULL, - PRIORITY INTEGER NOT NULL, - STATE VARCHAR(16) NOT NULL, - JOB_NAME VARCHAR(200) NULL, - JOB_GROUP VARCHAR(200) NULL, - IS_NONCONCURRENT VARCHAR(1) NULL, - REQUESTS_RECOVERY VARCHAR(1) NULL, - PRIMARY KEY (SCHED_NAME,ENTRY_ID)) - ENGINE=InnoDB; - - CREATE TABLE QRTZ_SCHEDULER_STATE ( - SCHED_NAME VARCHAR(120) NOT NULL, - INSTANCE_NAME VARCHAR(200) NOT NULL, - LAST_CHECKIN_TIME BIGINT(13) NOT NULL, - CHECKIN_INTERVAL BIGINT(13) NOT NULL, - PRIMARY KEY (SCHED_NAME,INSTANCE_NAME)) - ENGINE=InnoDB; - - CREATE TABLE QRTZ_LOCKS ( - SCHED_NAME VARCHAR(120) NOT NULL, - LOCK_NAME VARCHAR(40) NOT NULL, - PRIMARY KEY (SCHED_NAME,LOCK_NAME)) - ENGINE=InnoDB; - - CREATE INDEX IDX_QRTZ_J_REQ_RECOVERY ON QRTZ_JOB_DETAILS(SCHED_NAME,REQUESTS_RECOVERY); - CREATE INDEX IDX_QRTZ_J_GRP ON QRTZ_JOB_DETAILS(SCHED_NAME,JOB_GROUP); - - CREATE INDEX IDX_QRTZ_T_J ON QRTZ_TRIGGERS(SCHED_NAME,JOB_NAME,JOB_GROUP); - CREATE INDEX IDX_QRTZ_T_JG ON QRTZ_TRIGGERS(SCHED_NAME,JOB_GROUP); - CREATE INDEX IDX_QRTZ_T_C ON QRTZ_TRIGGERS(SCHED_NAME,CALENDAR_NAME); - CREATE INDEX IDX_QRTZ_T_G ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_GROUP); - CREATE INDEX IDX_QRTZ_T_STATE ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_STATE); - CREATE INDEX IDX_QRTZ_T_N_STATE ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP,TRIGGER_STATE); - CREATE INDEX IDX_QRTZ_T_N_G_STATE ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_GROUP,TRIGGER_STATE); - CREATE INDEX IDX_QRTZ_T_NEXT_FIRE_TIME ON QRTZ_TRIGGERS(SCHED_NAME,NEXT_FIRE_TIME); - CREATE INDEX IDX_QRTZ_T_NFT_ST ON QRTZ_TRIGGERS(SCHED_NAME,TRIGGER_STATE,NEXT_FIRE_TIME); - CREATE INDEX IDX_QRTZ_T_NFT_MISFIRE ON QRTZ_TRIGGERS(SCHED_NAME,MISFIRE_INSTR,NEXT_FIRE_TIME); - CREATE INDEX IDX_QRTZ_T_NFT_ST_MISFIRE ON QRTZ_TRIGGERS(SCHED_NAME,MISFIRE_INSTR,NEXT_FIRE_TIME,TRIGGER_STATE); - CREATE INDEX IDX_QRTZ_T_NFT_ST_MISFIRE_GRP ON QRTZ_TRIGGERS(SCHED_NAME,MISFIRE_INSTR,NEXT_FIRE_TIME,TRIGGER_GROUP,TRIGGER_STATE); - - CREATE INDEX IDX_QRTZ_FT_TRIG_INST_NAME ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,INSTANCE_NAME); - CREATE INDEX IDX_QRTZ_FT_INST_JOB_REQ_RCVRY ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,INSTANCE_NAME,REQUESTS_RECOVERY); - CREATE INDEX IDX_QRTZ_FT_J_G ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,JOB_NAME,JOB_GROUP); - CREATE INDEX IDX_QRTZ_FT_JG ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,JOB_GROUP); - CREATE INDEX IDX_QRTZ_FT_T_G ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP); - CREATE INDEX IDX_QRTZ_FT_TG ON QRTZ_FIRED_TRIGGERS(SCHED_NAME,TRIGGER_GROUP); - - commit; \ No newline at end of file From e33d97676bd7986714ff5d3f5e8afa5b2a10cce4 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Wed, 10 Jul 2019 11:06:04 +0800 Subject: [PATCH 126/149] install.sh update --- install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index 33acdf5eec..57cccdf22a 100644 --- a/install.sh +++ b/install.sh @@ -134,7 +134,7 @@ s3Endpoint="http://192.168.199.91:9010" s3AccessKey="A3DXS30FO22544RE" s3SecretKey="OloCLq3n+8+sdPHUhJ21XrSxTC+JK" -# resourcemanager HA配置，如果是单resourcemanager,这里为空即可 +# resourcemanager HA配置，如果是单resourcemanager,这里为yarnHaIps="" yarnHaIps="192.168.xx.xx,192.168.xx.xx" # 如果是单 resourcemanager,只需要配置一个主机名称,如果是resourcemanager HA,则默认配置就好 @@ -144,7 +144,7 @@ singleYarnIp="ark1" hdfsPath="/escheduler" # 拥有在hdfs根路径/下创建目录权限的用户 -# 注意：如果开启了kerberos，则直接hdfsRootUser=，就可以 +# 注意：如果开启了kerberos，则直接hdfsRootUser=""，就可以 hdfsRootUser="hdfs" # common 配置 From 01eaedfad7c3288acdcacb7462cc17dee4cc34d5 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Wed, 10 Jul 2019 11:49:52 +0800 Subject: [PATCH 127/149] determine whether the tenant path exists when the tenant deletes --- .../escheduler/api/service/TenantService.java | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java index 555275273f..21d5f270fb 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java @@ -239,24 +239,25 @@ public class TenantService extends BaseService{ if (PropertyUtils.getResUploadStartupState()){ String tenantPath = HadoopUtils.getHdfsDataBasePath() + "/" + tenant.getTenantCode(); - String resourcePath = HadoopUtils.getHdfsDir(tenant.getTenantCode()); - FileStatus[] fileStatus = HadoopUtils.getInstance().listFileStatus(resourcePath); - if (fileStatus.length > 0) { - putMsg(result, Status.HDFS_TERANT_RESOURCES_FILE_EXISTS); - return result; - } - fileStatus = HadoopUtils.getInstance().listFileStatus(HadoopUtils.getHdfsUdfDir(tenant.getTenantCode())); - if (fileStatus.length > 0) { - putMsg(result, Status.HDFS_TERANT_UDFS_FILE_EXISTS); - return result; - } + if (HadoopUtils.getInstance().exists(tenantPath)){ + String resourcePath = HadoopUtils.getHdfsDir(tenant.getTenantCode()); + FileStatus[] fileStatus = HadoopUtils.getInstance().listFileStatus(resourcePath); + if (fileStatus.length > 0) { + putMsg(result, Status.HDFS_TERANT_RESOURCES_FILE_EXISTS); + return result; + } + fileStatus = HadoopUtils.getInstance().listFileStatus(HadoopUtils.getHdfsUdfDir(tenant.getTenantCode())); + if (fileStatus.length > 0) { + putMsg(result, Status.HDFS_TERANT_UDFS_FILE_EXISTS); + return result; + } - HadoopUtils.getInstance().delete(tenantPath, true); + HadoopUtils.getInstance().delete(tenantPath, true); + } } tenantMapper.deleteById(id); putMsg(result, Status.SUCCESS); - return result; } @@ -269,9 +270,6 @@ public class TenantService extends BaseService{ public Map queryTenantList(User loginUser) { Map result = new HashMap<>(5); -// if (checkAdmin(loginUser, result)) { -// return result; -// } List resourceList = tenantMapper.queryAllTenant(); result.put(Constants.DATA_LIST, resourceList); From a71bae3eb2cdffd514c5979b05758b46fd7b1c38 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Wed, 10 Jul 2019 12:41:59 +0800 Subject: [PATCH 128/149] update change tenant --- .../escheduler/api/service/UsersService.java | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java index 512369ea8a..2d0012c18f 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java @@ -250,30 +250,30 @@ public class UsersService extends BaseService { String oldResourcePath = HadoopUtils.getHdfsDataBasePath() + "/" + oldTenant.getTenantCode() + "/resources"; String oldUdfsPath = HadoopUtils.getHdfsUdfDir(oldTenant.getTenantCode()); - - String newResourcePath = HadoopUtils.getHdfsDataBasePath() + "/" + newTenantCode + "/resources"; - String newUdfsPath = HadoopUtils.getHdfsUdfDir(newTenantCode); - - //file resources list - List fileResourcesList = resourceMapper.queryResourceCreatedByUser(userId, 0); - if (CollectionUtils.isNotEmpty(fileResourcesList)) { - for (Resource resource : fileResourcesList) { - HadoopUtils.getInstance().copy(oldResourcePath + "/" + resource.getAlias(), newResourcePath, false, true); + if (HadoopUtils.getInstance().exists(oldResourcePath)){ + String newResourcePath = HadoopUtils.getHdfsDataBasePath() + "/" + newTenantCode + "/resources"; + String newUdfsPath = HadoopUtils.getHdfsUdfDir(newTenantCode); + + //file resources list + List fileResourcesList = resourceMapper.queryResourceCreatedByUser(userId, 0); + if (CollectionUtils.isNotEmpty(fileResourcesList)) { + for (Resource resource : fileResourcesList) { + HadoopUtils.getInstance().copy(oldResourcePath + "/" + resource.getAlias(), newResourcePath, false, true); + } } - } - //udf resources - List udfResourceList = resourceMapper.queryResourceCreatedByUser(userId, 1); - if (CollectionUtils.isNotEmpty(udfResourceList)) { - for (Resource resource : udfResourceList) { - HadoopUtils.getInstance().copy(oldUdfsPath + "/" + resource.getAlias(), newUdfsPath, false, true); + //udf resources + List udfResourceList = resourceMapper.queryResourceCreatedByUser(userId, 1); + if (CollectionUtils.isNotEmpty(udfResourceList)) { + for (Resource resource : udfResourceList) { + HadoopUtils.getInstance().copy(oldUdfsPath + "/" + resource.getAlias(), newUdfsPath, false, true); + } } - } - - //Delete the user from the old tenant directory - String oldUserPath = HadoopUtils.getHdfsDataBasePath() + "/" + oldTenant.getTenantCode() + "/home/" + userId; - HadoopUtils.getInstance().delete(oldUserPath, true); + //Delete the user from the old tenant directory + String oldUserPath = HadoopUtils.getHdfsDataBasePath() + "/" + oldTenant.getTenantCode() + "/home/" + userId; + HadoopUtils.getInstance().delete(oldUserPath, true); + } //create user in the new tenant directory String newUserPath = HadoopUtils.getHdfsDataBasePath() + "/" + newTenant.getTenantCode() + "/home/" + user.getId(); From 5d0e64f228b75bc290f4907a504a7f0e7aac0ccb Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Wed, 10 Jul 2019 12:54:20 +0800 Subject: [PATCH 129/149] user chanage tenant update --- .../src/main/java/cn/escheduler/api/service/UsersService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java index 2d0012c18f..8004117e91 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java @@ -245,7 +245,7 @@ public class UsersService extends BaseService { Tenant newTenant = tenantMapper.queryById(tenantId); if (newTenant != null) { // if hdfs startup - if (PropertyUtils.getResUploadStartupState()){ + if (PropertyUtils.getResUploadStartupState() && oldTenant != null){ String newTenantCode = newTenant.getTenantCode(); String oldResourcePath = HadoopUtils.getHdfsDataBasePath() + "/" + oldTenant.getTenantCode() + "/resources"; String oldUdfsPath = HadoopUtils.getHdfsUdfDir(oldTenant.getTenantCode()); From 1b94067bb9eacd7a1c67658d6be687fbd1f4edc2 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Wed, 10 Jul 2019 14:13:24 +0800 Subject: [PATCH 130/149] update dag.vue remove console log --- escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue | 2 -- 1 file changed, 2 deletions(-) diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue b/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue index b2062b7769..e1226585f7 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue @@ -459,8 +459,6 @@ 'tasks': { deep: true, handler (o) { - console.log('+++++ save dag params +++++') - console.log(o) // Edit state does not allow deletion of node a... this.setIsEditDag(true) From af217278d8472852b124d13861a8607640df9fd3 Mon Sep 17 00:00:00 2001 From: lidongdai Date: Wed, 10 Jul 2019 14:38:42 +0800 Subject: [PATCH 131/149] add note --- .../src/main/java/cn/escheduler/dao/ProcessDao.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java index d393339a5e..16d2bd078a 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java @@ -1011,11 +1011,11 @@ public class ProcessDao extends AbstractBaseDao { } /** - * ${processInstancePriority}_${processInstanceId}_${taskInstancePriority}_${taskId} + * ${processInstancePriority}_${processInstanceId}_${taskInstancePriority}_${taskId}_${task executed by ip1},${ip2}... * * The tasks with the highest priority are selected by comparing the priorities of the above four levels from high to low. * - * 流程实例优先级_流程实例id_任务优先级_任务id high <- low + * 流程实例优先级_流程实例id_任务优先级_任务id_任务执行机器ip1，ip2... high <- low * * @param taskInstance * @return From b6ebb56777c424124d90fb0f8d772013b535cd88 Mon Sep 17 00:00:00 2001 From: xianhu Date: Wed, 10 Jul 2019 17:15:03 +0800 Subject: [PATCH 132/149] fix bugs #516 --- .../src/main/java/cn/escheduler/server/utils/LoggerUtils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/utils/LoggerUtils.java b/escheduler-server/src/main/java/cn/escheduler/server/utils/LoggerUtils.java index bbb404f536..1e0bd196ff 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/utils/LoggerUtils.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/utils/LoggerUtils.java @@ -16,6 +16,7 @@ */ package cn.escheduler.server.utils; +import cn.escheduler.common.Constants; import org.slf4j.Logger; import java.util.ArrayList; @@ -31,7 +32,7 @@ public class LoggerUtils { /** * rules for extracting application ID */ - private static final Pattern APPLICATION_REGEX = Pattern.compile("\\d+_\\d+"); + private static final Pattern APPLICATION_REGEX = Pattern.compile(Constants.APPLICATION_REGEX); /** * build job id From caa9ba48117e60edf6605ed1311b87372768e1cd Mon Sep 17 00:00:00 2001 From: lenboo Date: Wed, 10 Jul 2019 19:47:45 +0800 Subject: [PATCH 133/149] check tenant is available --- .../java/cn/escheduler/api/enums/Status.java | 1 + .../api/service/ExecutorService.java | 31 ++++++++++++++----- .../server/worker/runner/FetchTaskThread.java | 5 +++ 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java b/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java index bd1d5e91b3..18ece52a1e 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java @@ -211,6 +211,7 @@ public enum Status { DELETE_SCHEDULE_CRON_BY_ID_ERROR(50024,"delete schedule by id error"), BATCH_DELETE_PROCESS_DEFINE_ERROR(50025,"batch delete process definition error"), BATCH_DELETE_PROCESS_DEFINE_BY_IDS_ERROR(50026,"batch delete process definition by ids {0} error"), + TENANT_NOT_SUITABLE(50027,"there is not any tenant suitable, please choose a tenant available."), HDFS_NOT_STARTUP(60001,"hdfs not startup"), HDFS_TERANT_RESOURCES_FILE_EXISTS(60002,"resource file exists,please delete resource first"), diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ExecutorService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ExecutorService.java index 9602ac6cef..a2a480f43c 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ExecutorService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ExecutorService.java @@ -110,6 +110,13 @@ public class ExecutorService extends BaseService{ return result; } + if (!checkTenantSuitable(processDefinition)){ + logger.error("there is not any vaild tenant for the process definition: id:{},name:{}, ", + processDefinition.getId(), processDefinition.getName()); + putMsg(result, Status.TENANT_NOT_SUITABLE); + return result; + } + /** * create command */ @@ -190,15 +197,10 @@ public class ExecutorService extends BaseService{ if (status != Status.SUCCESS) { return checkResult; } - - // checkTenantExists(); - Tenant tenant = processDao.getTenantForProcess(processDefinition.getTenantId(), - processDefinition.getUserId()); - if(tenant == null){ + if (!checkTenantSuitable(processDefinition)){ logger.error("there is not any vaild tenant for the process definition: id:{},name:{}, ", processDefinition.getId(), processDefinition.getName()); - putMsg(result, Status.PROCESS_INSTANCE_NOT_EXIST, processInstanceId); - return result; + putMsg(result, Status.TENANT_NOT_SUITABLE); } switch (executeType) { @@ -240,6 +242,21 @@ public class ExecutorService extends BaseService{ return result; } + /** + * check tenant suitable + * @param processDefinition + * @return + */ + private boolean checkTenantSuitable(ProcessDefinition processDefinition) { + // checkTenantExists(); + Tenant tenant = processDao.getTenantForProcess(processDefinition.getTenantId(), + processDefinition.getUserId()); + if(tenant == null){ + return false; + } + return true; + } + /** * Check the state of process instance and the type of operation match * diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java index 2d88fdb843..de4666d102 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java @@ -210,6 +210,11 @@ public class FetchTaskThread implements Runnable{ Tenant tenant = processDao.getTenantForProcess(processInstance.getTenantId(), processDefine.getUserId()); + if(tenant == null){ + logger.error("cannot find suitable tenant for the task:{}, process instance tenant:{}, process definition tenant:{}", + taskInstance.getName(),processInstance.getTenantId(), processDefine.getTenantId()); + continue; + } // check and create Linux users FileUtils.createWorkDirAndUserIfAbsent(execLocalPath, From 94a37429e0a64261acb75a2f6af3a5d99381f14a Mon Sep 17 00:00:00 2001 From: lenboo Date: Thu, 11 Jul 2019 13:59:29 +0800 Subject: [PATCH 134/149] update poll task --- .../cn/escheduler/common/queue/TaskQueueZkImpl.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java b/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java index 73f7499fa7..de37b5ffce 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/queue/TaskQueueZkImpl.java @@ -228,11 +228,11 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue { int j = 0; List taskslist = new ArrayList<>(tasksNum); while(iterator.hasNext()){ - if(j++ < tasksNum){ - String task = iterator.next(); - - taskslist.add(getOriginTaskFormat(task)); + if(j++ >= tasksNum){ + break; } + String task = iterator.next(); + taskslist.add(getOriginTaskFormat(task)); } return taskslist; } @@ -245,6 +245,9 @@ public class TaskQueueZkImpl extends AbstractZKClient implements ITaskQueue { */ private String getOriginTaskFormat(String formatTask){ String[] taskArray = formatTask.split(Constants.UNDERLINE); + if(taskArray.length< 4){ + return formatTask; + } int processInstanceId = Integer.parseInt(taskArray[1]); int taskId = Integer.parseInt(taskArray[3]); From 233ec756cc9f1217452928c286fc342aa96853b6 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Mon, 15 Jul 2019 10:55:32 +0800 Subject: [PATCH 135/149] fix bug 569 to resolve schedule doesn't really stop --- .../escheduler/api/service/ProcessDefinitionService.java | 2 +- .../java/cn/escheduler/api/service/SchedulerService.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessDefinitionService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessDefinitionService.java index 45ff487f5e..39133eeb5f 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessDefinitionService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessDefinitionService.java @@ -487,7 +487,7 @@ public class ProcessDefinitionService extends BaseDAGService { // set status schedule.setReleaseState(ReleaseState.OFFLINE); scheduleMapper.update(schedule); - deleteSchedule(project.getId(), id); + deleteSchedule(project.getId(), schedule.getId()); } break; } diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java index 5ea5faf83d..9f6c67ff94 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/SchedulerService.java @@ -442,14 +442,14 @@ public class SchedulerService extends BaseService { /** * delete schedule */ - public static void deleteSchedule(int projectId, int processId) throws RuntimeException{ - logger.info("delete schedules of project id:{}, flow id:{}", projectId, processId); + public static void deleteSchedule(int projectId, int scheduleId) throws RuntimeException{ + logger.info("delete schedules of project id:{}, schedule id:{}", projectId, scheduleId); - String jobName = QuartzExecutors.buildJobName(processId); + String jobName = QuartzExecutors.buildJobName(scheduleId); String jobGroupName = QuartzExecutors.buildJobGroupName(projectId); if(!QuartzExecutors.getInstance().deleteJob(jobName, jobGroupName)){ - logger.warn("set offline failure:projectId:{},processId:{}",projectId,processId); + logger.warn("set offline failure:projectId:{},scheduleId:{}",projectId,scheduleId); throw new RuntimeException(String.format("set offline failure")); } From 1f9719578e40ec565426e404ffed48f326e265ac Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Mon, 15 Jul 2019 18:09:39 +0800 Subject: [PATCH 136/149] sql task add kerberos doc update --- docs/zh_CN/images/hive_kerberos.png | Bin 0 -> 37052 bytes docs/zh_CN/images/sparksql_kerberos.png | Bin 0 -> 37390 bytes docs/zh_CN/系统使用手册.md | 51 ++++++++++++++---------- 3 files changed, 29 insertions(+), 22 deletions(-) create mode 100644 docs/zh_CN/images/hive_kerberos.png create mode 100644 docs/zh_CN/images/sparksql_kerberos.png diff --git a/docs/zh_CN/images/hive_kerberos.png b/docs/zh_CN/images/hive_kerberos.png new file mode 100644 index 0000000000000000000000000000000000000000..1532934f928d6b3d0d74c330523343878b5ed7c6 GIT binary patch literal 37052 zcmeFZcQl;gw=YgY1PKX2BvB&jXh9HV5GBFr%wR^1PW0YOB7z`#Cpv@C+l&$ty^P*R ziQYTW@5uMu-#Nc^?>gt)bn4g{`g8GGF%O-htBHplB~GcmqN3t{f{4b$(wo=s!on&R!ta2~I(FB| z5vBOpPjjkc8fUb5t@78GqqdiEUMnKp-DBajvD`j;p)f6pn_A;SK9?K%-O9ele5cK% zGcmqbE4~6XCa7EMDo3*yg}zl`t+%no{Mz!KW~OX9o$i0N$e+w9StUm1+z1axg(fv4 z;_B{VsNOu=62r{@bY+gVs%7e#&_4H=lQd z{{t)M72k){v@<><0lB>|2nmcgR^SwQ?OC$jT0R~9OQNpa+S&c4&&@}$$3+@1h4_qg z>!&qMY9~`XD?S2#gvsS_t}e+FFws=rVX_}<^zhU(&6X055YW`J+Pal65#j&o!Oe{Q z+K$H|*5($OXuRl*I{V;h>gZi+3WPB<>bCXhEmv~haUh^sFv?-OpORu-LS<1Q{wodZ zbS?>4mlx0g|8&gRZEBTY9Mx*Xe+4DN#hY!hT!_(q2th@&)AWS^%CLcHU zp)5DOB~3*cypsjz&N=oI2E zgBlvdRsDr!l4l#!#Db0^)W##{;2VkT_Ib(!N~gek7+{I=V9uyj317(vazx;EBl`g| zcJ(u0W9hf^qdV2m?k&`$?bsG1!GT*y{Sh-ef$&XFJ|lu!Yn_0@S^%cyH@I$=DsoRq zb$&gQG29hWH>i_8*jxaPklHuz+_x-?isY&)(lT85QxC3|#GjW7$~8=k4{U*C1|ULZ zI4zH>g+J%zOaiB3qA(XBupc+g55L0%fUEg)2LI5~Afg_UYb+N+OGv;TMRAxXBhjpZA8DISHC5sWpCS^4H`=P#=BKmBW z(bd=0^u)|KOV#$a$C+ANPz$+P@VSYt>(6Lyw>#AVILnrX$eL@4I$==;U5p_$r|Van zuW{myXt1`}O$W)4c7&g`1J{HpG(x(b89Z*TS;dqn$SFq53;+4qd*d^639A;vls6hL z9-^Lx?|i!69Ln6EJzEQCwjv(n5oK(SRF#`mB`yCr+!eHL-!7a3w#=DO(0FK%*tV*M zEfnFHD()NFZqT-CF2iZ$EmB$jga!aG7nDV57b<`q&VAxi)-VlNmbADLT9#1f8*QXHLfE9;8Um9Lz*CGe5W?=x}`v-6CcO7 zH)`PfLu;c_Xk;brkcPD;Ki{&fgJFTeS9N#D>rRvjJzFR4c?kL7=MuP|={qJMpTY6- zUNVg(oj?85CMK~BH6^8f#i|-V>l!N*ald_O zSD#amaE_)--09Tl$n;|t%n~QZyxlf>oiaa<7TpAVMyqb0J3bik7A(6FJ7jW8X=+a| zvio3k!4{p&qy0uRe~8myez)F6=HZ4PnLYMdr05lD;^Lg27@K;0IH*rirYw zR|bk7S#qO!^HEz?WxVs!*(VZJ#r)QT?rPb#!p}<->gR+FZ7VA4+L?njHGjNsWqF3M zBVi@qJX`$zA;M%Jtn#$%4=Cqm$Wso7+woaKzkY5|>PcR*4>0KE=lRtz@sRwGNGgiI z)g$O0O>cw}^SCZ8KMS!>PM3t+j@bd6@r_V}vH~lB)+$!F+>Np+00OkqZE%>JINsvn zyY$-G3qS<89{Kdi&Ptn&s88Goa=hL;oua>|&n%a$M){G>n~gU>3nY=6;PMg%^)Oh& zIbe<~oTU4(lWR?cJdT>h4x`x7(Gfr>5xCM-9uSs5_yRE*GHo6gPzIifBNogMEHN7rI`ZuDV z&wEffJvJ>S)6RilsJA4<#;*4~sn}k=Gh0DC&umP6W1||H{h7Gs%MfoaI+@>F&3L)z zJL5J4u!G_)f&B`%%^m3=QELwvtt+?DFN2QtAmfY`;tv*`68p1TMxHj5zZ_&Tsu3*v zYp1|dnUsZ<+2Fet?B=)*0qeE;p+At z*m4%FMQ)nF_2q1Qm9o0t#^;nA_Bg1o;6Xy1J2}jv>c)t#=WI8_>&&!1Bb*aQna~xS zLT@_7;)f-hy_H-=__hmPKCoE!QeQDvS71%T*4Mnu88C zajx1>o9}aa?oD$F6${m?w8RI{5-|zL2bX$0PYvX^nV8OveyNa*I_W5_g z)*m+l7Jz{~t7(jP-a1UV+p8dlkq9Nck^2q2siG{_6H5qLyOuf};3#9I>AU`Tij| zZTG!+=XhD;dtS2y(m#v3=-ataI%0SKWLNKkxIR5**|)u35bwBl-Vt0rqvQ8|bG51O zC3lr4rEKHasd9hDbIIPzg*J`P_q9(1Nx(>W6~Mt#_wrI@@nnYVG}*61dE_Xo6|`O& zz1fLhG$ZJMxjM9dcJ={k6RGu2d0MB1S25<`k#C2}$DsV(?Gd5iROK&+zZhL4d&)z%&K&4#KD!PdwYmiBuEzOg zGt!0?Uc)3WqX+2#diuBy^F^9UM!|BqOXpr_Jcf|=zX0xU6(q< zHtJCBCyt31O?0wUPjo&qgVe7RQCtKxznz-aW z2d=bs5|`qK_agV(MNlSHGrthmYP>s_;>*aRlwxQ*is^lmD^aLbu?VU$b7k~$ddZf( zQLSF-V7hRM1x?w|t=sJ z0swaG?NK|+Rdcc-zU0%3+Gf8f>hPp#^#0*|Z_CKmHg9;Nr=9?M5TR)^71y%Tv%#Zx z9$AyNc?CJEpGxr%HH!_G8aX%AYcI?>vHw_dzvLwL{Ey9BOoYDbX^ODYcH9Npd~Btz z-NwBH<(3+C`Hb7Kr)O{W!c`uqhlcqZR2da$j#eO_&bdnaxJfOxw|8+BODWGT>8%0l z==scAcpV*}_$-mGCBfPH^(+n@+rLkA>^>T$B{A{x*yuh>h@YMWtwR>J?nd%k4= zg55Lnhe1(e+#>oHzo~lDub5=HD}1l7;2;`v?bu%;)PxieX;bi0Mr$1>J%&4w@u9Np z`X}{_K3Pju;C9Y5K?vNs+qeU1Yh$Uz`+S3V{7mJ+(VLfv=k)#hcmrCx;?FF7>xtN% zc!GCaeRsj*?VCXx#x^wS>mLt`NC44w?TcW&&E(XOdE;%DhF&*7+_{>a=5Q}n@!2<5 z0(vxeTbtLmQ@+P(9e3yTuKr4m0u8Cxz{Mr^^$Lq=;$qW`*U~7b6J#smG(i3Qkb8tm z3nWQtYmzO({W{Oso}ckyp;(k}eMA0=h&{V;01Wg4?gWixzAS#Ga?QjTxz6=RO|$)a z3w+ebp`|(R=qbkKA7JNs5C=629TV>Eo`$~KIU1g-TfvkhHJ)_U+j8X0#>Wx0=MJ8~ zb=U|VWAz(7tLWVDw)7VJBp=7W!*5G#r1_-vQYgz-z5RuTn`qILCip?Zw8{^sz%n_r zuu+#R_NRrl$!O`$VwM9Xo#o+kzVpOIzPpd$TaMNK3o4HvH;J=kwCn_(sI4Cri>aEb z59*eplijVgEMe{?M^=rS2RmoDj3;Nq?V^A(fX}0X!^!=#hrMbSS<0e0@%QCWvpX>W zJ%X^$+PJRe5z8ajKJ@$hW71dhm*-VG?p&slvHA3|qAaQFNSiXG$8P4QpWad1BAm@u zJ0T$uhaEFZZvxl>otKdYvc;%8n~UqKQ$hQu-gKK&2Ypgf#$n;f#p&s9*?Huq-a^P$ zKtc7+g>G!dzR*jM@9L_)M;Cwm#`z(y$Ao27Wld=QgV<}4V8oYv(g2q+#f&e~(BZAV zfPzn_w*=_!7E)_lmvKJ@d`a!SgeShFTgRvCUk??|g`ExueLQzffEJAHb*a*Q2I6Wr zWP$XT#7yIu9AE#2zV$8x@3q=C;sJfENmIVWUcJ9ZqiUre+hZAgPILdy^xC#BGN+3W z?7P?G%b~1yaov;NShV(cP?N;6`9S)_R*jjfHJm6RuU}a$Tg8L>TU*cBdunZm^(zOt zwu+gs4xE$s&5p^)0)WNCV7SAEJmlz*lo8G`8zyV+tg05P#|_Ru^|Y13Du(mH>PRE? zeeY;uG9M{$Z$t!KTFIkcx!Xs~t)rgLA`o7Wrap4F%Xn>^o2!9iwKZ#3sJomhT^-mB zRu#av!HLPWoC_sl1KLpPvYCN|%*g)vXhCib`ugDjeWt-S{Yb1F>+HA?< zc|677d$~2ebkuaHQ)DgSx=~?@SMRn9w@~E8+>S-uNWU`ey;$p3svYcJHQ8vM6nE_}O%GZr3>i)1#`eZ|z9`{2xE<^h?#x7QkK|ID0$oFaQUiLCS#>@Utp*SPv0;)7q zeZP_wl|LL-bkB8mr(Wz7K;N1fytx%kMKtOIa-DZXyhf`p09`#YiOdv6Vy|odfFQHS zwpV}HD!%d+$e65$s?nWfPJuRyJrE2*;8i=MO+p;wiIv`dxT|UnwMgl&F50IUWaj9+ z)2uy`Bn|A^plPbS+2#97y9Ol^9JA`67O*BcOv_X9)9)%EVne@PsS|`H&x~~+M9fU4y}61m<3qf45c!wlr_P14k~JYIcZa`+YF(M-j*Yikjx78E zdtCwXd#vnL3C4K4HKDFuUiy>!P+Hw~1z(6_n*;hIR^OJFd~B00^}wsVuM?X=ie$vB zeDxmWJL(1H0ZHI>E0uUZy%_-Ju_2=DbX>B#26;rFz?HP}Tvaq|yB2w(1f@*dsEk7a znLl=yO%U4VelixMS7!h&jX)B&EfTUnTO;l7_0kUPgOh^G2;5LRc7RmVl#c8k z8m`qA&2=Gn#x#+Ly<5kN6gjKbq`qB^M z-MVIz-hLSfE7d(HtR#YR9{WJ_>js-P;Uhp>ZWCm4tKx=pesg1GcayDrJ$Fu*k$USn z^C&#RcUp7lBIA3+2&jAAo1A>_iSNSRgzBLXIKal@UZu_A?N-`k%b9js@p~g~Uq>Kp z!X(?BZ;0Bj6X{594cas%6r;&8fI(eRCC8snyaPTFp_^DRz39^(_LEj!A53h!mi9s+ z^DN<#chmW)65F}3xr!6M*Q06O$T=%brq^3etti&!I-OGOq|#iebhal?m$afzyBh}A zcox2|I0R{oC#6NsELbmBm+>*=RTOI!leu%UQnAFr0m!r{4Ro4%FjlLfct&eK4SPCL zBsHLKMMeLFA3po$;bk+n;)dUYGuJ71xpLMr@k=bp&iKre zD%_TR#?(A(!Bj(KeO9^#@z#C&FlOE3SbQt?`Zq}CM5JcT(U8y9lK0*^gWdJTLD(!S~vPnR7h9!~xD*JqZrR=wunSKC$} z*H5*VE*shQ8mCvw*bW+bjMPXrI@vEG*o%(#%8;*Tigtz6>^ZS2?X?jH+7oSztg{|} z?%Qv~Mj>KqDvqwC`eL)>hu;Sbcx3Q{1B$lNg-}dB7K~9rKvmd2IX2 z^*jVwJXZMyqaG0}NCeG0mRYRSiTQ=;k{PR5TV<1DAHXVpL4X~@d%*AnA)A2Y^`HoN zJqzahqcWfLO6zL;&e3B&^#RunzCy)@2u(2>g`D4FN%~0wdxqvf!KLuBcfoxSp~v;O z=_1}~0ALO>7j<#zs_!kC_NRsGAdh=v>Q%#6y=tFL?$sSlA-&$}0o=|ES`pR{rW~JW z3F4_Zikhx|s%1odaDL9)6o917lLGx%VWh0`*|9$Mu1GXe-}W{#9Z*mIR(8Lhv|;$7 zqVve(N$_k`QDpBKvFmxs*Z;Xw0h{>r_s@+@baic7g&Fi48`Oo7Zjq8P2bmj5p*pNZ|%)15!W5D|QiLLX~d>z~xBvbZ2 zt<|hg_3%I$^m!`R?d1a2$JjW8nR)$+Qu>8vdY-&;0mAHh=uH=)I*IpeQ`7a+(`MiE zCB+XJ35h8ZzH+VLFHX+B0~sv#dal3&H~qzE!u0-iPO);BRbaM9+%!V2Fb^-{c77Qdq?0$p0*V`$(G@Y zt^--3GlF$d3EER}N8_OfVMi0$u>JQ+ntFMaXGaM@+14GjxRv6g`sL!2s>FR%h|!k9 z7;Pw5vW?ov$V2S)IOw)3afuDvs(y95&vMr@)Pt_#70I?u+v8}^wd|Z;(8`J~IFLi_ ze5I&sL8ZgpcfFumA1bi${G2#mu;O!Dd7kP-kJSs8=PJLC-@5l|dR*zYyN({Q9zFR& zQGaK@4K$&9C0`<L7^@UWo|$3KTj2O!h)9E_@jzFwx?O{j#gY#q|s{y!Pk`^ z)CbC^1Z}GR^ehn#NB8}7Tt-*ZHnl_}vJiUSN8pJnE2?H9k)SJ+cgk|>J!x6zjGn1( zB^fymX4EHqQ!4zlAqyk2)w^{=D+PnRV*p0gqhK=WO3{eaXO+M)K(K z4{eQ<-#P*loN8(*wNj(UPd^#Quc~&Z-~Hg>Nl~L+3KW{=jy7u;eJ`gd!{pOlw_Mh% zp_93%acr6It9C}LX~wPYJ_{dj3fOW-R3!0Y_t~ZaULjJO;#3`6caaO{YMTDt(9!ti z9^JE;<_SZg;u7vcoL5_T-#z!Y2^t24RNO&-Ber=jw^!}>qNe&n0gnrubyM%cpL&=G zKXo{C&}4kA%6yo=JDLOGf2Ap=k+{oWZosu65u0LX)5N+qKhJp=tc1f_@dZBDZVJ$;na0`7}i5q(q+iBM4DeIWSOn9c&Aq_*t#a(F zNn5Rqjk6RfJ-nG+uh*!Bu~VnLgIDPB*jjCE6~^G3$+hw9AWtz3mz_!)`77i_*77wo zP6IQquhCn|7DRPU^bjwn_n_+pQka}Jb|$)^+^!DP@j+9N8#qi99&YAbY%O%^L;>DHlYJ; zM%NZj*VA176EdCotx{)86zlPFC$~7CTiD8V{H8fNBgQW} zXW25!xYeJjnJ7@|nD9;3ukA#DE3*2u0>dsN7vNu(5yu^%2j;;B>) zFYe)_@F0079~ANf7ko?erJ6rWMW&4n_dwT#!L4(!kY+p+)otQ%`}N#xjopyVuNdqC zkH1XKbJwK0hVnb zd)Dg75VX0wr&AhIrfAbkmiV}ai$tN@>otdw?W6Fqq{v3Qd2{x{SP;Q0T`ux^C%>ds zjis)@7wR_l1i90-fCYzfoBKl>=FuO_J*{;ZTs6kkV0F+}E=4oYML!P@hhnx7+0I^= z$8`U4RP(xg8(vM_!GKP@@X2F%oK5gCN7A-O_vIzSUM&$rB%lqbtzf5qPQePUABo|h z4D5VDbOgGd&tK!BUbBsFb(6&J)G17~7_bB~6soHR3oW!9Ug@$91Yl!Ir9W3ib3Rg3 z+#3D)4*6o5R9f#UsYAcOx9J#&ZY`ge4vbeSYh%&{x~aCzi^-j&#?4i^f}g2IF8;5$ z)`SlkM~+jQ3VDrJB!-ffD@|#H&Ei4U%0We?Be#}Gppb9f#aRnlK^ANB9*SBSqwz(S zWM85GaBod~GJZ#$^BWONACi+>^qstzcXhD>ntph^h>GSFD!Ag3X0!uEWOONkpZk+T zLLu@&fFs-09Dr2NnrVXSM4M;Jy&=rbIi$Z}Mfza|ZI!YmjF1L?`0E+4E^N(!&C<8G z_GQDh2gkUjqAoUNGeiW*Q62ffW()(`{*-e3hSd zX1^Qfbgs+S2pAXSD&dHb`@PN1P_hvRH{7@qA*9|+i&r{%jo^Ir`eF1H1*V0p%0?xl zpvesUo4vJrJ^14NuaZ7+qbn?<>kcBz zLXM7DzR;#s9=;`;_>%dQ?~*$HoY70jH&Lfly?F--*m7h*I`slq7&BqD8mA@j{iZ^y z8M?^%dH1!mfhg%a)u#JLk{-9d$c-HhYe?QI#`j+9+4N_KB+h|;w0yyD<(ld!8G}7L zg_FU&oT@7eQ4Zn=yWl5*E&C49{>C4phxm85ggQSVH>ypr;m@~Hnm??E1q_^%`P8G6 z5pF{Oq9ge{;Bo$q%Yqv%TUz2&Qx{80#pw3ZyviVUbu?aANtx1~g_F3?4XZTnn5Fjl zvqwFC++A(9)}M#E2-6|wduqsQi>YfW{7ExmJ)KMwdARo_@$NvQ4A~IhAWY?IIn9vz z?7MQC&AG4_e`dQ1e)eT4yN_|+1Mb&NBcCU9Ni_jI?RtOmFP8)?mHlU+XF8grLKRT? zH1)z6DAci?_plz&EYIv3&V?O+(4%Hc5zIf`^sOU;i1cITbZis`2S4%CoGA>9v$jX| z=tJK07`44YFOf{VS#e9Qv)=sLvG#!-Tqvza+|^3R4o$QP0lW@7!3QC#`n!UfKd=rv z^$+shqD-Hw)n@Q+FgY&t;obRV!jrPATqQS!`J_|Xu5ceH2*|=1ktsAXSODYX(JAp? zJG!LD#p-N*y3t*mGKoU&(7v^BS5w~5mI{TV5Fx7(UDWyP7W<&96h^=>wNi8DaSf8t zw3|F~h?g}Og};z4ioC9K9{-s)d8epMQ>idM5hrW-V-FrBML3#ym3$S+l4aMX7P!eH z?veRr_%vU7i$0p>squI;a1~@<_0_XAr~&o`V2u{yj-wS^D&k&z;+x&>m3VzvYFrmQ?Ehe-(*)7tS~0aaEUHM1GP>+(%>5ChYAd>>4l^4fx{y(pFKQ z>)0K!9}u6m`a1-VrBAL9_}NnHa`p+RiopmloN|6pdCw)HOL9ieKfdTZWl!1T>+n%u z05y(deVdsHI@!5{?cLRoY)F80!?G1rZ zqIj53i>K}1$z6N`RacK*)n>oBaELpls*Q1O1R4Z>t~Nu{y6?_4yHMS*SwvD0>g!08 zh6hFc{m0uFb~@jSGOztkmP#FLHG`i{G-_i^&VTDJl_86%^ZhX>R}lS!Z|5gF$$~%6 z&dyw@2ydGCvVD~Yga30cd9yfZ(Sx3Ubix{p@WU4RPvKquJMT9U)~>jI@?zH)j&;7* zjb6tSX622I*qa4*);~xFhld$Gk2fP5JWsl|F{q7dngEI?NhIJzg|}%Q>um04Yk4MP zTu-#Ig;g4=s`#I zT-K{izu$2>X=_vAd~oB<51(ZqV}J_cHNz8>D=5dpg_%)pzVcs ztckb24=-2+(vT`h^-KHbxg-A6MQcF)8nVgF8*+NI0O!8pyjaecZE@ka+`_1hU;{Lp zg;>sD@5~s}{KF&kUxuk_vxSy00ir_ zl!+M|f)o#`$u0r;9z$$Rmy$GknYGdy%XkI~)OiaMmTb$evSu zxo>~EGz{C1&g<(Etf$&PJQ19Usk!Q7TcvTgT3@LX5ijpP|E0ME<96*B^VR1(&rRso zKh{h3Iz6o*^H2_A_ngHJU&U21Pbn3u4}(R6;<>C{wfiOFm|L(ngt}4oAvu~Drypu; z>s9u_n-$2Du#VENC$FeVBwSx?(#8hzPOW;sU|S)H<$foe8fWmbHluvw`+m*-QGiMY zEM_K3|8s~`gw1^2mFkuCc|_q-5;z~pyzk8N>mqXwN!8F3;mej$k#kFH@3^P>umv05 zJVH(MXUCt3;Tu2pV{JPoKIF4v z-sed55zuYLuUtt47SydQZqj+tt6$4q7LLw0Jh$SAF>|dy5&tHlEc#s8D<&#X-z%qU z^o7XR*!o!_zuujt9sTxhvyn*G9ZxP+tWq?v!fxjP66vgRyPxZK#)U;#X_P|M^+0hL zVCk^%9c&~dK1MT+fC(tdnL37KyfAzd3JLSdIWO|UvL*o&Q1HvPuCKa8mulOh=P6!N zVh0O%o7oXpS#y0MRi|(DzMwN8WcENTUxM@sa#oPJ6 z6*>vpBUY?mG_hxmv~r_-T{M5z7sHoIuH^%TPBnb47T8XiiZ!qTgoPX6D}%@Z97p#P zDI+20)qxK^Gl^iD1ZFOS?w&vi{PXwEtTCi|Gx7S+Q@U*QSY(r$P~g3xlr` zSbVbL!!=eBF~yArPlt~{bJ|~Nhso$i+CICM$VG5=&`~;3_gHCnmqSY&PS-gT&qP-C z$@fuuuvZm?Vi_%odQE)XsLAaK?qr(?_3P6@6H`(wAd%q2E-;m_Xjpt*laD%rYM<8C zxJt{N0|7&5%uQT>FboB|@1PrUBol~ux4GQ(g#h}B4NShC3#5hSK=?GpOW%*;Wl(sZ z2Yw!R@}6d|qdM5;DR#C#*k-G`baCU2H+#OGxxT$ax?BTUqX%=Dk4$q@F0x#|(a)cL zA1ioS2TV?{EAFCX=WAQbx^(G|oLWX(gH>mKJ!7I=6GL@U6JXVNsVW}3n?tVB80OkZ z6v=`>__;e^+h#tKo{^Q`|Hj`4(6Lk^em7saS+yJL36RIAX2@#vyDN$PL3MFLCUD*( zZwZ4QRcozrx#R-bcBXY8td7bysfpg~9=AndOzcP^tMHbef! z-_#Jd{70VwZGRts+;I!UKF&vuVbOt=mDQB)jWONCS(1$X7Z7T5;sq(K==fiH=J#f> zWx?QIyHAA$5R+KH=n~d{8K*3E`YG;4I58UI7&O-FxeGCZ%mwBSXEMgOb(sbFG z(XH9K!KY67UB~YJkG20dB=H-sIU*mMxHEV(Zy`rg84HsqyZwTpBn)Jnj`YAHGfY5$D)zlw43Ny-=_@qH>*mNQLOr%iV7Tm-*sP z9q5?xo7^}?n(sow%q!(CcRqLYkJaxq0h#Cd_(qqEqyV+`^gAR@ovZgxB(W4)GXe(E zUf_ZQ7(cyvVm_+5BYGl@BF4$?30qYPz$hN+Xl}zU6-$oO#gPg4Q#L_K@!&l(eZmh% zF)mz|m#g0nydzf^v$Y23nfCIbZ|iT#QcO8kmyL6kb=t6nymZO8RDJV;(xE!9>rc4r z+(5?q@Ulf~Aa*2)Aj>LIoATcWsr%mtR7 zEQ#5v3h>WnB#1NS@KOzMtV8(4awux~7a7|PW>^Wc{^TbGk7&i3q~1P8-3DR#L=3?v zDaK$}A`Ec&EIfvU!v8iL4`GmfOW9NgGu>FX;$4V;Tsx8qmS@bih=B0=qVr6Ph?$Y%DCH=(HR*&zk|t zWfD@g(a{7}b>KRGar{qnm`d@m8%UiNbnI?G#+sf}AQ~^J80f5R6hY+PypA7a^@J)< z2RSZfUFmdQVouv^rUb>PID|~)Bc|k;wkg@aq{BC<}{P?-q z{(WyuWn&$4Ne8jZx*ME*;E!Sz!SgmH|UfHa$?FYf7UuQ z=)RTdQ<(y)PTW%^20Pt*MQ--BHn*#Gnq@vfMKisV9iGMFm*AwUtjNUJ5ABcd>!D>O zt5bpV=L@J+xQq$0NDDQCYT&7?LuAfkTx=%`MDh8mtQ_2e)UpDvg>-MvrbR2{qikT< zyj=5FSfmC$eaV_f0A=##be(Egs&snx;YbUr?7+j_RZF^>iDdUW3M6aaBaI_Au~ z@eww(Q;3(Vf1yg5wftFx2nr_SyNLA~zH)e4l_z1#>5X9DNYUnGLKlE-Ig2gzw1HyVyAOxq4ILsrM$1)p7+YrugNF^MFX5Yd`?KCUj>cP5e@#nL{BGZ11r zu_%#}I4e&I)XAx}#Z>>5HXmR)Oj|vk%<}rymdiIH8XhA!bA?gY;o$}{wy;Mb_+}41 z;Dm@o3Y{XhEo`d7n;+qD6~Ob*G9L-qrWT;v@!^^ zUSgh<&EH)}I}$OlRnJo@F2S1m98i2UiX%Tdid-`$YO5`;-7xG+LMQuj4xe^^ z$dd4$-Of`vp_Lwqd^6no8SM=AIXkdX(coGo-p8=1TiE6IOU;PPGqA~G;! zWBaeOXQU_}pZ9-~#*&}Ik#QxrhK8{+?V0Ob&XTPL%WX<~J%&~McP;?EEk!{={x91J zO(+8ys-M&U{xsFX3gMOLDz*m+NI_bgUI0lcWfd&1QNL zT2pjf*dszHS0&B3Ua3;$1X?WhnD%sesn*P4&rTn?Bh0B{nao`fi$Pjc*@2OC*a46- zA#aJPY4{0OA+CH__dRWcFT&tw*BZLw(0b!332Yo1U+|cbshnx4DE_OlhSgD$<(YbdKfO3UT=yq`NLUP?O4(deNS`rM zUNOtnN6Wq+)uw;pXq(6Qee@}O`R_iAd2AFIbr`cTL>AY`XDCNXnmD8fHhRmczcvkN zu>HfCD(L7i1FPHEGatTqx}ILyr<)U?T;!{riKUB|Zbx2q`e?4Ij&Vyf<|@DXfGAXP ze$I?RjOO-FBuMwQ|I(`>QsfCvy{q9Zj+G~^*#F?}JtPb`>h$NgdQVi6h_fU2ULu`C zE28R2e217-(lI+hTep(2YA%Eo#Jy0u)z1#-@J{SqS$RO&@ z$;~8d0^2Rj`y`6OF6hxFFb2eBX}Hz7ynt%@sx$vFC^^fPdO=2Vs*#WQkqkdKVTb34|Do zu=*SWBTEW=$F9qJBtg8C+tG>jQ&={byV9h*=gA$wEjX$!vzCT0mU!lM0}~u@+c=_mpR=62nt0PqQoQ|u zrkl%%W@Ghz_Uo6``BtH9o%s%rXzY(tm>{>9@7}`kJNwQ)=W@Op#Gt6i;Ctlz)OU!*EZdg;`AKztt%#m1LAbC+qq-ot>NVrL^7v zeh#T=k!H-)(=T+D|Md7$bRq*F?p6oAHNqR6#u<&f*7!Y6Td`wN1b@TUI^N|-fl-u1 zVtF)E))O`S!V!C0Aa1sta$c{P3%MA;($@}XC`tn;pvC)Dm#5#HrW4{_cRXfivFXB{ zPb%+?SOQ_Kmd&NtxjOuc7_~X4Cf;}>-uSUFP|Xu*re%2La`DDR%8Kn(!O8FwQo7F1 zw3x|n-U`hEJ>Hf;HnT*w5>|uQ7f*`CShG#Z8Gt+}1_VqXWviVDf%0FYC@`4tea#fy zBJZwU7{#{f5#L~?L{@W)?9|2^x-RewFDYz$YIF!4ZKKLDwWN-pp%6Mrhc=NUTO*Wx zTcfmuq5Y=f`NM;A*;bXT0-#AVb8L1O3vQk?TcUtd@2k}AH*xI`jd>#!Tq&N#{M!0- z@AFSGTG0wuXgJ zQvqhd^z=%oo72`2WSWQ~P*V7=iSdN4NEIxNVWLd%c>KGZmG@;a;n01q2y!kD2i+wy zvR*Of(HE_pY=yzoyfVcg50s@lFg#C(7@QU}tzl9MK-cj|nO;(nbik22e!3+#7U0jk%H|$PQKK(}*PLnaxM; zbxF`~L;@udT`ip!)Uzwo)%00IsSf;!uZlq@rYoVe{Env5pO>Serp*G=8^_4^11!P@ z-xPq}n#cmh-iwQ6{Dw+Hfz{i35z}Bp1Patf!nid3oM$0_Iim3V&SBK(cq#VJ8I0@A zh$ri(i*a4p+cLM~C;&ydpBFm}BWte$#f_EA4WMb8FBhCXqP~R++}QCx*gX4`unCk( zhhi+wW=%YDDx_euk1Gk1uCr9F(K@va&8rcq^M`yN`C0_$UF9+_haZpWme~?6I^z;= zYK}pOXnjF7Gj(yD;nah===TgB_hN|>gX1o^Zup9MQ26G1Na_8W%w4T=(Wz`ncc+Ai zzDZ5dwwk4)u5vNs^X{!sibl~{C;W{rU&UbeEg6z`iiNrzr}lH!N0#0N`arX(89B1Q z+YA2iUa_Y>m*KL)YeD}&B5pf+SVGD_w9&*C4e9HEUaq~ zIt;82{wTTsJ0N#2^VI`8usL58^w3 zcUTi|w>FCX2MDNu2ng8dU8FY^6{HJ<-g^tZcafqXpdh_VPY9ugUKNmDLk~!=2|WP< zSk|ru!}B z97G>~mmhU$(M4TY_Y-yL-esA{&Mi9!WoSbEVOuX?f7*u3Jst&59JiBEaahD#<;0uR z+i5>=dJZb2VXm42bIca$jUkloQAx4~%i%+nD$m#6aY3%;$+J9))8{N&eK-!cIXr!$hH_bw zw3}h5B>{}K*=OoKlpM|B9q3qhIO(vnD$YnZmY*83XT@7=@zAbogS@bwXruYL$`d_e zGIJT3by*r^-LK$Q7ns#!T|>3(V%7&mUZ41tn3?XyV()T(Stp_rT&yeZ;&i~oESIF} zTsiBDu&$G_gin~2T2?ow>a3)P&?Y8;(}ymNxGs>P7KtaI4^*Bv=60Sj zcS|*o-GiuYmgZx5C9n6S%se_YoDswTpl_7El|(e$@N2f2z_4)Hx5h3yvt{r~uQsM> z+HBaf7HHVzQz-{DfNAfk;;~03VS+e?msjLpFpmiT zptp_WG z<>Nb>5`=X^)LCMPA0HgUPTbdAGNj>qiMj3AQFC*qMR;P)r3H>B=Q(cVakm= zOD!oBaq;wWjB*?ON687dW=nq3 zTh7aOY<}%>J``{Ua1{{(sEe4miFzHq1-N=5;1NpA>NI7puD=C*dj7*$^EvSUZz7Kn z=<9jqKGJ*J`<(UjW6n1Zz7Q#}1WCB)@Ym{_S4&>4)u3!QM6gU|8567R+6kLiXHJ<`5R z`+^T!RXO0=zm6G!IW(p_=mu}IGLP^Us7S9?=1NGPWv+?>L%Q_0sosMJ!PG2O#l>8c zJwV=@i0H+HR32I{pR|=EPFGnu8t8MAux<%_ZyZtKFF#$a2;@Flsu~(344V9OwBUw$ zE*GLG=L&zh>BP#vJ0$3J<|d@4!`j+j9^!D~(l=#fWRe~30LRF`C4M@f39-){#&VAU z>b186=S01k($XV+7iW$&WqWY(|LWiNenCNhQ_Z(-{cZ1e`ENM#{Q2LM?>`^^R_gq> z%lLo%$Abro!bJD_`cj3Fb8~QdVFzn#YqpU-V4hm2V&miU($elPFE6u;i;IJ|WlGy} zEoDF|Dlsd)$vL&ItF}|mA})$!?$x)pzA&iw=v!C6d-rZWxBieqd#)uXNJyhEO*HP0 zrtH5@-!wiKJn)=kVm?s7viFyp0B{7p_R0UcCn@RbumAK|d@(6yA-1|X@`vR7w{t~$ zb&h>@I&ZM!NY}g!cU8I5U2pJ$B@idQv%odN9p{lx>RP~8S+~sKPhbqff9~8hv8E*5Tv!eoUJ3SDcWW;!UYU=z)2$J zenLJ~xI#9_Cq#A8w%SLKm-Q;`5{@#70fAVRzJpGJ+GMo&@G=}YXB@XaBE$N?)1>@5z_Ok=n zVdsbgrrzeca~v&LIY8+HvDIuFO^?v##DYIbsK@#@m>Phh4Tq#wLU}KHyL}Myy2cTY zgp8RFM$-7)vDeNIJTB}~+kW#Q))oJbzGf@^hnQcPB7GZ@2Op85OMTuS#tU5agNW=P z^~vDo*Z_3BVwY58)5v05Hv zd5-Vj-g?F*EO)41V@-fcZjN_0fAHy5w6d-?`}y4R!+3PafO)X^eW!tXhH?UD@nnfp2?tlX**_|f#qY$L8U?^n z_%5$#qi&xZU{hJNsiO#j@jac49SII@G_BT!oQ$5N0$sG^Ugr#ru1%^j(i>1E(ftY^ zw#z#)w9acFu5D>!b}a8!Y3*HYg!;2<6;`vb#r-GWm!FZ(` z^jYVbXfZC&?8w5GpyrwvR^!`4*u!vCcv|**i%(dxiwMi>F$m- zsJbxS|FmJ*!E;Xy{?SYtb#ZZVLr+yH3SUT$)zi@GQ%x3iw~G0S0Askmx7<_bbI1{Y zp@{jMik6vb%Kf{8qRMo}d#Ydk{hB5w*(Z2>I#XD7cDADMh2U8IBg45BCkUYeG|O6l z30P9AK%$j_%*;QYhv+Ro)rB)9`v1TK{13jz+9#i$=<~`-U+Tv8_IMP%@Uv&nzOAnS zV>Bk|>h4wp9OvHN-naDhbOi39wuxFrHl(0H)^(+KXr|Ux5KQ;_qH*nK)bHO}fPY;% zU?(Ld9l2YwmS?I~5zj#nvFuO#TQylTDV2*>nEGS=(f-2-s|VM%cA9AgEY0gKWaUW< z|8k{${YBr^E_`v8edg2j{loRL3pn!cyyeep!TcVH`n^_s`*U}Ly$sxn*Pp?o}xALdKRH>BP24r}A;+?Nb}*j}x( ziaR67tmhgv=!q}bjbKFs8{VM4Mrn_Kn4SU3KG}HUxKQ=@jl1-P&G}P(q%ZRP(l_kC| zs9I1LHYnQKOK>20wKB;#cHigZ4TZ$5q2!K^4rJ09et~rS3QsCQN|yMDg2ckb!<-E+ zC90_goS{KBXH>YGTff{T#ZOn%CZ0#y1oo7Mqg1Qa4)Y9CXTN3W8kCrq6;;FDvaZ~M z0HbrD4yKOv^=O_F=82j?iLic#~S`2vDE2%j_>B>dnij;Xg)8<_o-c(u!Pq3Ebr z@!3fcppt~7-)e>iyva_;!ysO_$?+yk6IOL`{a24G4b?gsWl5}NMoci`_3xGpF+;-p}7d^ER-RjK|_e5G=_*WA)@fh^?B z;YRoK68QzFBXALN^uWA}eS&0Kr$-kpfJKDYfj1dAe3L$Dv}dsP2o`oGL=!iZyJ!*3!&!MK#km z#I$l_Zzv|&^1M5{VqkkZacYPCdKA^`!J-e7rz#PvcY7$TRl6!SdSbJ_bF4dyXYfab zR5`+)4&XKl{ePYMwh_A+PQ;NJvT9%tUb;8TK9NNCpKX9Q`S3=C%jaiUgQ1qIpGbl>j+~tm+(`xz52}HbIOL7ouJ7k9R>9FyKrh~ z_JY?|1DDnN2-X9SqeJKJYKF-EAM)Sp>-E~(+ZV0j#N1H^97SJrWBbhZDrEM$OV$QE zo1TLV=maGA?u@2QKjw0o*K1i|Z_JiCv*@Dv{^gAqMeRJy3_(A;1)eVH+~WH&vZq2{ zA#*p;$YD4eM_Fw6Fsl4F5Y4otTb%m$3`zcnr=aG1GG9!zR0Q@_@4D_ z+r_OL@2ju7#E9{Zlmrs zr->45K(fc9oD$8B<()%P_wwCs^%LvivEA)G2@ zF<2H-HiQduBJcaPU25o8cOj;y5A8o`9$!&gci~97C!?I)~IAZNSlwY&h{(-Hi)v8=OuO)x9ou1NYnhUIiS7j%JIAnX()`Lz~4{#L^ zB8)@7whus`l82e{-fvi0u2~%qU9vATg=W018QI@(;8Yoi-OR)$v$?m1Jc^s%P#8Tt zU!#Wp8kDeNGxmx+iU1X9iY*@cq(%5n&W*N*eIUWMs@NJdjo-DiG3UlK*y$|u47oUJ&)Qgdr4y^UHmV*@-ht6+ z7DWh*YsT;O`KAtzZdZXF%H`vXVz05KEpp23=x^;#qVMqH0MaK&!>ra_(RN+?4ka)Y z&wMwYYANnK-wU@CB>q3R|9^#^d_@)(7F{K}7a-=U37+(Z_S^r|0(?U_Ci7h7RJ0Wp z6|M6?>*l8dH!X<0|6e!?fCxbAd<%m{pZM-SkhTx|5NtI3cZd9fi!jjK$_iRqs(N;I z)=%+ca8NZr^+G(%{*#;l0EOv-O8*`=0ACDONE7X^4h;N>7K3u{kz4@l{~q99eEgqr z9x$$d3l#ou#pKT~v9T{L)XR?ELk#)WQ;~$m`hkb% z6}n~{RKkVBD%a)al2WFC2>azct~V>Ad9LJ;_Zifx0j>iynnb^o-ML$M9C|o)j$5?3 zxDWZU>&Xs8r#$k~T;DQ;7$?7=nA&ysb}2(s4UeFx$4d66^`vQws5}w~OKs zjy_jjo{lccW|79cqfhgAwViCNQI+F<6gdr{n!1gfsK6teoNW%8h#f(esjxnTfqHMd zC0gvU;_*iscA#^BEN`nTU#i+$WONk*tHu$B}4p|%nU zO{bJtxPej^%>)9?53}kU~{QKIPcVSc$NQ|NkNqFrdc)cfVGUiEKQ-p z#NC}R36f3-K(aJ8nf`>KcFNz_Wq^^b?hhHZ^bqTK;8A7M;^+jMP0SMhpKdbHP}b7*G{#k`PJ7qc ztND!%w_JjUNc$Z4PG&vO+z}M7S#C0X9ot52UT0-B2ajibZ=B-8NN5yp(BL4Svx9(H zucp2WPvYul5GEYwRh~=_2l5dD@N=v)Tz8Zey^6kxYi$3a5;jH+a19Z2e$zT!-?u|1 z=N4+zl%|zIp&ng4{zG*+9OcHLFu&t%3L~e5ZyvDCu`#CF_rIkVNK7Yrzilg33je#Z( zJsy(nV?(<%B=yt8K`phM-tB@; zQHd3RnJ6Pa(- zxc66al@0R~I>5XKv4XOjnCSfl3ybq8h{_;RBKd2Suh9s&xY95yf@x; zc4gFn$v4xJGFjT!|DYiW6 zTDu8m9J5SKS=&<7!m{D>=~L*E-=0o^<)r%=!@F1PG?IofZ}-$v@AwrLG0_Fa8`XrQ)HIQFy0B@->e6e?oalf3Kd>p}Wy*(CRXLpyN(H@XL4@`0=b@NH3Szj6Eh3tvj~@7eRD z(emD8GDhwna<)7X+`( z-yKFqF1U;GiVBdncH#MXz-qY31(mLJVIZkY!wG;N|1TP8eUd5Ycz@OE55@6Z?dG4? zgZx{D`~UrHz<+|ll#KQ3L^-wI`}qKi<-XWK2K-snU8Gc!UIKIWEiNtx$d6c9SrwrD zFX%G+!j1Lyp)^s~XPy{+rPMcHFCd1Qu!u+@5D$v_cek~*Rfcl<6ajnYb%#Nc1+4B| zxI#V$1cePg_77yS@y)D6)5={EKXvT*Oq=WgL~2C&BL9W|)Bv8j@B#rnV4E&bT=nz! z`NQ!2`^-sxn)+1LZt2MaQ-EV4z)wdJ+%L$W@12y5Qd9tG|JdFJw_G(piuvNvNK;Sv zWd@wL1srA^&mlAs;=yiHS$63M&TsCkQqI+E^Ms9L)|gFXO^^#@IcrRVi-CYyk<6Ay z-$yhj`KLaoB4N<7@U5q_-3w<>a5Ji!LUC19)< zix*EU&?F^u{$nf?6z`pct!7X<^{cTUmM(Cs5^h4xYW3oNqj935t0I2IyP56xqzj(- zdQ7FB-ml#{R&-ND804IBO^vh6qG9b>kTXfZb|Mehq}RxI611UK`Q-N5RyP-Z4xX25eYC z*vffI;&jy{se%xfCb|?>v^P~ba_oqFvz}zL3E;1OyGI!)I5vv%pAT;@2r*^W=Z8JW zaW}bw8w)}<{ONHCYp5B=KQvsv%uaSc`4Yt5VFruHF-E|>d_IX83Nr)^$8#uQXQ7;K zdAnbt`-{fAf6_-4R-?(P8=b-5A~fD*Zuy7%u66k=H`|G;UBi^=xb_^5erI8@jn#zl zXtf@t(hMDAY{<9@RsG#7U)Oq5B=7j?T3y-l+_LZFYPF(w)NEqx?m@2K<}J;DPEw?7 zAL1O9dO96HciKijCbUj=`xCl<>7n)p*2%j3q{n=vwGclfwtROGM`<%=n?5b~?W%m| zuE)#tZ$AQFlr$k#FfU(KZpac+nbh(wVc?4I6(Z0lq>Qa7s8 zR}dpF>baUeoZB2CmX0v^W$+zz2u0z}E1&mp*4;_Fg0C;{CDS-$o&}n&p49xrrw>D$ z1|l?`zMsn``T~?fY3Lwy(CF%oz640QdFHI<2f8UcRMx0k^0VBDhzq=n1@JDbNR==R z9`bewjpfGPD9f9&C%3J@7ZdB?HD+eAro7Mn9f#*1;mS@^u2+q}x}xc~#&H2&&lC)q zIrFl+&4LFYJxV?~O1h18hRa`BXUI3=%jdH@JwEF8*H-J%^?~;t--S-{%!NRN^9mFT zT0E%aGd-YE*WP~kw7w&OD#*WmMSBZh`;bFRp~*rlVS9Sk)&8Rk0`#d?8bOZD?id*o zUnMUplcozI(JUV05L|y{RV#X8W!Glm>I$=+`g=>+6bG?}-E1GVduQ5$!gH0gz?Obxx)gzZQCPHh?!o&< zRh9bA5KY9j>VZ}BXjq-piq&jbnm*-KyuC#S!uxZkx(mkp9)D~fx7K9wn^miZC9}D0 z+QqibC^3d+{~0E^Hq!vv&||`Hvn$_lzJtVHwtbtEt}Az*lSed+(v)m5K@Fw3@e!m^ zG&2ZMd}CIJWyPeKznHkwi-N|v_E~z{t`R6m>aDad(Lay1^CHNE@saLacQj6sj5d7; z7_re^*xo%(#EBuVaL_9bHM`1Y|BAME=Ib5Tw>>h+x3|Z}xD^_4l``1x)3wux)Ip0? zZTHROU*~p6)#@Hvgw=;{U|Ge&2ye&CjW1f^d!o07o#EZfVKssP{RJg2AzjZ>kB^-d zx|E0SlA?Uxc$P@TDIFlqZh9qTB;8f)JB?}P=c4Q01S_!EtG)B7`@{y=DtJBo(38sL zFL|0Ff43y9f5fKhnP*D0=7ROIN6YDpfkrUY%Hk}1bRncV{2-f5t>QaF=3Y|Py@B)g z8U^MY4a8Qhj%eFHu1@H%)T`nDCv9lLNmr?~F)F6+WlnemZodXp{>9-aR5EAz*yDUENR? zd40l%H>)oFN0Le6A{DXkVG~Kq$M_+)GU>;&P9;0O1xKzMr$@o+=zsk5T~RnGU(plx z(C~d8@8QS~wC8C)(qeW2D^FETDq7p=qphWz$@lAFy|tN3F}#L@Wu5zB8GB?*9#F!( zt7WF=W!I%sX88B&pBkwWh~R(@{OsQOT09=c%;7_*tnd#f8>R>-EE#agYuJgjw~*9g zzPOQBTR9kJ#Y;X2rM(Ip1!CJCO$In78r5%K(buemqC@L@>KE7>+lsE;oQ%Almyip$Ky54-ET?c+KJP&eD8}F%*2l+8nOwzkbHq5eq%95PD;1@E!X^tQtSBj zjD~EPZ4Zcq37-qjWG~T0X^``HLV3v$-5crhA)8Eyx^B-W(nd!1+0qknmg%+#P)hpr zZ3d4enq$(kOi2}n6iKeb`o-WK&#F80s-9zn9~;gt#9KYOOv z(>W4A^y@(|i0&X=@De1~ z8hKjf8A6EkgsA4n{dPN0EP(PmBfELjoITzpPp_9M!FXIXPsP&d&+KY)q$JY1LL>@& zs9`N+_|0%R;tPscQz+K^vje3Vj2!`vq*&+`IWug*cYBShz} zh@}_0h;$4*rM@nvc*O9LX)2*@piAfKO7O_M3W)ic5Rosa{WZn$thjn++_9)PitHJs z&QHrHd9$UI*v9?EGvE7fS5WYAYS0sWF^7l3qn zD<+dL-WAb%dk3b7ob=P>{yI)wWIl>>hJp}_Ykl7byCZl6nBE!|x>x*)Yn$4%W23P%T z0M{zu@dG;4L&CC1Ohr@O_;KTqsgah6$11czHk>SskYo{fo-5-h#P0c9^L*yYbD+xY z*)Jo*qCE!elxo#b(EQab!7_Tr!jX~)Y!cX>noasCms2=7F9p2KYE>Hr*Chg%OuYI#=Cdo38z=)4;www|r%7px~IH{v-P_NI0eWoe{R z`<48|0*AUaxmtvpHD$z8;O`E}RsY-%orwSi_kowY-J~M!{?sadvvM;>ZRQr6WAXx! z#fn%@s+0NV6V7`xws3JJr19O{tcjL3{n|(JC>O(NeqU|t4STFu04DPjh>+!rP7cL- zp^a8#^~>yUa>gAm@%VJ33#*P2vvFb1j>DXp7Zlq98x>&QSLn1=Jrr2svXF-?ylD(< zYC1>z6*6&4n#`S*O!XTmyw*}oePbf3X8l(tn#<$qp4!-Yq8>t5#Xv}!l3U5PkAl#U zyWkpO^zl*r_e(YER%UbJk2BR5LRHkPjv~??zpW|HT;R~0-M){nZ-E7Ks=cE#q-K)3 zob~L^%m!#fBkTJmE{Xd?>2bCJ6v%Pe@OOsM(f6D@(dQ*3TA?VD1PqQWQj;i29#{~3 za7{8^PfjXYQ>)7zJ$j%f9*5|7VwQMrmXPAeSx1T+tn?{{LYG*|Niyp>vIM9G?3-66 z#lT{*zv)0`ropM|?R&xMRpafBq82X=?JFb51uL`r#4~nETTD6|H58QaSnnU?)d^dp z{Tl`x4uNk_9wOfn8`{3G{#2Y>2{^vr$>Ln)d`%YH0i@`X$UkZ^xG>V2wRfk>P>Bn- zQ=d+0E-sMuxFo9M#KJ1%7hZzAKoUk~ey*iH(sbu3ztJwNtxUF`H0x&ikJgh0Z3Y%U zi9&N$yNcwJqBrnGdX7`fOduX}K;Npl;%KhPx z*lFmb%guexe4=y%QKZuI!O+oO0=nM?o6$U&-o~?ZZA#q0r?KcZ()!+Wy|P2?en?Tc z_Wo$Gp(;~-DTZ_LCWwD7jvvZ!oiy*qDrGcySvB1A^h*CJ#M)i{Q<%%iiuoiT^)YgR ztIf6bEWJ()i~sJ~udZuhs<$PfIXBZ}=_uHVQz>+9H!obd+%j>iSnX%BR^Gzel^>9| zYOrlFt$xAGG`(}+Ef#mS7Dw$mLF@VowiT2SZ9n!;7s4;US&>SfPF+lRSN-GU(e}2+ z_l;Qn2NMPdk6xrjAo}!dQ4Zt38Ovh8k}pPOR7OGp{RDefn{A)_@a4Rno;Q>bHSwbEi~{+2Dcaq!$Vi8zl5ls89;;7dwYwqY9zcyLWyNNohCGBc;&&3?Yzq_}$} za$KW(HF5NW@uQ^8n7mvX&EH(TgKNE{d)&dlx$W-|qw+_z%YMHE?hdxCcTQ|GM82BB zCj8~3o#?TLPGfg+4b#~cYP-Oi)%8)QSkuNEKfNuc$0Z{8%%fGR+TVwzqPpz&Pu8w7 zZd;X3;f`SEX>F}ayW`Gv@#98&!4tuc_LZLQRUN#cJ~oTg3$@!V5zdP~ilpH5a87ed zD!0#)uPL{EFU|y?Og*)n(sRQEdnSMKOaG{EJ;4C+D=d5Kum6Ljh3fw#Ve6q<;ko5} z+Vx5E%a^}mwn*7IzT%NUbo@~^XW4ia?HGSlvYEyCfW&}dB`%mow_#IW)c;boOkSC$ z^pfxPFTbLr4kq4gd||FjQ1Yo+UEBJu zw`*9Pd=z?VIAUOZ?4HF#;(gayH}6^;?N5Urob3Di=Lhx$9JD1O+|65r^jkDHe;|v@ zY5rmC@f-Z5w&^^_JKGe$vs-~twP>RbqPpif=W;W{7bB{aDimc*t#}&YTWvc#nEsey z?(`qFnOb?IJ0kY2OWC{6FIIOV>+=sAkJ5&@&M5?lfm?_Q?H z_SEXfnVl%=7xBkh5s#D|QPfMbS_BQzgAqU?&64PLLpb+(CXj*P&$p<%NFoyPA^#=r zzWDsVFZcT!fe1J|scgRShjDHJ_k=-!zMJ`H026vK5C?^74h@~Z3hwH3Jz5CY--cb& z@_vP;(SLHUpJl$Q0xb2+F2JQe6A?-Jt3D^W$FA`&74-kAOpm39sagEbMD*Ks3f#UH z#QuMCW}S}L3V27rE%mprUTFOPLAp31fV#K?##|+V?&{o`>6)8JWq+r$n2j@+=fXFd zN534;r9bF7<@KFHany)wQOx1V;s>}u?ZwP>xRR~Prfnqeb@!!SGQm{1r*vac#-`Ew z03ofmBEzDoNlH>76gQjKxnc?Rc^7TlJKhVoKv@3K47B8838Vp%6==a8^$toABoi|Km_h(9*m|nk4_!OpU{Zn@(#|aCZaG zmpxW2#!Ajd+?10&Dhez*6wwGxHZp^4P|r!w9S&UlwnZ%cfc!vj+1W z2faArAc7|@vub``U+~*N(=yul&y&f*pqHoXeiAtTbvEv+YUt9IX!&Y)hJY`IrCYOz z(OwLGmOlO{+m@pYi~v`68pkYa{;60nR`cH0tsViA1&8d-7_UFg#j3cm@7hQr|a!sJz^z~R#*H3)>e}0nD9S_5t z;sOb*R+wRQ8)Kwoz8$HsFNBvog^kWMOFhvlGR-kn;J^(NJ2*sV~*sQ z+n-@6x&z`)HY)j}DnJeKk@rG{Z?Jugt-xz6)iym&4MWA)r@kC-u3OhX6;2{KLmQoI z=%r0d55k#_Yn40v&wSpwZ8Aw8eOHz`{ZKQ`Y+3EU*-8TTK3eT>KJPrk-;A7d$n__$ zSS!efJ+8KD*s85s6`2Jd*Yrrj^+Y&@5Y_tf4k{3on@tvZC&1wNy!RcpLXPPaO7)EW zl5errh?gYKg&Atms0JjHsnrm_c1;F5N3Hr}F$7E>;OEzmRUmh+^9VmUTtAP1Z5Xaz z;hPyQaNFBqUpBO5+TojI6sItZrzhA8+UAj*1m<6T=RC0~Bzbz} zZ12O|=hAFq6T%XyVj|rk#WiUy81aktl|wYJ^4=~G_nWKxuSjQh3gsN!2 z{S&`!td@c8QNf!tmt3=Z?DPbM32!h7jsk>at>Oj5V)>a)b!LDe3cSpLut--3uUcmob`z}h#X&tG$Vn}^8|i30 zYO&v#ZR52p^=o#8U0NI$HnP>zG<*<+|3u6qJ)+RB+Jf&qbkU32$QX|mSrXCQSbEIa zIGyOWglp#8HZvTb&Z{N~Fg7FztTy?d4R1=1Pz*fr&hb9K`M-6jqap<6S*er3r_` z&t1m=yzH6MQgX779r~s_Asy#qx9(@)J(@PxjE{4y|E=cS+x2IqJJ5KrWoads?`M06 zP|U&?s8JhD<|mN-UX;QLHbP!Nr;n;Ur4GFvw9 zP^RpCi7UXjen6~onN+D+^@G6nM(lo)f72V*Lobq-@$qZ-ar!D~(}(RWB4jAMmBr-i ze`JfaUr(wE$%L-LizWlEgegQ8oaZLq^_OX$P|bD3>|QGgQuFT(_>X(~JtBU+Mrt>= za`9EBt=%3HD3~}{5)u~a>(N4Ycu@CT&3`;lXyR>|;_u2w-D5m$;e2o6}kNqg$B2X4B@*UEu;q4%&Sb9Lr4cL1geW@r0)J(`@<@^M6wvCet-#NrBe z(M4vC^kkE0yjICAnvnLL^m6lQ4(&<)akZ+}x5E^14c^V>==o3tN7Vx3VH!c7gRg9< z`kZzD@bqkWci%(%x3IQepc}NuIQJ*B?{nhxBJVI0WS2%9+^H=JhZiw9hrOG)+pF@oWFRT2X$_ z0q46#AA4QJ_DLlJI#PcuOBmC@zPN1f-gs=Y`&e2_=}39zK6FOE0*zu#Zx>ubFOCbz z9Y*nebBZlCO=!lT7JyG_E01NcxO5coO{Y?anxj;mP43mgj1-iKuR z5jIaO37LmGj`$hSb*6N9W|};@v_<1jzpWWef@BU1G7POBU(4cfx6dGSsZUW)6)db= zM^4q#-XlqpL_7Fr#FAbKr-6#BqcRJE>vLzFw_0Jw?PSObP+r@RAuYMP$=k5YCmaS> zuE-vuco{ttf@9{5YT9n6j|jfQ-tWYp9B>&>Xt#dqFRSaRWfJ$4cVMgAUMW4+9J4pN zW{a{LuQdI^)mU`^X@VY4nL7XP@r;wKUvSOu_-2zptTo!Ga;yQ zq(lI~f(Unpv52qAw?o`k^w}R%LxbyG-7%IV3C@{XkZ??0ZlK6MeWQ6(+O3>R=}$Wk zhncH6DwpTKdX_EEXUa33w&@8DxvrO=+&vBL?h`#(W_Fw~R@oYCFpo6+sZrbzba0gH zFBnx-+7p>L%pIu#gRlmc4|n6SbH5C0eZB*0n{?&=vxIK z7g_xLzvs<Kg{H(D#= z6lH@q-QQU-p<&9n=j(tLjn-;pldVwmd;dGrFED}Quoi1Ts0$PNJpZ!U<5DD9WG|xP zP;NW^S9^%jFxP&P_o-}=-qai;RC2{F+w@|Mr`n?4<#9}A2a$ax#ZQHD0QB`z2=uJ4 zU2IlU%ATK8n!Xp}QR##0uBsBP8ZRy9u;d=Xs+_C0G)_3Vy)AH<0AVrX;j4Z~Mtc1% z_wOW|ht=1ZNxW$>aZ--TqYa|nIG#4>4WrSnUwfNn)Hz}B#zG}T?7{KdGp8ZbM!{Q6 z9pA`wSb~_Xk)_!|t0f!avm8O02US106Xu&%7n_!+=bpmnsCF*5UO=z3MK3`Xa|yg= z1kWwMkq~y@GYGzUxJfe5B|+9`6?VdkVyHjxAqbw0Ugfp=O6vQ{h5{=*&BKAUr-u2d&Uz=?D_kQA zkpzx;*T6$dcuHN#+`?Upj*VE~Lnte$TGQNWhOhESzLMFWir}Y?=u>%AdVEvpHH-!} zZOS#Zx7pV`p~*w*M6)}UYm0x%OzCe+-=`R(rI`YdYcnj(#;C_l{V&3`9RNQCf&&Mz zmxz`Crpe#zy`+%uXRP+fI#%)nFt#MMfby(2$HWQOI)6^M+6=5)({)>IHo@nb%32sx zB|F^e{RbWZ`{kz*7T*(I;8d$fdNX8vc{(>#krJrXj>uKZ2uT)S`|8Vo;BZ_R0|HUM z4B0~n3dbU89XM&VowNhQ&5 z``*lW;H#t-HZ7_^je0=SQ%cvrc|+bxu?u+_oC=;s^Upx!LHGWSFTucX{`oq~s;;8W zN}vFyj&(#S<=Iwk>FUo6df(`da@BV2$#%vmf>O3Cnw1`I=?~uPxIQg3{bpKodPT2H z_4OnD2NxuF;5mZRK|&-BUt6<;1Ocz~)oWq(j5E&%(N` z5+J5ycTm1sKH>ESXQrwEpjE9n_Q8#$Zq%N~pPvpSi*E|O^7r>iAlYtI0Bnio;rIK< zBzQ6{N#H!`Rh5BX7Y%BO7R|5xY|hyKxF4$5c|AdpVs_K-VQrV*$i=oG_Tc}0BC!7~ zM*nJj2sFL6DrW!&K*X1KB=C*07$7d(7Jh#EEci!;pHG*XZJ$t{KTQwR!Q0x}x-e=t z`PE`-S2Gk57wzAOSh)3y%CkArvhDM`Ay6oNd8@6H(;%_-JTK?zMddBg%z7J=omI}; g2v`xW=VydVjx`Lp=S_I%#m2Xo66h=}ee zDavUR5nVGPBDxB{b(3&~JIZ*Qi0BEClH5yOuhh-C0EfqGX=`|1!QSraNrnhpfWyd0 zp>>ORV;{N3omyM@gIm55#McW0w=DRQMY3*Cu$Cl)9&`M}oDcwnU_*= zOZM&3CK>V_72g%J>*rYl37?&eFB-Yo_(djtx^l^Uj<_h7$3&&=B z58XEpC@A{p4Z=B5?Yj@qxSUjs_?$4D_k@^7=HZ{mjPXuE*q?d4G+>T#;f;)iW7tCM zw%^(feD7lW-momi4Uqoj)06rBXBg8yjesKo{CA4^=z!C%fWPV+o2aqw^){l3;K^g> zi&cb!m)T|6Z^TFL=&E~&1^B}YRkB)}+fRrfIZjhoFEbifuzrRxGsN>5VHx4~E-$^< zau3-{sdQO$-Wq;ivQrK3{c%|}q6&TSuh^bQdbR9NE~3)%#-z!l442oDLn%dmT`rde z5g|KyRT8fATpqrFtX!kv23;Ojyu&)k-=?`dydL!JPwTHR{dwTUi(7>8B_g_e_v-(^ zZHxV@s6)O`g&fMX*QI8Q^0YdAaaswJShV0mf z2!@m$HcAQSU>H68%+83W#n>}H4l_+ptXGauSPe*7{(reG(L1e}oab<;^p`twYMu`hz zmtB6nc=5eT{rEu|4f-JiG>M$cS3~8OAjchJJ>-ZWzo}~zIWzD4PLs3AarT&ey*rs{ z{BF9;rj3*rFyvEWE@m&AQ~(xyquf-}tdCyXsZTs{{5UEjdV`gF*{k-gFha_Zv);rvP2Muah1(rNpF9Rr zHqbR3N-S%qTiJkBY}+A>>p@I?Ei3Ly-K7i%go231b(XItrnAUK&5UyM`zM^Y@X=;a zQ^N<9j%}`^o77f5%vryFX90YEn2%Tj9Mmg(9m*!FNl*qKg3e~R|`xzohO== z+vTWEETkY8DG5rfn!BL_>c7nXdO6d3z{0V%<7qUxadHyLb2mYl!6$CX2mMYh8yrq^ zqrG)5)U|T8N(yt%(%H|2igV=^rS*x6D!586_D|lqDapgh(X-aZ+%+=Z^?68$Qfa7+ zAFxaLab-YC1>7RS8?@iRcUe$M9@G00$2ZD8Lqkfx8$|`svWE8Sdse{>kz$|j0-m5r zB*h~8`&6B13c6@aNR`^Slv*7nV0_a{Hd_NJX+`q2tji?i~Ac6(m? z>wW%2ytRpOtJlv|5;2u>C)=dJTrQH8DT7`Kx<$8|!fbx^hB2mZ_MfqkFHNQP-;^8|6FkkKz~rT)uD z{J}~B{l`Zb9th3&87*yn(~ZG7&A!Jqz#Qm&qTkLgE^<1bGlSgm*shT$c(8%LKKt(K zyz^l9<#yo!2C=ws_r)>@=-DR<3*;-2GW80C6!QaW?$7Z@$Mgq6I!K(`Y4lvGpT?&b zJ->*yA}Mhl4EXtFsj7E=W^L=&n!ARs0(2z^_%cPZ`VCq&#DBRL`JN}3koPxem3j5; zpU8Kao$B7i;&}%0$&DD~MB78oIWQ0Bh|OZxhDl+8B5$kYBYGXag7Kv&1wAiJ0u}WK zjhPXXn0+_oa`{j)pQ)L_35&Phvt z8cRiuhODMn@bYVfvTX`sBW;#4;fmYmK#=Ej?tX*ff-cY`)e8w`Ub3=vi-J%hc!Kkk zHFAwsAM3O29QHIOtD~DO)W8>|_HAEe!l$yg9l{sC;;IYd^D#@OC)?dy409dBq|Wq1 zwGZkHk1Eso%n0QWJ$f5=HPLmqnFfL!9RMjN}C8FwtI)dvpH z4|ql9%tM_V0gNJlHYD@7s|NHyqodB1AG=!AJ_>5L$0`;`KJELd`;|7bH=QNjdF#?o-C|BK&U{O5=!OU5tKJtX@qB0v8Fz#xeCN=Bv1|3 zl}9Dp?)gUfz#q?#o``ZosL}*A8ZWopOewwE>PcNu*54yu z|CX`@{iDnBF$1({*?82GpL0z8Zc_p@CdQlUOY_=s)zzq^P);7BsEJ`Q&zXrsEjzJF zw~?Fyky;b9#A`T@z(`k*ZBxSVS5ZD^=akg+B+AgEo}cgUsP|sB_KjNIa4zl9!NxZ4 zgwSFmfJ}|Pey}T#y?sZW(+qwaIh5|XaAXh%tuia~X>IT2DweAB<-V6k@IHX}P$2Up@2TSd>I^9L1H1uxiaJsi30sBvl z`8^=?DVLV2f+uCTt(HZFGquJW_$Bp4EMv}!K7Ww{lf`SD`k6T0B=izw&{9unM1+Iy zB{ucsv7v#2rNyr)zod6<`u_b>-#O5d3+y@N`KyN8N}smg>HD1O9qWlVHv-#!oeegv zLNhw&B>h#kPL%Ak$;T4380qGi4aSWtykzby|FoqhPqY~PVF+v_)Fe>B#^$FTmKK_& zVw^A7i*wLY53Cvr7Tj(eLf2V$>Un>RwHizW1Cbp;wP+d9Jq$8V-MuH%D4vSZU1`C4 zdlCRK;VtPiNhF2j1G0UMM*%vY5?5vwKamYQAg=n_<4j^Jhu||eJCjBJtfkESL{oo!I&(=$$@${XJo?lqPO~e(8A%rO!(9L zOxj45H1OA75-LrEYe_WmQS#WFWKvU8!!P`eTMDlszSRb^4O}J&Jh}b;_T|A(J!{zCdCx55Cb?cvQZ=N1H%VFoL{q+rCm)kJWx9N3>}3P<@yAJW8tREtF9@R=siI1XaX>~z94R%vdiW}2 zCEM@;p&J?k?lxcn0Okd z$9{3JzbDP;=N_k0C%AJw@9#Az6tawa!|XCgqc3-UkbaEM>ZVz^kME1_)|fbK&r{jB z8;|C=UOtZ^og+SUGy_A7Oc9ozB#&+4ta=o!G8oAt{x2{)&dGS|PAwDPL$4Rk`HoBcA`;o*c(AGlFzYMbUN*|q> zt>O|hKQZ`ppROYwqb-lNW(?elWVr#c{^IQSoNrhv=y3v5XJkgt{NzS^YAm8eskc%s zXvUY*dHse9fepFLNJw>Db!_-+(|+Wq;6jSex{KFj0$%>0dhc&De&OW}@F7vcx&=JZ zpXxgOD+(g zkiP4^2w)B<^AA|{Z4FPZVC8&csjgB0F9S93Fo>{^*ko`WrbX}MdCJN2G@c36wqApE zP_*)M8jtW9>e8&ef{&YL|9Vv)-&p>Nc+`&G(bdO85t=`01-Sta2ISOjpBlwy_2Qpv z1Uy6|$n@+yP&l;ZK~=4ttuu|*V)5^7PWXpxsVCFM!t!q^?X~K~Z(EK13ZL@REM2k4 z>u7Z-4D=}8F&M7FSK$0yN_cmIoTzmfA7K$Q<0_h6B`+X~M?n^=`8<$_lcXl5Mp-@- z|Hkd|5YqQ}*8@DpZ~b-lDUH<%ZS4_t4=lcMVfTfU=R^?8C`0V5Or6bz`|}=ppUwhi zZ29!w*4911N`v#b>9{lb$>iEadk5eUOW_9~CW2MK zHiYUc54q=k;~|miJJ^UGub%#ogrNuCvL4}~ZPA|L*^nBFlGsz6SUuqPnfkgl7U2Ev z21jKpnZD24bc!n_34O&+HV|#rTS#uc0D5P`kpRgfLqyqTO^wQrwRX;KiimHKiG96Q zXCsCfsgR=*-XHD>6u!KkDoc+T@({y)@kQO&j%W4Ew=%>7Cx_a{1iQGYEM`roSr2Gn z>?`YGGAa{mKJrfY>Agv7f}h-%#o;%WI}Qdcb4>b$qDV(!?LKYL=;a;06P#dJYGkGt-Bh1SVhf7m{8reHmT zo)1MMprH1aQemxBISM$A69rwo8_uAZCxuhbr}J%_of((;mac4<^x2y)Z(Jmspz-y< zni&dwbL;?LZ%;%RZ?#xUohrpA8BwA0H0_@_vB`~f?cafwa4InrDV6DK^Tw~MQ)^HQ zi=_|nJe`mMmIv^ZHwtf$YIeLCjiSG=*GTS$_!ol%Qte$oPv#Q6oU_v7=!Rd0x)+BAo&enp+1ne1KlZneJ|Eyp zz%0v@IJ$b>$g<56K{SHR;Lc6^`?I@MqKx^HOhuFZIyF&ql-q{UKav}AOTe#6zRlG) z7Y-pv)kD3+0(>`0t+!C_1~1ma_9Y~r4W%7lo#<*=IpX&%NiUgXqE<-dd6K`{bM`ec zoA7dW)=ADE8rG#y-&qJ2nF~zi`CKq=ysbY;C4T=$D!8dBPcgX+V3|~=xW%_GMuz4+ z&pp5GZ6RyA;`4#bx*r*Qp$YUJtv^eV_UY$ZR?XKM0nQ6!ae0jj1r}I{Ja~zrhj=s6#=+!O9zsM@}Y%*yWd~+S@xwbot|MIGW zec;)vi&Z{6RL|RR#8`Kls-qlFnh@X_UG$ps%wQo_pboWTS`r5 zZ|kHq+yTm-Xpv#TnGGrdUGH-U8v9|;+9Zz`+lwia3h8@n*3k@yBmy zI2Fg1&}s#z>UjrIm~b2V6C0U0v<8-5+5kS#4eVMiB$_}(V$Eu5>(!msCS(9wssWfS-t;oRHIi}G>cYB!PGrY`h#gMqYnkN{*+B zj=iYI-s=@(bMSrRVp1%ywfs@OBaV@xN5omuJ442-M>IH`{bLGi5az|YVif%oqySPT zsv1E07^!i=RHW@)2WL)3e1$U=1@@Odl|6^q@5vqLJ{lF>IY{uCHuai$XW@`#Ay3u~X% zHB&?s$f`Mm4#|}>N>#om3MxQB@z)|HbLo`4#PIu4Z&EoH4^`hU|9U{p zb0Au$`x-b(WjYDd7`R5;SU5Bxc#kLb_wV1)LA;@4E!j`UVo4lU;rG89l4sC8Xyjw3 zZ*NC7Hb{vu6;KpdJ{RNM|7FpId^&Sey*t%u^-m3f^v8dK zpgAlVnFaBIWA{!96GPQHjh9JFvBVRAU;4NZhY|Cw8I1&SG5wOfvBPf?)akp_`t`Tc z#RpWwG7-(OoJMf?$olhmshp+?d^2y@@r=0n<6Iln;M%=h2s!oPiKBfoGdFTFn+`)O z0sGjVHA-9K-Y9EbS{7ynn_oKz@fjLwl_R!&>TII~6pvMTjR6pCb4WV335ZdngiKaW%6?#gdk(URIukx}aBauq(9j1&N24q>2kO7+Ay1vSTq zNz#gW+e6Z}YQ84>$dhSRHG;YF$WPGt;hcokubrSjBdoxip5YZvhU*BkC~od}(RV`H zuZN~mrxs4x6VI35`_i{&cnVTi$(?0XKZ^R8xPLyb^FcbG{Y(8Oj>p-GPi;j5SCx)Z zSaK-r-vX@)nr`7e=@)2GIpg+E7QOG>kLapuB8QrKmTj8*Ej;~I-I@HT)n#V7Rp*rL z!li}0u4jBljTdpcCn>LQV9YIKy99Dn`uItrm7Eq?x@MyHx9t!wd>{?aP{jxzOQnZw) zDplrP{n^o~;Ob&$i=!OiEX|*+0ez}>haYY7 zO{pJv*By*+4>|<-^h2=hXC@0jMP~{J#Qn?4KG4~~WWoQ7t@~{Ha~h9X_)H2! zM)d-JU9v(%6+&X5qEt(NT_nG>IODMqlY=PYl8_^^wJs@p>dH)&p;x*(YUFL*5f zYsB0?J5#3`r*04-m#T-P;>10%-31x`oH*><%;plIU2o_JogO>a1%;1Grlv$ z_Wt%v2sx{E-czS2UX@CiEJ8ab-n1Au?GZK|FS)lS1x&P;yX| z)bTU3ORCg99}^AdDZ<3Vou;B4g)9Is=+~WsD5Ot!tY|K2+L_ccoej9<)4L57j_Z7& z8V5qQZ*ttzs3{d;U|B)n#cJX5MG)^S`wKOZvc@0r#Q1?}ZFgKljd#d`X=z zaPrl&@M31`8+0UYp)iWCe9!8w#XaT+QO#yA=yS-F4YBHCWeWxiVY(Dfa%Z1@c`EBV z!Ub~iwbF$1fT6S4?YdlIxWz%3;|2etlk5Szp4iuw3m7u&h2ug zSK`2qRZA{_k=ksRtn>BvB~1Eqtwo$^rUn!1G{94uDZ04DYajVV1i07_ z4(T`_s)>GIuWR1-FLx)`UC{2@k*D3>?R-M-qd+?2t$u^8wk({=3@lr&y4oIPg8dN0 z&Kbl+y#XID4zU7!;->0V_%&cjU&Bu|xV2_s>(y|@tdHa-^#d*Hiyx}tq~s)2k5%uA z#l=TcvAF-nw<<~`Py+ZZSZNLVKAS!-GV>i2dN9ozvjL5@G5-X6K>dR9p0#iB7v~dV zCi=QEsWk2P{k2ZDep0v4Z}zbQvD&EH7=tSVBqHfQb@EmO>4&^r#oItM^9x!Yt;P;Iw>30DHr@>U>SKm&{jgN&!4N{g1OSb4<=PoDe z8GBQ3ZDxeWt1U{Oy0^qqEfK0wuVLWemUvH|fxgp(QT}7%j)+@I^<}>?;U27rB+|Ri zhA=Vp4rzBTp32}LBRyJ{Y-a~F^oJqNoO!RGDQ0Ca%*W{YOMMG#N?d|$bbLX5D-*Zy%SjE5Hoo)?K#1RTr8@V0X7 zB^Hn3b--sX=veg^?>;0>oDIS?79QxMx(*LP9*N)H3)w=ySlkS2y3a%Z>^jX^!#!Gk z6zf7;!giTRbc_&v%^s^uy?gR28eqp}7uv-@=pp*K5H)mBISYM!gBD_^%uj_q;x6f1 z96k35fdNaq@1#*2HGWi8l!0x$Zo#mW9rqgRn5}D;`c2;2GGKMN!A2eA+PCrRh#!^> zQnkt8Tbft5W>W7EwJBNsrXfi94*`&l;j`gYMHEX4^qEHn2ep z_wb9tuK6$v`&8it+dJo=FtKXEHZ)B-Dn01W#=<&>()2Mxtquwf1?F)#8 zINyTjQ9>oEdHH*O9`6SIfMsnM`e%Nh<<(?zeX6u#qu^nSUT*9Yy@&OdsZrA~Yae>Y zSPHlDFDje)q-J&i8kNrc8!-8!l7fQaW9{%#qn7+GRv0f|Dc726W{M;guls;o)QBA# z>NWHMedL*gFl9;By~3*B-oP17;%1~$~)Yhgq=*dWjnVBJ^qo3Q0M2#A41%>Wm^)wjV5Cz_sxC-1v&o8H5_9i9Ld z_Kmq1fx3n0`}VVA9oN=a9&N?twNg>G9z=$5a2B7c!nR-7?|v-*qHN%<`e331AH#E) zREQtpw!~sa-q!$M5xkW2v3W%`hNkbyWoc_6YirqLQ~2SUR#BKKvBu1*_RQEKz^*o_ z@mdn~4k>0`WYi4U$?s2PS$Jq^_th_FUWU?^Ldl7Z8?U80m1}=BOnXWRK-6JR_M)Q{ z$=M=_cT(2oICK+Y%d+?RqVFF@MnYc={8lCAP_z)UW>5Aszv->ICBsFim}>p^ZP1lx z-6WjMl6D#6b)#YqXp)ycq;zBNb;i%XzO(kZIDB25P{YDzPydSG7OOaW9s;n@@CU8% z8YOPyd67bySHoU^$E0E(6oUN7fpcVP;nDp)RmUyvO1b+9RvdQnRo0a8$=|bi0)lyh z(?zO<*_N#+!{t#CtDuJFuN0oez6yEURn&L;;OCLF2x=@{B8~GGrDJ3`bNL1oW`;RW zv8CNik5k8qRNf?|S5dsEa*7EB2^T(}Sg2<1Ssk@J47%TAAG5byZZ~J8(fs^r)>w9J zbV4fWJ0{;zGPIJS@+UFh*l@XtscY4-HH-#4s+R0j8-n-ap4>?rKv`Or_45}qgW@Epo8fn(M_g0Xw z>jSj=(;rA}f4t3K_+hNs3#IokCxPPaKS&(|1;2&u#fC_^xeOjK^l~( zXPZGel4p+<<>s4X&hHJ-O1PkJ*=bE8qZm>`MLl2@8ctGLYB~5p#eq89v;fM$(xtqu zgk)nTr(9vL4r9_Pn|_Ke_L5k@sj;tvp(O%JJ#fY~<7{=a%BnUN7W|7mPype2mtxrE!$RlpaLImXozTREs>eDcN^X2xgTWZKpWSZ57dPv-WB_bH|B)&ZM) z-6{Hc3)qVq8d>%Sy`EEt+_X+r4?H5nVpsTqL_?kk(H$A5H@;<6J9MAMS0Xw1%XRkqHPo=~S=viJjQj3s$C9doN1 zo@fE%mEIt3NPq-SZF+C7>Wn|P;p=%?Ps^w={>2p_yg}8uD40T0M}#-Y-dC@Q#Hq&8 zN$tdXVCVH;Z0g3!C1g^v$qICGC&xTlF1o1$NEv% zL54WNgYm*I9#Kjn``KLRN}vxLLRA&Yo_(Yg9u5kFS`YRd>{7V`J%0+s*?0}b65gds zLG_zXK`36u1#0k25Sah1x8v-FQxdO+s(N>3uCsTi0p9(oJEr23b?u$k=1r;k`~tfQ zjl{{J0>zIrz{Bh}Jv2TnewhfHG!HwtPCX^?E_CJ!yYx|a1$**vVyqo4E{p{$z+UNQ(<{r?b&IZWBxLY6r{qnHYk{>w+f~G_DSqFd-n~~#Vx$Xb z#cl)y#=0~d=?f8TU1sW&4JdERLG0#arR>+QU-u9sMw3E+M304u3O9H<|J9d@NudKS zUY`eQggoCobx+eqanrs37VKzF0P?Pc-vH00s-RBPxL_RIr|`tZY-Y0BI?C^0s*-n==)`!z&IZciJ zNp}4zO&#@F=kGR}){3bP!d)UenM9e(|9>Za_)nA(|E;$-XE(1B+3!wSXY{0rd#;RW zNGJH>u#O(@2o5#2#K%w=U(g)AOib>F=slpJ!p5 zEzct|x_vFTQ{ppzFW4?zg!Esut(D>Ynd9OT7(lNt=nT&5I;&dQrIQch^_r z=1Z+p5J`!`M!3YXAx?z9(bno^ja#+|J^ZF=gd1ec@6*vbO`kAw3L&6vEkxuWbt3eP%Ll&dDxl+?}3(+)@ z5@l`uh(x*!3Fa7fi9Sqp4pjR1kXPlXi}MKtZNHc2I^(}scRhVL;iSg}99jX$0N zICrN6Y~4S|Zx+Sxx1}*ouR>P(_9AG+R!R^BP>5{*YS$TxElL`q%)GrZY`INGsQ#m3 z^lGAMYk|gbEga4{xQ#LmfK>Mdb6dP)=7g-J*|+c~j9gJB=1nLwp-r4LDM|VHnXJ!C z!wpU9?)d|&m0VsCF^}SRJ$|$5%EisCB=XoqX9^sM2jGj@9o8Qd3a=){j6Tl=1@Q#D z_M2(Vv1_!VuHy;yn;;QJ>=@y467jpdtggjI&jEAkQ&5xF^)8kH!i@O`uG<%_0p2Rk zC~hM|{9HNW3H|&qVe=+3mjw6OnfRbt-LzNp^Yc%guhQ7%$c~jTx5HP9Lpt`EwtmN3 z+%(z(dHjxD3_o8i{cxv+!#%Lkue$AORCnT5pf=^(UWua^c3kM`h9@A1hu)8;Z-70j zw+O2|VC2(?TP-5}Udj)60j>Gzz0&^pvWFfttZRQ3Op1$Hk98;K?O9iBJSrytBVT@@ z!2ha?#b*k#;HZQUV_#gR=a(F1{aAH0tXCq_HzFlk{INf=m91dUEXVJ5BDwT-K!Zm5 zF@(CxDkD}0anY)J&VC4jwFk%|Ujr7LAlogx!i?T>4EX1OeZ+6=2$-6OBET0xO_s92 zJ&%Z_Q7?MBJ&%(KB*;m>{bF*-x+(T7qW6@#MBEJ1pXn~btt-!M>LJ|jig~ymo7eiK z*~?rLN|5pHYbVM{@6-fN^d!PHdJ;FGIaM%FI-il1gYHVA>wXVm`s}qPHR)=KA7_la z;%{sbS@V?qgr3Q@p3b8DfDH!_;nkRhlMp|>bLDwIoFuw%vKOpFZLK{o?Nehz*I~>I zEBk;yt8U_Y;HbwPfH&Y=>0#g7XVmEBwVfa=>Pg+}Ji4_aSu(l{(Y7ybYn@XVE7`j% z!18HD0%8`07I*X~zxt{N*L>WVzQbo9re?t+aW7##P_josQ~V9EzBkIApGt2wG|4I8 zxXRDVkWCFUqL%EpT<3@POhqz_e_L%LV&Fjx;QZO7FXV2pQ^zwTe%?$ua9-uK4IRU@ zZq~PL^!uN#oe@^T*Hvpf-xCBwjA`-kVdpE=B*Ly#VR*}^u-Ng9(X>;Z#pI^h?!U0*NX4?bjMm%eb z^&08LaQYIkN>p;yqr_GgQ}N?gRgwTy~*-)xajwMvPkWsr?#(@ow@5{Z&mT zl`WHEf)EI3p2ik+so{EhE>2fTt7!?H_T<@hp6lwLig!tI#KMBji5CrQCBc2x@8(AH zJ`A_Aw{ROJv)`e>4bt?G`)(K|Sy0XsUeNkRg@3Q(a6s^6^wdlAw zYJ8}AqBGLiQCjgUUAJ9D5{s!{jVN9&wnJ+d_U)d2qS-f_Qz}CHJdADTlK=(2=m7Gu?ihPB%-(Ihb>j}q1MBPvS zRp@e=`oA^6gW=(>S98|P-xczDvR;v(oc9Q>_9vty@MPX9h&$qsxbkOfouQe^-^m)B zz1ts{|50;hlApE_4g>iK_{;(f2LCTRo~b?gZy+e*k|8Dhgqtp7VJ4S~YaP&Hd1WA> zs|2#NlR%xW} zZqHlMkPs5je7@wZ>;4VY$s_n$cY=8xJT5~80>2LD-6j10%No@GR;2YWW-cz=MDz{l zcQj>v*;xF+n8qqWKbT^CX=@yw64bU;zBs2T0z%!Kt}S7jVIknvVKYEtUypf|B%6lB zZqVlB%WK?(#CG)ri%sY$L2$3;v+?L*`3UNn5JJbAVMbmtvi>2D2i)nkQLNxUkhp3f z#NkHawYi9$#ZQd$n{Np@yh$1R2b8@dX%!B=%+?{kOh?F0ipZ{PPt}B{}!YSj9qkdkL5U?`X6UA5O)$Ys&oWe~2r# zFQIePzA(N=dmMIgh9cN%*J-$)h2o8`)~jjn3A&pllR^UDy_LC=(`aqPQ~csD5dkBi z=Xm~^>Z|eOx=&Z3I+mN-p5y~U{0^euGuxPpmok!UwZ1c%efUTsV;xKB`XPv1dVwai zK>MwH2tm+kN4tBQP0(Rzs#B(kMSD)0(s8q5BKxW$^OA!`!mi`RliYUXM1c-xbR!Z5 z9U_pz4)YKN?F}EZGtNxWD_vB6#qrJg%d{G0g+$3!S97OKWMWT@-uH4nE&jr>@SG6) z=Yn8zany4@1;jWbIQF^-dR?Ceza23zAP+fjBQVM-C1nn#mJcrSRc%Wi8~ruLuQ$v( zl09y5qjo1Md(?BjwMd*Zhae%ZUYtH^r@r-Sz+6SojuwQ7cE^MvT5SyW!i$J@p%cZ+ zx~H;TOOznRZoPz;%*UNjC#6)dG#m5KV@f5S@}$FZN8h(s6??BsWK>v&xyvSSGFSRh z7S%|Cw~LmBDaFQlvXXD~6m>im+mSnFT{X#MW!~bpHEc-M!3CF|=p0pQez*Bvu4+Gt zm{qG?gnHP|_|J@88NFV0{nAq+@%)w%XoB&KF2?sFPI2sB3_|xi=2P+Lv5d-L7v2Ud z%GB}{!tZH(qwFIe?a&Vim38a~f|xlAivxWpV6toB{0}7A7pG(r+w2~&muBh($ZfuJmxxjO~r-n2s zSYK%DD)RVCscI+d;urC;6VE!;aDvMI8zj(>w7xVWj0|B1W_qlhg_*ekuCyP*Y~N*T zclD+WmNN#7>}9P?CDr2 zt|v`gg-)2MoJC5Qnm^>5qjuxm|E)1@2EXIKK>FgPy5>E!)=6$%0@6%H_~YQzSS4JIqb$ z{9|8zmPR9ZX?irXH@CL`sUB=*GnB@P#T{7lpej~}iN`|YE0ZLN5?a$*!!@Q*%=e9c z?9p;>aM7lP%qZo*&SuUPaH`3TSan)7Iu}WVV2=7l4Sldd?j~XdjNAnRj#j4=es}1} z27>6HmT`y;;*`dzjy@#+l_TV{6dI55@!;pq($~n45J<^V+;nI}<{WnqP8#o|x~yL3 zpCk1wmFb&E&b63}+ULTH0+82-U98+maSPb?Tj03eBeWOq8C> z8*+~UP}Lor?KX`2Z~_DzPiig9ysc*wB-Q(Z;_T5&{4IG8qz+UUo8Ode>20w!j6fR& zw=x9V%}0_%z#On6-f<`5(*J<}Ypb10jw8kJ`0kGiKKb>d>*bg+SpgxkQ#F~2BT+Sf zaV%mOX5J7P>^AcM30W888+IN&_sBY!wmQfc;zMGsnj~q(C2iTIDzhY@?eXiI-AO@{ z^^pd=^G9R7CATebn!xl~iL>GeD04R8tBF5;)i0C`kovZ7}j-MX5U4npCVxdYKoK?IuV zAIy>d0p0R#)8B1>`^*bTj7bTp{^PKte-V}a2SH2!wWq7xs&5mOc3j{vLl&z!;(I*{ z5YCc+Bi%M_>|$R@AwWQenWq;kS^p+02-r7u9X2f}s;E+0H+4XqJuahH(mN;)RW@{$ln6C)VPtlZ+_`l7Gq<7iR=l$EHnj&N zmtEg7bAA+{(Oajau2PWuLr~$g3C2GyZcvrUPw%#lSiEszms7mU2mN6AOCTnrFSC|$ zsW1rYPghf{7rn0^4b)-+mk>dye==9c#=2qiMSzFtN=WfRxEBP@6R+L*i~ z<4;JGJax_Spg0O{p_!kn^UJb^0Hy{9Mr==H%Hhs{TS~a8cy~-Vn_>b_Oq!C4dT>@y zTre|VXf2DyB2XQ!_l>a4)d69t4=#;7y|T)lF?A*L@k{swmnD{PEBc4bHIGZ7$PDRw z%==pAy#n0$5QTi}@BD7lV$I#Zo!O4pXnV-KoHlzDOPN;5l+|7PIH9#o?UstnTTDv4 z7OVv_$lfZ_3|Lk+&xe=-yM51hklefz*Tn3zfWr0i|do=6HtM@QfSv!pF1z%t=}v2ZUch_ z<6zN+|C7Bo1G#g>v-2gK9QMqx=k35rJAZ)p6x@}CYv?EyN(gS>&i-*+_(MrAic^epSlDv8poVR?&+6Ymx=$XxA?^=(u zMKOvFCVmt|~?LK#HxiQedeOO*QD~s#%5q}hNG?5j~Oz8Ex zI|MRum~-)|3cshfk{Efs3e!vkbzv1jKL{RD@?a`(s+Z9UCp8h3d>>*@Qr&FL98udn zXEdexhOkk8JCgY+bLT7T={UhFpOLuZzpV({0*q3{4cFH6i>+<+<`0m>njifx9OG;Qto|1W^6Qwaw_yWaX==pb!Z4Y_I!XQQAF33O%8141!TV;> zVvTz~C_=z-;0f^G`&02t@^oMR_^aY9Xsgp4K?+=XT4s`CDbwr*YLaR3}g z?4F$b|1O{ZIhT*fE+1Cs4_^>`L&Lw@&LvwsH7ZLE_3=|a zHw=|J<1Wohl4C67k*x*>!=jUxBr ze%-lWZi3+}i&ApuG|B)w(i$qIJgG^q&@)y_;^->M#1O@Dmg!+gJteR@MX27OO20`v zJzPe_{W`wn1Vw*uUs^3D7QM*;onQ5!-cJk4z0j_uI}fOFFb1SH1J#CJL(zSQetttL ze^QM&`fr=hG32PwQ+{M+B<~O9j~3zZI)X-n^HtA1gCRp7QH!gDt?w3tF=>4)MZmMs zahAQci-K;(Pq3d82Q6U~XbAVyfBtk=(HTzb@DhUz1 zA43YlRFpw(mpcg~ATzhBwu=QB zeApO6+%a>kXO_o4K-VO$?1GNYla?a4j{(0*Ti zp{YMQ-|Hg}h~4*C>{}27pzWDLA;L+}7D2w3*02+#*Rl5~cQkcVWN~1;z1GQcld9ZX zNv0k&vNU@}mY^##Ebmz2R`yX?g~_IzlU*U&`S!%ckyWGaH4NBZ9(H;#ai>hxQ}l8C z!C?&<;Y&UAihU;nAAi+PT=gKtcx%x%Ih{prSRU-9SJa6t&e9}&r#*?$L+QLmI!lh+ ztZQ4fqI5k($4@Zl+Sj6D^(S}2)&t|?=&-<&Ic@5`sIAH_hP=|l!}@$RPdKys`Z|Yu zql${eMUq@pd|h*C?(M$#8U^%L@#>*SmNGX`zRFcE;d`+zO(#q8Va6OUV59ZS1JabI zr-w}JV|}*GH_F6T(7SynUtP<&5LT;bqw-f;5X(4c?KVhrQ~!ICGsZ+cK_j82cJB(v za8RjGQd{Up$lu|5UbyjD|1G;+do@HpXqvT(HT@|8kdb{^oVGjPpr)?FhCbF$yr@uG ziBONBu`8l4`nJDO*w2^|yW2A8xX&*(!9gQHJN5|dYgM+QHOBtR!ba$6a*O`64PQa- zp|3~)eJ0#OH)MR-{Al_A>FzC~+WPu+QR=0X3KVH6P=ywk0>xVj6sHh85TH0o@ZeHG zN^y56UL*tw!9#)K?gaM$!HNWjv*>@{z27s=x#Qg*?l|L&%Lf<?Q zni?|(%?RteE$mcVm4M!mm$VY&Jzgc~`p-)1H=GthqI7XC8cl8wdZZVvL<_;9)DC+E z-^vajgqhv)e})Kn^E-LG4y}fJ+`O#hgt_DLM3kE}sYpT_%h!&UE@wfrPxN`^7gB*p zleM{!l0I_Q6*np-q?D}%N_{obqZ0Ff@+=j=iUG7kZ^oLG3h|t54Ky4r8UhUbNVk2y z`PniDjP$Okw7gHZgS_o<9e|dB*!djQw_y#)XKK;x-Ap;E4D?%byhN&$KlWFEppCKE zBZ}B~!vt$B)Lr>qAM_F8Rs=vrZ8Wy0=9|&3rD9X4SmBu{tLIhjUd<&vJ^wiDB zZg&6YjvhqqvwReO+Cnf%zgSTyBp@-b(ri}+x%-QK3!O{LoJ5id_IE%D=M}^1y>knh zXYx)oy0z2Kk|6j)&?e(E!gkAsKq<6G7qV}WwbKNOC)!#l?zsB zlzr^5Bqv)YD$RCR8D*U%JOv*qxi^GfWy(eL-Q2FuHqj9E`>Zgj^1IeVr8XV&x?WSsREn*H`-{Ef~7MHww$D4U;?DDs++q*x(pDek&5 zb$9lQX2twG+col!l~ha%xmNQOtj<(d{HarL)8mO6TuxBGIK2WXpDZ=ci{8KEeE9CM zK%>l%QN~pP+Aq-h>S&s0Ref-lc^q^IqbPGv^!`_wBOC%vz1+&9Ix(~}E9=HDI-#>T zI<&}gt0U`qw;)px%a{94vxB?$%%0C4Ra%6dyxiEJ0}6@*k@A$eoO?Mt?xr_Hhq*T`GAy_h-oY5iL13nLa7Rd<$Fx5QAyAo7FL z^8xYHWG$@SgEl|^tq`J8beQ!YJER%APO_#UlKC0VIF1`+!blrNq-OoOY;aFtt|?!Z zw_siVUO|HPO#y!S^kt2>+K(ZB;&I38mKA0FrsH5N-}`IIQ>5Y!W!Mh}5i+rJC9~q- z9~SKHkPub{O?OWux6Zaaj%*{9U^cVr2GJT)RNcI(ihE7)!%E^$>?7*svWzmcgjRG+%xKHu%rh43B37AOR za{9f(gwlU&bIZ&0f5i?^xGm&iixg6m&xX$*^R?8Yy1*d@zKbg;P{ za0FOZO{rg)(^<9~aw&hnYxKhkA>SeM2j*`o5`jP9ZD;zLBv}vB-0Jpm>HIzC;}2F~ zjh)&i2;D6v?G$ou7Nx|Fv_D)yz0T^47kS!l4Dd=|6qWbV0s)4t?!Eg<;pPLCnMV6* zfdOZ<1(U-MW*k4jhD)kyNm+*>Q>kZx~fHKZ8?ap zbh_LxdClzivjS2@o7$~ZWFcTU0Yv}~p2Z)9_8@%OFNYHV;`{Vp7**TZ)Z_`t zdj!2}@)_w|%kS%#DGXXKhrZ`vWeV|I4fcB!W0^LRNADwJ!W40XbA-^MRfxiT8Z|EA zQ~$*6YzZxK0eu^Fy~NV6{gJ|jN4xq?vgQ4;>X%WO?+Ymo?#_j^>$Kl_b6Nh;)9Xy0 z)NWlhWmVQ?m6<~+V~PPA&CdtC)>G`Q^TzPQ_-^tu&xI&W9z70Yawqr$-TI3imlQ}?;w6^av%q~bq7dF0G`w!+M+))B7*Wjk&K2*rOWuf zRBjb}b91CHr6Z6KJmS5s`%4m6;As2+c=ZfOFfQYYAoOt@=Lv}o_+zvL@b`a50So0@ zMaL71&+)Zik|RQyNO0Dr&t%$0O@a3-9oKZS{xYQ4%m|SftGHX2IgEpjkKOgK?%J9t zNao+H#mJNpis7*Mr-1@lNPbvCLS9mCm6a^IIb8W2QE{% zc9Ev|yN3~*R^g-m#`BJ;{@=(^Q|NZM@TsWWWjpKY)vFG3P2>}m2-f~a03LWl-P+pP z8+e~LG9?8yK0c0K1qf>Sh7B6l)~LmdQSR;H#t)~j;t%mGnTQDr`N zP%)AkDr5)q!#d;n0{-&TxJwW+;28AZnQ0g6FQze|7_su^HS6kdI4>uMF2IudhS>W7 z@?gGUGpzS~tv>k|BibA~uf18LI-kCa|3>Vn-X~;3a${Z9e4_3iwQk-CeL^W4j$bEl zpon{>q+y26s1w$n{^xR!KsFAzS=rD;-)dax|I z&^0A;8(aukUJ)(Ut35b`tW7{Y`D1_Vj){);#~IXZdF9ycuE0w^{& z5U^G;ABif5HjJf@gb)N5w&hJnxwUi^zWD&H2FVtRP~?OxDHa-3oGzp@8P!mdf9P6! z55H65bc?XrXa~Pu7L4PwB&h9x>Dtb(_j^h8wL04|>v9ZstV~+lQ$+6uC6aO@F4y}u zP5w_+ADpWP!ISx0rhM;7P6)6`qLLKkHm-rR+upGDg!;KuT*%w2mpVcb^Ys~PukFTX zx5X`-GbBg>?Wi{=W^S`h>Y1?JcHm!RnD%b@`iJKMP1H7tp_uv8Fd7}KHhud- zk_{r(rj4%Mz$4Q>j?w(d5aNvgB?UbPs8xT{76>cVPI~rVE|@Jsfl8n3hGJmp!Uxhu z9<8zRv9iF$w`+T?yJ(h+-h05cyY^8K`qL2h?|~#$BlQ{QR8HEv?7B|z^45g%Tr~W2 zDL}gVDAzF_5gcSfU#rsOsScyVJAO6zI(Sy1=eM9#cvn3mob17KSr+PQwZ$|Ut!x-e9 zXXYdcQW`y13nBqn=@WO0%jp5$`u>9l4^;Q}SW(sC%XSwUc6N4tW<0B^d;vh|)5LrK zvllrL&2c-wEQ#d5tt9{+f{oMhA0G3Q|1w3~Mol*XyT<<4fWL_@`A@?x?dgf-(8uQI zX`=f6vY7wXzWFU%;K;a0?doK z)`3^ZsEsSn={s-amQ>FlzJFT;_&+I98@e3FLBd5XU_CkT>(?i%@$%Ha-tINf&6X#E zf`akfM!s9y+ffM#3DF)D<)M7Ak(ITzL7~U^FQY2(>Qx3nau!6irKP2z z&v$6HJiwT=`y#bVOz)oCNiI{&Eek^U-!mkLHHuHe7TUoPL|Zo>B&!gB=+a-*dOI~h+M&cYOPCwr;4M@He`!4jkF z(ODWug?h8&aE^g0*+@2Kk&;ELM9zdXML2_Xfn100Sa#z;AcJ}6=SRpmNbmjNfa}}F zsMet%K$C2CP^YBWCecJZqrZf8@I!<#Geq#?JXM|+jf0`h{;Z2WowfG)IPYHO2wb1g zyMYvr-ORznhdMTiVu%a3;018C_sFXCtP&kTU z9t{@RYR_Aph3UDvHJYLjdHQ|=8hP2+cgiUf8JZq>Nx%ae-qwlT?br@@Q0h|QboSfU zLPhs_0vl6gK6X4`U^J5t`atcj%M%weO-=a-o$iVLL6zV9c|{L_gj}L z8!l?4h`oN=>0%RUe;y^gxl_(nxsWiJ3Yd=7-l9VW^)ijP&0RFwHo`GV?%Hj%AW}yi z`M5?0cb61|*|nVdTdw|(_}`D6T=gT+k8pE`N?I4TGp{WQD}q(th!=^p_W=z$jK^Ig) zsJ8`qriYl*oMs`;~An%M6R3Ot1j!~*M+gbrVErskZw;3eM&4CHP zZ4pX+nImI5(h*7M9g6N`DKBjISeno*EvZqYQho^dU4NyRj4wgW14a@fGwkc-1t8R# z+8D4-v^b?&*GhRciu>CLh-hwKKWWn^MKyG0n{Sjl3z<`$ec&tKJoyRyclEjCyS$vVU!RhVjh>KEtdWaAuS$wAfbF69d&hprSu)p+Q@=eFOrYi;4Db-mQG@ zlx8A10uJUsL=G;~_FX>;rCSlgm#213Vipc7*9x3sKvnx%MX^k*r8l$4a_1=w|IlX#z|3Rg{!#)MepDVPg=K=oXh;$iwE__TC! zIL9{K-=J#GVvUzdHIp7?auL_WuxJ^5 zSd!Y4hu>D=Nl^6&N>7fTITiZyW^U|?+v6lOq;}-26ssNm2#M@#iZ(jhKP95_%3s>D zYQ8qdaSeP(mlFnZxtxiPIu4m7Ft+nScetTA-gktQ?_;GHlcq+Aw2#teZF^3=L8c}-B+ zcbcX$o1px>g-}v6`O;Cw#5N6c3;g4n-qW5GzEV3#Csr{6KsiO8c2CCxEfc zn_wxw+1G&MI$t8B(Fb+?p znEFkMKkT|DItx2T1pB5^p9({EtNH-i8K^y9&cW77d33?d946zPr7Nc^Yppe~K7Y>$ za3pK|li;8;6D~qy&yXS5nhXd&yfFamZcq)iX7l1*nq_4!({@M-tB{3sWYUXtv4@kZ zfqoYh$Tt+i)=O0d`)is-EE+(0_4YK>n!$bZJE>0W1=X24Gl&i0HHUuQ3(#a5jF?{T zV1o{3wxb03;fm=LlLEQ3NkUA%NgHSDZ>S@HGtbyyqa%U04>Q;Uo~ zry6TFU|_gHKIjnJUkXZ7$IxTAUf4Mv_uZ;KlzAGC6rie%0{+;nt$3k#$ajSPS+_h^`99lo*X)(4g zL=|hk?YO6-4u9yOOf?pqQ-pc6u0r?IQF0^hms(9ae825Ht^Ve=8h5fXQqJZ&VS=UBqaAGrYq^!I=FKzN_VU~ z*C@8PYZATdSRZ`JU64*q>Z%=NGNvDULT@ep6Vp?-bb|>H7pPgeUG8+-*U@r~7dq(p z=&iY(nt9&F;FHSb24-R~GqTcm+afF}tj_Y%~w|*IwMhr-QjS;_tYnJOck#?JMZKx7X)p*X1aI{DfeyXRmy4_trT_LrliX zU@_8k>)}^fIMHx2E*OFzIh;J@>clYdV{PxYev^fv5XJheW&ydIK&+P$<5Y=fje~yD z*bPLH)R_M3hs)7gYS8E-M->O~AEbJc8`?AE$=L3iHajP5cqV{IG=%GWjl7(j^c~3r z<}wuUMP*4x89o@{ncErqXn7Rsp{<3$K>pyU*{Fk!p3JqRb@#q8YdKNn9r^YeE|3mS z)4KbW+_4GTy%8SxI#+7f)y$#X5v#AZFq@JEyZRVCPp8!_r3#a_O5p>3rZ#eLFUi8y z6bSxaGn+jiH&z&O06mV~kTIQeAfk#IwT% zrW~1amc$J)WzPyMvOThM-&GlBkDHIF(8s-Kz(gKw+)i1V{U?CisCwIaHBXY#>^0IF zA|DfNO>8qE;uZUBtBZ|(|KhhzI_$L0+#<(n0KY~ze_G$AjJ|b-kwKgpy;{2GQKhs% zqnx|_v3M;$bYPlHdM}E=O*zd0F`zhSSKNG0HoRs&qc2;#Gl7kAE;6B0G!LVzCgQ9< zV~(cn$`Vp}mwkc>)dv&XS!$)3zdT?@rH+_{1lBU4Ih66mh}w2wEev69TF+E#+;p$4 zeJhInqjkz{(97ZDd4)As&2Z?aHR968X(2D8Cz=O^8retd0xHYZPR;jI2J;#?^9ArM z!JW#o&2WUJ@_3*RS^;=^AKksENlg|KW+(M7L^wL{O^A4JI(?NsUx=E;nEO%S8q-Ls zP=!(X6ERQq1c_@@Nng3Dk><`qXGJcKVTnU5UIud1J;&Y69ahdiAgPy%+KnCi0=F(3 z=bJCRfw%%xLFHbB$N+aKc@#YK5Q`hEjWr z{iCrb1^uM9>%~MN$jiWuEjT4GA)ssQx;WS=4||n^)=@RLHdx-t z(yWss%0UFlAgmz|M&zd};hp@(q4te>yoBR0525`I`_X&`Yl@)Dpz3yvgiAZSK0oYu z&|L2ZJ!$3LLjcd2v(5h9MN+QkYFH&aQE8IxgA&bO!=8FAkdLXGncYYCW2-`5>)nh9 zYGpYjx(+4qofL-v%x;ypF-LapBJak9< zehRq^FU$1+!gFQh*23cpi?=x&etw2VzMKc$fF=JeE*Sr8Jbyw!a(58z|5HE@K>c5T zou`Ve;RU}MY0q~;{Wn}5nRY|h)bT?GBG@ z9z*Bxc*52~d!_3x3%ee030hvT9~vS1PlMRx=o*rUM*;$xQc_YtJYX~Vj3_Yyw0e^Z z6Ob4yHu=xu1JzNJR-wV$zeNd>4DY{|>fi4L`nO@)`JgZVe@`)`xA|m}hujvbMas7&l3JV-FF!>F>$jM1aXOghy_F`v0+Tv9Y z;B6lvfZPzMzQlvF*;rWQX zvLr*8b;R%WWj%e$W>K~fH8Zdt>R3Zpb<%QT~xk##>+yfWb=BDVBmvR4# z3oqkLTo3){T=_W4|00@E?M>KD zxrte)9jELV06G1VEJ$82&D9YjKFKOjhIhUnjM}MSqE|f5*mEx#6_edm8&=`t3qo5t z9CfaF7`=cV^S@`EyDnw{vbRsbeW?n{iY|E8eY6)f?+n6ef*ct2*oltd@joHE?ty+z z`HBtt+gPG0U)&rVDr>{>q|#vkTd+oSZgSxWhue#wX1;|p<{i1lZj~`A9=n$tlj>*x zXrt4&+F)L^wppoB>Jq%g2t9^|%@QJw+m&1*Ql7?$ruYZ9MnlniX?g-PJbFeiHkeO` zTcQPfT~E6irg$91vE7l=r%V?fiDsEqlElV-8O1*vvqVdcs+54>Gt9TV9k{G;-{H;U zq4Q=9GnM7f$I=PX!$nA)-O0IY>Wts@vOCB1kL>*T-5$63wye-&UJXLx9EHv_tRsWt ziW#iMm+~q9ba*0Q58uluZ!eE}ptcu<)`8;MlaF7`m>3`AwU6A~(Wn@7RxKQI{GN0LZ zDpd10uD1E=hKC7jU&HCu?B9fW2JqgcMJ?#~5&y4;c47g?8xsSN?3p%n?pvlrbo?XL z=$-A%%{r+E?)~1QJZEmz0MN6AULwr7562*T`nuk&@L%}Jw(R#I281DVkuR+@4652t zzEPT1ELILB!dsntZaztZ<2IkPn~$?N!2wkg)UH?3(V6 z)_yvN0}s^4vrwWW&m_)mljxDW0~>|GG4fp>0ButiP{XNsG^}=!oC2|uI)y`)Ud^UH zPuq%vf+p8I=_jYsk>F^TBedRGT6;yiV=R|xv(4_(ato0#pjF$xd}?+nn*F`P$ah?s zvSJIgY)kM?pDjwggWL&PN{r*D*QM!un@uI}I(9iGnfXwWYZs%hllEnQO@V@rEzvR= z^tR^Ck+A|HY){-4q!PZj=I|^KhtPK#-qn1xMNLg0r>sm?psq}m8$1V7<8Elpwk&9L z9AD_EfrH1|uHDKRBt_g_zC>4O`kqyiiu(dj@`-N+I5QpXjk7PY^u{+owZjf_)ngkz z_v+hjVX5|_zhU?*sfw%HMr>AnyMI&tHdL5~+fcJc)V6KF=K5_+dTFjds2|&D6BKq7 zx60({CTl2KP|~mn%6BqVfdvH9Bp%lZW%UU)Q7e}1Ht1Et6t^-}-ID5?7dhWm>N7ie zeVsXZ=9)yYE__)-_s3jeySa8H7Og6 zs}&lxvYeThPF>AG-{?leo zrqN+X-`)YIj{!X#^>c2BP{cu4JB0S3taRjDnnzktL>94k22U?o~IHdGmZ?l7Y4CI zuR*eQO$rYm6n%Nk#7&i}MOwRiYkF+HfSL~DPQ|M4gYyW2EwGN$Z$D;l6Grwk9vpGu zhFdTqS0}oEez?ufA9kSNl|j{Xy5??U+H*OGx#mVbIffi!gJ2Efo)Bn$L2;4Jo&D-l zOvcql?E7jf!x<0dEyBQV@0DzXaeXsi_rxBqy;t+)IJ@&QB}AcPlQ6J1qrIKn{w5%i z{1v23`sff5-;J@udxmqGRFfEo;K@ao3BIdD>gy@IN7a-WPw|{Cd2UbzLS!?hJmt?R zrkb;13yV=}+(>ea-aubCs=GaKVv!ul2x-Oq#^(UiJdeokmVY6x+tQ;`pZD>h-XOr3 z6=y}|RXfu-_^HXtuqD27PRQh|8sTj=LH|_`8%Pg%GLafAH|EphF`yKkdWq>3{WmM( z59`FpT5!s!Q?8IPxUGYKyfn$+XN=KTkoR!gbGjQeHI`pM9O%W7$4mW?eD0R2lW%8wu7p-hTnEV5?0dBa>Hb%pXHO zi~XT-5Jtbl0O>g+&(AS-xD_uqk$X9c?I%K5a%MsGq-9&reRAc`FXId{hFDUoQJ0ycrDp01_S{$~%q zr<=qL47VPiV4CQmAA6&J%-7fLU)n{oIa~@Qf9(>m6YA8(b0`$IFy(CB1E`M&re1-) z@?nWZmVYA59l5_C>YT0;%)kLnYQBd0JtG>Z%{7e;``#l}rY)&3I&-Q?R6m7Aj+B{> zpI0lD#g%O_)KrO={dz38DrL4!ZUNzn%G=(1^`5m!)sv)L@lN^0A+Mp!d40LPRUM5^ zd8{1VXWPZ89bHSzQEI^qa^ve?X59QM?;Yx(6QlDeY917`tB4V1nZ~$MS4P4;awk>N zPT0Yvs4Shwbm$JpiTgJQNd>-LT7?_6@O8fd)8sNFWH)SDliotm{+<432CuaD!j83; z3^{$6Nbr19tHp#IHK4AZeXT8t`k5OTuOL-VDoislFpkPBX+VlVJ_Wc7i`1VTCTXyO zTx?iAW$_F}mPfTK1=~8>R)FspNg-c4nLZLuqB$Xs_ug5jHmOhPAEkm9(GlTjy{+`O zh%wS9pr09grB3nRJDwnEHB?reuu;m#L3XfAmgi;)o3HrFMyPq69>!h;d!!CU;7i$} zqH@bi`FK7|VFKsbAz{hXDK>;wKaZ>qp9YpK(psGU>n@L7eQvzlhiuByS6TZyO1^bU zl;5w-RkbM266O`e?CVG`Mhx0LpbM5K6QNLYth@LEqZWj~1x@Btn5MsNdc4l*y<~~`ZvA$A(_X&UP zk0}d1+C4U?(W14eimsHH5cb7*X&ZgXtGh<@WcMP;NX^m`_6hesko>u3)QFKVT}NzH znsDqm)g*|7{jq%s7sh6C$;L?Nx5sYlbc=F}7%(Rama0um3Kk4J#&7p|o(h({I&Ati z3g;i#5lo>&I@ZXQ2%MXsdP zHIPa}jc7zZp!A~Xx!XrH(!a>;)zb@AKM`U5LEnI=T{#9-3-dm;BEq@;WZZAS`SUBd zjKVd4-+;qiltlTg?tIth;F zWF?7=M8(Ow=-+c;E6*_u}_rMkWRJ}c6M2S?#!EPC!51Hz;E|Yc$Cqg%UOot{ za^QEIy6=)ipkDBPQuoTb30{ivfT<%(5a#S``q4d7nCJ=zxmmn*ME39@VS z8c?qfEd{&qrMx32?_H0LNf*62u2_|#KBHk=R+V++)Bvz^i>W@dnr{;Z_U%F>!U|lO zoU5zVsR*-8HgUgH2QBMcE0<1!ZGIFRu9#55o|IgyEV(D7ci6E7P01O~dP-VQ6h%R~ zl&HbU`C-sm{cH%tblfOiU>+kyw^%(#>csibal9Sf5ow(=>|qOCR$9x)Wrhc54;oeR zS>^>J29~RF+4k^)NAF?|QqFpBuLbpj%RjO(L7?H=THo1q*J(3HiZ~1W^%I~9)$4le z=naUEt)qqR$c^qkJZqqJpyrfc!*HB)#dzJGpH4cvk=64P2L;+v{GZ?Be7)QAXoJ`r zt8Nhj+S03RK2JAwlndOAFD}*hm1_jmEoljw>4t|psRy8bx*54Wp>h!p+N2B%rYgJh*$}C$nzSLL*Tt8Y z2tMW3vZa8{4M3aK7j(&>C6v+i6UWE1RrjQuG~&h*=Ca(%CN4caa?&YSiO}62NrF9n z+wncRGl61~V|a<7Xl2!5C)IK)Flcc^@td%zneff-me+BK+%hx@nJQM#XKH{YctRBs zJCw0cU0}19I1>F*AU|ke@P#{^!*gVrd`~a-m4(zK{k3%5(|fjwPofSs^8;BGTk@)B(e6GWb<17%>?s=9X#>zt*5{kN z+!*ao(6Tv6R6>*zuLS3U-6J@iDRU+3%X1VoII2Ud^Wtl!cZR!!4EvGbQ4V#Yjv7{R zDBqFk<~J;HeF}u$5==It2Sa7twO3>AX(q0PoFTsJwz!SZ7Rs(lEMI=SA*wa!CNhBn zz3zE=Dx|ht?HNhaLrt9^9v+6e@;bLAhRZ5XFVCrl4bh~prd<@V<^3+&hh876*_C9nva{bmfBrn3lXhsK)=|NoCU%6dbu+K9DR748ETX%N<|%gC=l({e zmJ)p((){J%5XAwJ{(cCRxZ=t6Io2m8-!-b=Kh&hWA2Fj2t2y##St=T)`pz&W1DqN? zJWJ~97Tm?5)QruF-6tiz*hq+ENLMg0oWys=XgZ_bj`OvI^K9&8FTO^`&QRx4#hj#& z+J5F9kB?fdZ_Wu62vQ(r+}oRlgGm|1363sI4i{oC8x?@FrvE2G_jLTLKfEMaP~EGz3a(#pc?-?}@(kE)d{BHKz^Nl3~lFVcJ()9m?tE8^=gDaogdER*x9GEi4m zzx!85>tCvv=K$fqu>xwq-3k{(glQk9-02)dnBxwt9 z0q$nXO1@PCu*Aq1|3Z_0da7U%cLPifG&z5piV^c4iVzWxR7O7DqzRQ&QVIqj^Zx*z zKZn&v|G!k*%R7UH{smC}U)b!jjZSnsWiVW2#y?-6pnP644*mZNrTyQKJ9veI=V{T*(sR;yT_UvtDe<{lSgdm zReWCGTS@VAnC~DjjZJY9tH_xo=r1XsB~}2(=6GU7MBhv8O=hCH+toW~@8|M~mtY-^ z(5!!#H|pUaLRKp2SUKEd&em&ZdTHd{BWAMj9phg!yQ-%y1nbJ=F8ZK znauOrYk{3Kj#u&ON&YPf6gNyI3a7nGIZhX=-Sx3QQz8fZdv`Z_RU8+b&a`cmx=sr0 zr2CXc6t5}5HPQH!w=bx-9%0ObnxZEdzKT-2 zDa5@we~^B`TQOa*)k%Dydj0;0y;=8+mQE!M+H-!StwWQvA{9|pn-;i!FF?qD;AVoo8IX-F#rxbp9c*o+lX8t-_C#V&tTxKGp_wHJCI{I;o4 z+{BuuzIVi^q&ZgUpKfcw_5_6K9vfkpLPTSTiNOP?-j2kEW{# zOl*+N7S7(X8_q)%0^?rO&1m6_(9Q82=Pf?v%mZ}>9|=+~e)?rb+d)v(yV2bOYvC$o zsbqq9vBk(qOL4@~A&o#%zyoE4%u^z#S3Y~!+Qv#u$gvvpoTq2S>&l(m4DKq-jHg!P zuKqZaDc3?(FvDri_SjjuSAyZ6_8@epA@$ivBw~ImFiNzH_^`R^5dRpT)tp$-$y!iyCs(f%Hv9zY$jmgSly(R6e;67$5EjHP0^G-}c%pEo|eR{qEQWCy)o z!|IdV}?fqRw53b&_thY*cfD*%1omI2r)`O$7sSr@6; zP#~iKbk3rBQtm{)2^~Q7bMYLmqIVT4jslIADveGVCg1)M2v`S)wSaCWY$~sKf0&zB zuC6v7Jq)-R(UuzWOgri6;@9GJ2MM;`dI~5KIrFVt1h% z{$i!ZGs*}%!||1rqyZyRjP<)8UV868YuEm&I*}@SaF!!0&bUk~ZE!xmGuQQ}+#zML zn}3!sgWVygryoUQHYQJ3uG$|hnjIFM%(bQDfay$cfjy_IkB;6?9enq6^x7)iOPm-g zM4-&OP^jRle4^GGH_Bsk($@N^`aieb&vVLx@KYyNOq|zZ%+HLIa+v5avNlCi+091| z#@mzGFuD^?*YrTR`iM_>a1q9u@wICUeuLho%!$jP$AS-<&)#F~dUnXjguOdPwJ{l@ z_~XDIVpn-@Qao!APEhYY9{N4gTqMWLsJ$?O&3IaS6o(7ZMxnsgRG!26XOwuXo)>O6 zWy#GfeAV3Z>!glWZgmTP?E>a^=k*xz2Q8}@?7@MG2cf|h6MgVZsCmPk=TE>zl-pY1 zkf7cL`#~)JO9gwj{7g5DY#3UEOyxKBsggVGM4kPp^%volw?47Say{(=Pw8OZXfp8O z_cl(Kz*e(NbFrYe=e8sEJ1j;5^}Ic}Y;;D%xmHet_!A$V)6ga0#2Ginnzw_tKeAPi z(nts$qJ-CM!Jj76ZQ^(R#;TR(}gTg~{Ej<%7)74n0m@Ti=|(+j={Pn1toIN49X zdty?x^e5)XK2pBIH7DLYj}r`7oLJ8WfNqzY2ifiCGb-s znPR6g{kHi()&@{h;s7D^)Y35LurtxN<^-UxJja|^#*yj;5q*(8%{~}st8Z0$CvsCq zNpPRT0?+&R#S!m39Fx88`K?KUtJB0zI}&57@Inzrl1(Z}2XPF-b3u3Q844pzWL)<4 zFGZl|!%}kj-I=X!YC@8=iR%$v84O#PKk{^ihwqodofC?T%1=9YyLYCxHdK#*r$~su z-LQ%dD?Z_k*0N-)7YN!I%_3zV7q( z?|EF~veDv5WbRO2)$q!XTn8BTtQ9m$HJTj`>^s0!-Ct6T6<)LQ;D1RYa!Wbfwy#QD z-^WST!)WBo*euoXLN`W&DhOlxde!GSpwbJyPxnwH*ZSmu+pO)7_LP+OVK284U~3*#a53C{)CCYl;0RUZ}=nb`uhaX3iJ(z^OK_p1OGMo+VNGC6t;iS#`l*chu5A+UqFG-t()>e} zNbjHW!I!7x&=yd->-6A zK74L}R@8!CwECfw=_gj3@#PES5d-5jVf$43&x`3@QXx=9TuiKA%+0EwTR^}C1@5`{ zM&4I$;hgTr=^GsyQDXgAg8mSa{H5-%5v9Z;!y;|RV20%?iVB%6`FJ=3Z-|%I7ROtI zkU?OCJ@vp!*<1q{b4W`Zhfh!#8TA+`PcS=f-n-z0{khO2gd0XdE>1U+>x6dMV?mdi z*rDo7)Vm2{{% zUEG+hciogq3RIwE9IGGL&!j1B#fW$~s!oBcC9TPQQDG(xF{GxXa(wynz*&HYSpuvx z%_m6P2#4|`_YQ&1W3cIoB4zdFpIuHXA^d9yl9UmMtz~J%uoe>0HK#)-`sP!~;zIr9 zsBP27k?X-pPdumNJz*E)cq@EsJ`|H`{fRJ?Z2E%ZHehlV%v0Q~Z`M2$nSW?6G%s>? zyj^yaoSgjS@lli}M7F&P$U{CkgT+pLPTLopc=ZBbF4rbh>2k6k($Xtjl$Gc*Gfp^fEvz|K>zC-~%9^=wES}>It;aZ*x$|y6MKTCRmE+kE@M!E8`bQ z%#L$Xp7>@E(#eo6I^+;nX?9L9(w1(1 z-AST;PZ7@p&8o&uYNsGvnG!MQ(f;zP9If;}0YCJhgW_c3-B|?nA~jbQ^yX?sF}@>N z*zH#H%AYjdlev{AQC-Q&gGQAw+#2_YQBiXwDb9@u^ajYek-UDIjVLWOMCHy8=BEqv2J~3GCha=+5Zg=)q{g2}` zCab=z@jLByzZ0@s^wsI;7ET#g+t}sY$@c0R*Djk`Y45jJbJ);3Jgn9oWJ^xf+2&<^ zm&(=a6OwI~@eJttugfC_e^7X_+2cnZuu$!z>6)(Fs`f{iYv&-uo9O?O%IKjo(`~@^ zabrQ?$|MGIn`U|5?7Ar)MD5WV08Na91jnK63!Lr%dmwB(D>;4Tyn~-|>P>%2)?Cd# z+8DWFHBp(lxM+$${Z7T|&@H-`k?no{u}~9Odo($C<|Y+m_;_}!ACx-o+qv(-{kI99*Ym{NV4LLr3vR$c#>x^ uFKikU7^c1~)GGeRJNRy;nW@vUGx79+dpGX2bGe?kla*4EERZnx@;?9zfumjk literal 0 HcmV?d00001 diff --git a/docs/zh_CN/系统使用手册.md b/docs/zh_CN/系统使用手册.md index fc0e999118..595850bade 100644 --- a/docs/zh_CN/系统使用手册.md +++ b/docs/zh_CN/系统使用手册.md @@ -60,7 +60,7 @@ ### 执行流程定义 - **未上线状态的流程定义可以编辑，但是不可以运行**，所以先上线工作流 > 点击工作流定义，返回流程定义列表，点击”上线“图标，上线工作流定义。 - + > "下线"工作流之前，要先将定时管理的定时下线，才能成功下线工作流定义 - 点击”运行“，执行工作流。运行参数说明： @@ -98,28 +98,28 @@ ### 查看流程实例 > 点击“工作流实例”，查看流程实例列表。 - + > 点击工作流名称，查看任务执行状态。 - +

> 点击任务节点，点击“查看日志”，查看任务执行日志。 - +

- + > 点击任务实例节点，点击**查看历史**，可以查看该流程实例运行的该任务实例列表 - +

> 对工作流实例的操作： - +

@@ -165,7 +165,7 @@ - 密码：设置连接MySQL的密码 - 数据库名：输入连接MySQL的数据库名称 - Jdbc连接参数：用于MySQL连接的参数设置，以JSON形式填写 - +

@@ -191,7 +191,7 @@ #### 创建、编辑HIVE数据源 1.使用HiveServer2方式连接 - +

@@ -207,12 +207,19 @@ - Jdbc连接参数：用于HIVE连接的参数设置，以JSON形式填写 2.使用HiveServer2 HA Zookeeper方式连接 - +

+注意：如果开启了**kerberos**，则需要填写 **Principal** +

+ +

+ + + #### 创建、编辑Spark数据源

@@ -231,7 +238,7 @@ ### 上传资源 - 上传资源文件和udf函数，所有上传的文件和资源都会被存储到hdfs上，所以需要以下配置项： - + ``` conf/common/common.properties -- hdfs.startup.state=true @@ -242,7 +249,7 @@ conf/common/hadoop.properties ``` #### 文件管理 - + > 是对各种资源文件的管理，包括创建基本的txt/log/sh/conf等文件、上传jar包等各种类型文件，以及编辑、下载、删除等操作。

@@ -287,7 +294,7 @@ conf/common/hadoop.properties #### 资源管理 > 资源管理和文件管理功能类似，不同之处是资源管理是上传的UDF函数，文件管理上传的是用户程序，脚本及配置文件 - + * 上传udf资源 > 和上传文件相同。 @@ -303,7 +310,7 @@ conf/common/hadoop.properties - 参数：用来标注函数的输入参数 - 数据库名：预留字段，用于创建永久UDF函数 - UDF资源：设置创建的UDF对应的资源文件 - +

@@ -312,7 +319,7 @@ conf/common/hadoop.properties - 安全中心是只有管理员账户才有权限的功能，有队列管理、租户管理、用户管理、告警组管理、worker分组、令牌管理等功能，还可以对资源、数据源、项目等授权 - 管理员登录，默认用户名密码：admin/escheduler123 - + ### 创建队列 - 队列是在执行spark、mapreduce等程序，需要用到“队列”参数时使用的。 - “安全中心”->“队列管理”->“创建队列” @@ -357,7 +364,7 @@ conf/common/hadoop.properties ### 令牌管理 - 由于后端接口有登录检查，令牌管理，提供了一种可以通过调用接口的方式对系统进行各种操作。 - 调用示例： - + ```令牌调用示例 /** * test token @@ -477,15 +484,15 @@ conf/common/hadoop.properties ### 依赖(DEPENDENT)节点 - 依赖节点，就是**依赖检查节点**。比如A流程依赖昨天的B流程执行成功，依赖节点会去检查B流程在昨天是否有执行成功的实例。 - + > 拖动工具栏中的![PNG](https://analysys.github.io/easyscheduler_docs_cn/images/toolbar_DEPENDENT.png)任务节点到画板中，双击任务节点，如下图：

- + > 依赖节点提供了逻辑判断功能，比如检查昨天的B流程是否成功，或者C流程是否执行成功。 - +

@@ -536,7 +543,7 @@ conf/common/hadoop.properties ### SPARK节点 - 通过SPARK节点，可以直接直接执行SPARK程序，对于spark节点，worker会使用`spark-submit`方式提交任务 - + > 拖动工具栏中的![PNG](https://analysys.github.io/easyscheduler_docs_cn/images/toolbar_SPARK.png)任务节点到画板中，双击任务节点，如下图：

@@ -563,7 +570,7 @@ conf/common/hadoop.properties > 拖动工具栏中的![PNG](https://analysys.github.io/easyscheduler_docs_cn/images/toolbar_MR.png)任务节点到画板中，双击任务节点，如下图： 1. JAVA程序 - +

@@ -592,7 +599,7 @@ conf/common/hadoop.properties ### Python节点 - 使用python节点，可以直接执行python脚本，对于python节点，worker会使用`python **`方式提交任务。 - + > 拖动工具栏中的![PNG](https://analysys.github.io/easyscheduler_docs_cn/images/toolbar_PYTHON.png)任务节点到画板中，双击任务节点，如下图： From 98252e990d16f4128c612d89ead5ce959fce2db7 Mon Sep 17 00:00:00 2001 From: lenboo Date: Mon, 15 Jul 2019 18:51:07 +0800 Subject: [PATCH 137/149] add 1.1.0 documents --- docs/zh_CN/1.1.0-release.md | 54 +++++++++++++++++++++++++++++++++++++ docs/zh_CN/SUMMARY.md | 1 + 2 files changed, 55 insertions(+) create mode 100644 docs/zh_CN/1.1.0-release.md diff --git a/docs/zh_CN/1.1.0-release.md b/docs/zh_CN/1.1.0-release.md new file mode 100644 index 0000000000..ae17c7d0ee --- /dev/null +++ b/docs/zh_CN/1.1.0-release.md @@ -0,0 +1,54 @@ +Easy Scheduler Release 1.1.0 +=== +Easy Scheduler 1.1.0是1.x系列中的第五个版本。 + +新特性： +=== +- [[EasyScheduler-391](https://github.com/analysys/EasyScheduler/issues/391)] run a process under a specified tenement user +- [[EasyScheduler-288](https://github.com/analysys/EasyScheduler/issues/288)] Feature/qiye_weixin +- [[EasyScheduler-189](https://github.com/analysys/EasyScheduler/issues/189)] Kerberos等安全支持 +- [[EasyScheduler-398](https://github.com/analysys/EasyScheduler/issues/398)]管理员，有租户（install.sh设置默认租户），可以创建资源、项目和数据源（限制有一个管理员） +- [[EasyScheduler-293](https://github.com/analysys/EasyScheduler/issues/293)]点击运行流程时候选择的参数，没有地方可查看，也没有保存 +- [[EasyScheduler-401](https://github.com/analysys/EasyScheduler/issues/401)]定时很容易定时每秒一次，定时完成以后可以在页面显示一下下次触发时间 + +增强： +=== +- [[EasyScheduler-227](https://github.com/analysys/EasyScheduler/issues/227)] upgrade spring-boot to 2.1.x and spring to 5.x +- [[EasyScheduler-434](https://github.com/analysys/EasyScheduler/issues/434)] worker节点数量 zk和mysql中不一致 +- [[EasyScheduler-435](https://github.com/analysys/EasyScheduler/issues/435)]邮箱格式的验证 +- [[EasyScheduler-441](https://github.com/analysys/EasyScheduler/issues/441)] 禁止运行节点加入已完成节点检测 +- [[EasyScheduler-400](https://github.com/analysys/EasyScheduler/issues/400)] 首页页面，队列统计不和谐，命令统计无数据 +- [[EasyScheduler-395](https://github.com/analysys/EasyScheduler/issues/395)] 对于容错恢复的流程，状态不能为 **正在运行 +- [[EasyScheduler-529](https://github.com/analysys/EasyScheduler/issues/529)] optimize poll task from zookeeper +- [[EasyScheduler-242](https://github.com/analysys/EasyScheduler/issues/242)]worker-server节点获取任务性能问题 +- [[EasyScheduler-352](https://github.com/analysys/EasyScheduler/issues/352)]worker 分组, 队列消费问题 +- [[EasyScheduler-461](https://github.com/analysys/EasyScheduler/issues/461)]查看数据源参数，需要加密账号密码信息 +- [[EasyScheduler-396](https://github.com/analysys/EasyScheduler/issues/396)]Dockerfile优化，并关联Dockerfile和github实现自动打镜像 +- [[EasyScheduler-389](https://github.com/analysys/EasyScheduler/issues/389)]service monitor cannot find the change of master/worker +- [[EasyScheduler-511](https://github.com/analysys/EasyScheduler/issues/511)]support recovery process from stop/kill nodes. +- [[EasyScheduler-399](https://github.com/analysys/EasyScheduler/issues/399)]HadoopUtils指定用户操作，而不是 **部署用户 + +修复： +=== +- [[EasyScheduler-394](https://github.com/analysys/EasyScheduler/issues/394)] master&worker部署在同一台机器上时，如果重启master&worker服务，会导致之前调度的任务无法继续调度 +- [[EasyScheduler-469](https://github.com/analysys/EasyScheduler/issues/469)]Fix naming errors,monitor page +- [[EasyScheduler-392](https://github.com/analysys/EasyScheduler/issues/392)]Feature request: fix email regex check +- [[EasyScheduler-405](https://github.com/analysys/EasyScheduler/issues/405)]定时修改/添加页面，开始时间和结束时间不能相同 +- [[EasyScheduler-517](https://github.com/analysys/EasyScheduler/issues/517)]补数 - 子工作流 - 时间参数 +- [[EasyScheduler-532](https://github.com/analysys/EasyScheduler/issues/532)]python节点不执行的问题 +- [[EasyScheduler-543](https://github.com/analysys/EasyScheduler/issues/543)]optimize datasource connection params safety +- [[EasyScheduler-569](https://github.com/analysys/EasyScheduler/issues/569)]定时任务无法真正停止 +- [[EasyScheduler-463](https://github.com/analysys/EasyScheduler/issues/463)]邮箱验证不支持非常见后缀邮箱 + + + + +感谢： +=== +最后但最重要的是，没有以下伙伴的贡献就没有新版本的诞生： + +Baoqi, jimmy201602, samz406, petersear, millionfor, hyperknob, fanguanqun, yangqinlong, qq389401879, chgxtony, Stanfan, lfyee, thisnew, hujiang75277381, sunnyingit, lgbo-ustc, + ivivi, lzy305, JackIllkid, telltime, lipengbo2018, wuchunfu, telltime + +以及微信群里众多的热心伙伴！在此非常感谢！ + diff --git a/docs/zh_CN/SUMMARY.md b/docs/zh_CN/SUMMARY.md index 0646577560..d69f530cd0 100644 --- a/docs/zh_CN/SUMMARY.md +++ b/docs/zh_CN/SUMMARY.md @@ -35,6 +35,7 @@ * 系统版本升级文档 * [版本升级](升级文档.md) * 历次版本发布内容 + * [1.1.0 release](1.1.0-release.md) * [1.0.3 release](1.0.3-release.md) * [1.0.2 release](1.0.2-release.md) * [1.0.1 release](1.0.1-release.md) From 8c13e7a7f2b7ad26d86b07369108ae9eed706ab5 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Tue, 16 Jul 2019 11:07:32 +0800 Subject: [PATCH 138/149] sql kerberos task update doc --- docs/zh_CN/系统使用手册.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/zh_CN/系统使用手册.md b/docs/zh_CN/系统使用手册.md index 595850bade..b4d6d6bd9b 100644 --- a/docs/zh_CN/系统使用手册.md +++ b/docs/zh_CN/系统使用手册.md @@ -215,11 +215,12 @@ 注意：如果开启了**kerberos**，则需要填写 **Principal**

- +

+ #### 创建、编辑Spark数据源

@@ -236,6 +237,14 @@ - 数据库名：输入连接Spark的数据库名称 - Jdbc连接参数：用于Spark连接的参数设置，以JSON形式填写 + + +注意：如果开启了**kerberos**，则需要填写 **Principal** + +

+ +

+ ### 上传资源 - 上传资源文件和udf函数，所有上传的文件和资源都会被存储到hdfs上，所以需要以下配置项： From 9c9310d1881967588ed8f0c254cac0a14055f342 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 16 Jul 2019 11:16:41 +0800 Subject: [PATCH 139/149] [maven-release-plugin] prepare release 1.1.0-preview --- escheduler-alert/pom.xml | 2 +- escheduler-api/pom.xml | 2 +- escheduler-common/pom.xml | 2 +- escheduler-dao/pom.xml | 2 +- escheduler-rpc/pom.xml | 2 +- escheduler-server/pom.xml | 2 +- pom.xml | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/escheduler-alert/pom.xml b/escheduler-alert/pom.xml index be8b5fbb6d..cda2488c36 100644 --- a/escheduler-alert/pom.xml +++ b/escheduler-alert/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 escheduler-alert jar diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index b0d92d54a8..b3d9fc53fd 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -3,7 +3,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 escheduler-api jar diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index 35b025b1d8..cc2bb36717 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0 escheduler-common escheduler-common diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index 4d8fb6912e..3eb6dbf3e0 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 escheduler-dao escheduler-dao diff --git a/escheduler-rpc/pom.xml b/escheduler-rpc/pom.xml index e3d7c91a92..4687d1e462 100644 --- a/escheduler-rpc/pom.xml +++ b/escheduler-rpc/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0 4.0.0 diff --git a/escheduler-server/pom.xml b/escheduler-server/pom.xml index 7f149278c8..ac9a312734 100644 --- a/escheduler-server/pom.xml +++ b/escheduler-server/pom.xml @@ -3,7 +3,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0 escheduler-server escheduler-server diff --git a/pom.xml b/pom.xml index a97650aafb..93ac6f8ee9 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 pom escheduler http://maven.apache.org @@ -382,7 +382,7 @@ scm:git:https://github.com/analysys/EasyScheduler.git scm:git:https://github.com/analysys/EasyScheduler.git https://github.com/analysys/EasyScheduler.git - HEAD + 1.1.0-preview From 900379a74fe38b3f8609571be7f2e3e584b2736d Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 16 Jul 2019 11:18:50 +0800 Subject: [PATCH 140/149] [maven-release-plugin] rollback the release of 1.1.0-preview --- escheduler-alert/pom.xml | 2 +- escheduler-api/pom.xml | 2 +- escheduler-common/pom.xml | 2 +- escheduler-dao/pom.xml | 2 +- escheduler-rpc/pom.xml | 2 +- escheduler-server/pom.xml | 2 +- pom.xml | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/escheduler-alert/pom.xml b/escheduler-alert/pom.xml index cda2488c36..be8b5fbb6d 100644 --- a/escheduler-alert/pom.xml +++ b/escheduler-alert/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT escheduler-alert jar diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index b3d9fc53fd..b0d92d54a8 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -3,7 +3,7 @@ cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT escheduler-api jar diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index cc2bb36717..35b025b1d8 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0 + 1.1.0-SNAPSHOT escheduler-common escheduler-common diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index 3eb6dbf3e0..4d8fb6912e 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT escheduler-dao escheduler-dao diff --git a/escheduler-rpc/pom.xml b/escheduler-rpc/pom.xml index 4687d1e462..e3d7c91a92 100644 --- a/escheduler-rpc/pom.xml +++ b/escheduler-rpc/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0 + 1.1.0-SNAPSHOT 4.0.0 diff --git a/escheduler-server/pom.xml b/escheduler-server/pom.xml index ac9a312734..7f149278c8 100644 --- a/escheduler-server/pom.xml +++ b/escheduler-server/pom.xml @@ -3,7 +3,7 @@ escheduler cn.analysys - 1.1.0 + 1.1.0-SNAPSHOT escheduler-server escheduler-server diff --git a/pom.xml b/pom.xml index 93ac6f8ee9..a97650aafb 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT pom escheduler http://maven.apache.org @@ -382,7 +382,7 @@ scm:git:https://github.com/analysys/EasyScheduler.git scm:git:https://github.com/analysys/EasyScheduler.git https://github.com/analysys/EasyScheduler.git - 1.1.0-preview + HEAD From 94862f6d51ed0084cfc3e37b60eea1ebda110652 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 16 Jul 2019 11:27:09 +0800 Subject: [PATCH 141/149] [maven-release-plugin] prepare release 1.1.0-preview --- escheduler-alert/pom.xml | 2 +- escheduler-api/pom.xml | 2 +- escheduler-common/pom.xml | 2 +- escheduler-dao/pom.xml | 2 +- escheduler-rpc/pom.xml | 2 +- escheduler-server/pom.xml | 2 +- pom.xml | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/escheduler-alert/pom.xml b/escheduler-alert/pom.xml index be8b5fbb6d..cda2488c36 100644 --- a/escheduler-alert/pom.xml +++ b/escheduler-alert/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 escheduler-alert jar diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index b0d92d54a8..b3d9fc53fd 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -3,7 +3,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 escheduler-api jar diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index 35b025b1d8..cc2bb36717 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0 escheduler-common escheduler-common diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index 4d8fb6912e..3eb6dbf3e0 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 escheduler-dao escheduler-dao diff --git a/escheduler-rpc/pom.xml b/escheduler-rpc/pom.xml index e3d7c91a92..4687d1e462 100644 --- a/escheduler-rpc/pom.xml +++ b/escheduler-rpc/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0 4.0.0 diff --git a/escheduler-server/pom.xml b/escheduler-server/pom.xml index 7f149278c8..ac9a312734 100644 --- a/escheduler-server/pom.xml +++ b/escheduler-server/pom.xml @@ -3,7 +3,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0 escheduler-server escheduler-server diff --git a/pom.xml b/pom.xml index a97650aafb..93ac6f8ee9 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 pom escheduler http://maven.apache.org @@ -382,7 +382,7 @@ scm:git:https://github.com/analysys/EasyScheduler.git scm:git:https://github.com/analysys/EasyScheduler.git https://github.com/analysys/EasyScheduler.git - HEAD + 1.1.0-preview From 32c6666c2807694e61e15ef952fb3accb3bcc53b Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 16 Jul 2019 11:27:36 +0800 Subject: [PATCH 142/149] [maven-release-plugin] prepare for next development iteration --- escheduler-alert/pom.xml | 2 +- escheduler-api/pom.xml | 2 +- escheduler-common/pom.xml | 2 +- escheduler-dao/pom.xml | 2 +- escheduler-rpc/pom.xml | 2 +- escheduler-server/pom.xml | 2 +- pom.xml | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/escheduler-alert/pom.xml b/escheduler-alert/pom.xml index cda2488c36..be8b5fbb6d 100644 --- a/escheduler-alert/pom.xml +++ b/escheduler-alert/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT escheduler-alert jar diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index b3d9fc53fd..b0d92d54a8 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -3,7 +3,7 @@ cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT escheduler-api jar diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index cc2bb36717..35b025b1d8 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0 + 1.1.0-SNAPSHOT escheduler-common escheduler-common diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index 3eb6dbf3e0..4d8fb6912e 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT escheduler-dao escheduler-dao diff --git a/escheduler-rpc/pom.xml b/escheduler-rpc/pom.xml index 4687d1e462..e3d7c91a92 100644 --- a/escheduler-rpc/pom.xml +++ b/escheduler-rpc/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0 + 1.1.0-SNAPSHOT 4.0.0 diff --git a/escheduler-server/pom.xml b/escheduler-server/pom.xml index ac9a312734..7f149278c8 100644 --- a/escheduler-server/pom.xml +++ b/escheduler-server/pom.xml @@ -3,7 +3,7 @@ escheduler cn.analysys - 1.1.0 + 1.1.0-SNAPSHOT escheduler-server escheduler-server diff --git a/pom.xml b/pom.xml index 93ac6f8ee9..a97650aafb 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT pom escheduler http://maven.apache.org @@ -382,7 +382,7 @@ scm:git:https://github.com/analysys/EasyScheduler.git scm:git:https://github.com/analysys/EasyScheduler.git https://github.com/analysys/EasyScheduler.git - 1.1.0-preview + HEAD From 6ad22dab5343554f3368fa85da3702f3b4575f01 Mon Sep 17 00:00:00 2001 From: lenboo Date: Tue, 16 Jul 2019 11:36:36 +0800 Subject: [PATCH 143/149] update 1.1.0 documents --- docs/zh_CN/1.1.0-release.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/zh_CN/1.1.0-release.md b/docs/zh_CN/1.1.0-release.md index ae17c7d0ee..87084169d6 100644 --- a/docs/zh_CN/1.1.0-release.md +++ b/docs/zh_CN/1.1.0-release.md @@ -10,6 +10,8 @@ Easy Scheduler 1.1.0是1.x系列中的第五个版本。 - [[EasyScheduler-398](https://github.com/analysys/EasyScheduler/issues/398)]管理员，有租户（install.sh设置默认租户），可以创建资源、项目和数据源（限制有一个管理员） - [[EasyScheduler-293](https://github.com/analysys/EasyScheduler/issues/293)]点击运行流程时候选择的参数，没有地方可查看，也没有保存 - [[EasyScheduler-401](https://github.com/analysys/EasyScheduler/issues/401)]定时很容易定时每秒一次，定时完成以后可以在页面显示一下下次触发时间 +- [[EasyScheduler-493](https://github.com/analysys/EasyScheduler/pull/493)]add datasource kerberos auth and FAQ modify and add resource upload s3 + 增强： === @@ -47,8 +49,7 @@ Easy Scheduler 1.1.0是1.x系列中的第五个版本。 === 最后但最重要的是，没有以下伙伴的贡献就没有新版本的诞生： -Baoqi, jimmy201602, samz406, petersear, millionfor, hyperknob, fanguanqun, yangqinlong, qq389401879, chgxtony, Stanfan, lfyee, thisnew, hujiang75277381, sunnyingit, lgbo-ustc, - ivivi, lzy305, JackIllkid, telltime, lipengbo2018, wuchunfu, telltime +Baoqi, jimmy201602, samz406, petersear, millionfor, hyperknob, fanguanqun, yangqinlong, qq389401879, chgxtony, Stanfan, lfyee, thisnew, hujiang75277381, sunnyingit, lgbo-ustc, ivivi, lzy305, JackIllkid, telltime, lipengbo2018, wuchunfu, telltime 以及微信群里众多的热心伙伴！在此非常感谢！ From bd2a35ab45ae811791c8cf2ca1864e5dfa6efe22 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Tue, 16 Jul 2019 14:22:24 +0800 Subject: [PATCH 144/149] update doc --- docs/zh_CN/后端部署文档.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/zh_CN/后端部署文档.md b/docs/zh_CN/后端部署文档.md index ea1090a1d4..e39901031d 100644 --- a/docs/zh_CN/后端部署文档.md +++ b/docs/zh_CN/后端部署文档.md @@ -15,7 +15,7 @@ * [Hive](https://staroon.pro/2017/12/09/HiveInstall/)(1.2.1) : 选装，hive任务提交需要安装 * Spark(1.x,2.x) : 选装，Spark任务提交需要安装 * PostgreSQL(8.2.15+) : 选装，PostgreSQL PostgreSQL存储过程需要安装 - + ``` 注意：EasyScheduler本身不依赖Hadoop、Hive、Spark、PostgreSQL,仅是会调用他们的Client，用于对应任务的运行。 ``` @@ -36,7 +36,7 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL #### 准备三: ssh免密配置 在部署机器和其他安装机器上配置ssh免密登录，如果要在部署机上安装调度，需要配置本机免密登录自己 - + - [将 **主机器** 和各个其它机器SSH打通](http://geek.analysys.cn/topic/113) @@ -66,7 +66,7 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL mysql -h {host} -u {user} -p{password} -D {db} < quartz.sql ``` -* 1.0.2版本创建表和导入基础数据 +* 1.0.2 **之后** 版本创建表和导入基础数据 修改conf/dao/data_source.properties中的下列属性 ``` @@ -93,7 +93,7 @@ install.sh : 一键部署脚本 ``` - 修改权限(请将deployUser自行修改为对应部署用户)，使得部署用户对escheduler-backend目录有操作权限 - + `sudo chown -R deployUser:deployUser escheduler-backend` - 修改conf/env/目录下的 `.escheduler_env.sh` 环境变量 @@ -166,11 +166,11 @@ install.sh : 一键部署脚本 ### 2.3 系统常用启停服务(服务用途请具体参见《系统架构设计》小节) * 一键停止集群所有服务 - + ` sh ./bin/stop_all.sh` * 一键开启集群所有服务 - + ` sh ./bin/start_all.sh` * 启停Master @@ -207,7 +207,7 @@ sh ./bin/escheduler-daemon.sh stop alert-server ``` ## 3、数据库升级 -数据库升级是在1.0.2版本增加的功能,执行以下命令即可自动升级数据库 +数据库升级是在1.0.2 **之后** 版本增加的功能,执行以下命令即可自动升级数据库 ``` sh ./script/upgrade_escheduler.sh ``` From a62ba2b45779cf16f0263d2d7705880e3c3f63f2 Mon Sep 17 00:00:00 2001 From: lidongdai Date: Tue, 16 Jul 2019 14:31:08 +0800 Subject: [PATCH 145/149] update docs --- docs/zh_CN/后端部署文档.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh_CN/后端部署文档.md b/docs/zh_CN/后端部署文档.md index ea1090a1d4..4d94279b1a 100644 --- a/docs/zh_CN/后端部署文档.md +++ b/docs/zh_CN/后端部署文档.md @@ -66,7 +66,7 @@ escheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL mysql -h {host} -u {user} -p{password} -D {db} < quartz.sql ``` -* 1.0.2版本创建表和导入基础数据 +* 1.0.2之后(含1.0.2)版本创建表和导入基础数据 修改conf/dao/data_source.properties中的下列属性 ``` From d44425362da939e36aceb9f0d2bbf59f66ba52f8 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 16 Jul 2019 14:32:19 +0800 Subject: [PATCH 146/149] [maven-release-plugin] prepare release 1.1.0-preview --- escheduler-alert/pom.xml | 2 +- escheduler-api/pom.xml | 2 +- escheduler-common/pom.xml | 2 +- escheduler-dao/pom.xml | 2 +- escheduler-rpc/pom.xml | 2 +- escheduler-server/pom.xml | 2 +- pom.xml | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/escheduler-alert/pom.xml b/escheduler-alert/pom.xml index be8b5fbb6d..cda2488c36 100644 --- a/escheduler-alert/pom.xml +++ b/escheduler-alert/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 escheduler-alert jar diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index b0d92d54a8..b3d9fc53fd 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -3,7 +3,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 escheduler-api jar diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index 35b025b1d8..cc2bb36717 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0 escheduler-common escheduler-common diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index 4d8fb6912e..3eb6dbf3e0 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 escheduler-dao escheduler-dao diff --git a/escheduler-rpc/pom.xml b/escheduler-rpc/pom.xml index e3d7c91a92..4687d1e462 100644 --- a/escheduler-rpc/pom.xml +++ b/escheduler-rpc/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0 4.0.0 diff --git a/escheduler-server/pom.xml b/escheduler-server/pom.xml index 7f149278c8..ac9a312734 100644 --- a/escheduler-server/pom.xml +++ b/escheduler-server/pom.xml @@ -3,7 +3,7 @@ escheduler cn.analysys - 1.1.0-SNAPSHOT + 1.1.0 escheduler-server escheduler-server diff --git a/pom.xml b/pom.xml index a97650aafb..93ac6f8ee9 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 cn.analysys escheduler - 1.1.0-SNAPSHOT + 1.1.0 pom escheduler http://maven.apache.org @@ -382,7 +382,7 @@ scm:git:https://github.com/analysys/EasyScheduler.git scm:git:https://github.com/analysys/EasyScheduler.git https://github.com/analysys/EasyScheduler.git - HEAD + 1.1.0-preview From bd30d48c4fce31d8eb9c9d1faec8d8f607f2124f Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Tue, 16 Jul 2019 14:32:45 +0800 Subject: [PATCH 147/149] [maven-release-plugin] prepare for next development iteration --- escheduler-alert/pom.xml | 2 +- escheduler-api/pom.xml | 2 +- escheduler-common/pom.xml | 2 +- escheduler-dao/pom.xml | 2 +- escheduler-rpc/pom.xml | 2 +- escheduler-server/pom.xml | 2 +- pom.xml | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/escheduler-alert/pom.xml b/escheduler-alert/pom.xml index cda2488c36..be8b5fbb6d 100644 --- a/escheduler-alert/pom.xml +++ b/escheduler-alert/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT escheduler-alert jar diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index b3d9fc53fd..b0d92d54a8 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -3,7 +3,7 @@ cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT escheduler-api jar diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index cc2bb36717..35b025b1d8 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0 + 1.1.0-SNAPSHOT escheduler-common escheduler-common diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index 3eb6dbf3e0..4d8fb6912e 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT escheduler-dao escheduler-dao diff --git a/escheduler-rpc/pom.xml b/escheduler-rpc/pom.xml index 4687d1e462..e3d7c91a92 100644 --- a/escheduler-rpc/pom.xml +++ b/escheduler-rpc/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.1.0 + 1.1.0-SNAPSHOT 4.0.0 diff --git a/escheduler-server/pom.xml b/escheduler-server/pom.xml index ac9a312734..7f149278c8 100644 --- a/escheduler-server/pom.xml +++ b/escheduler-server/pom.xml @@ -3,7 +3,7 @@ escheduler cn.analysys - 1.1.0 + 1.1.0-SNAPSHOT escheduler-server escheduler-server diff --git a/pom.xml b/pom.xml index 93ac6f8ee9..a97650aafb 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 cn.analysys escheduler - 1.1.0 + 1.1.0-SNAPSHOT pom escheduler http://maven.apache.org @@ -382,7 +382,7 @@ scm:git:https://github.com/analysys/EasyScheduler.git scm:git:https://github.com/analysys/EasyScheduler.git https://github.com/analysys/EasyScheduler.git - 1.1.0-preview + HEAD From e76853954300f22454e79419a75b55e342d55d27 Mon Sep 17 00:00:00 2001 From: easyscheduler Date: Tue, 16 Jul 2019 17:12:13 +0800 Subject: [PATCH 148/149] Update 1.1.0-release.md --- docs/zh_CN/1.1.0-release.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh_CN/1.1.0-release.md b/docs/zh_CN/1.1.0-release.md index 87084169d6..a77cfc9989 100644 --- a/docs/zh_CN/1.1.0-release.md +++ b/docs/zh_CN/1.1.0-release.md @@ -1,6 +1,6 @@ Easy Scheduler Release 1.1.0 === -Easy Scheduler 1.1.0是1.x系列中的第五个版本。 +Easy Scheduler 1.1.0是1.x系列中的第六个版本。 新特性： ===