From ce6c53eb126561ae2ed41f9f9f63ae8a9b222702 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Thu, 30 May 2019 11:53:52 +0800 Subject: [PATCH 01/68] update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 96cef561d7..23fbd6ee7d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Easy Scheduler +Easy Scheduler ============ [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) From 5e89fc6844aade8393058a52d73ff85f998a6daf Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Mon, 3 Jun 2019 16:58:57 +0800 Subject: [PATCH 02/68] if hdfs path is not exists return result --- .../main/java/cn/escheduler/api/service/ResourcesService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java index 291c59ea22..64b81d5ebc 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java @@ -643,6 +643,7 @@ public class ResourcesService extends BaseService { logger.error("{} is not exist", resourcePath); result.setCode(Status.HDFS_OPERATION_ERROR.getCode()); result.setMsg(String.format("%s is not exist", resourcePath)); + return result; } } catch (Exception e) { logger.error(e.getMessage(), e); From b1a0cf3e7fc366503ad555d41c96b5268d9458f1 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Tue, 4 Jun 2019 11:37:43 +0800 Subject: [PATCH 03/68] email address trim and shell error does not determine yarn status --- .../server/worker/task/AbstractCommandExecutor.java | 8 +------- .../cn/escheduler/server/worker/task/sql/SqlTask.java | 6 +++--- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/AbstractCommandExecutor.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/AbstractCommandExecutor.java index 9e617e65c0..10ef9f72ce 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/AbstractCommandExecutor.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/AbstractCommandExecutor.java @@ -213,7 +213,7 @@ public abstract class AbstractCommandExecutor { */ private int updateState(ProcessDao processDao, int exitStatusCode, int pid, int taskInstId) { //get yarn state by log - if (exitStatusCode != -1) { + if (exitStatusCode != 0) { TaskInstance taskInstance = processDao.findTaskInstanceById(taskInstId); logger.info("process id is {}", pid); @@ -548,10 +548,4 @@ public abstract class AbstractCommandExecutor { protected abstract boolean checkShowLog(String line); protected abstract boolean checkFindApp(String line); protected abstract void createCommandFileIfNotExists(String execCommand, String commandFile) throws IOException; - - - -// if(line.contains(taskAppId) || !line.contains("cn.escheduler.server.worker.log.TaskLogger")){ -// logs.add(line); -// } } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java index 3e85b55b91..bb57f60b6a 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java @@ -347,14 +347,14 @@ public class SqlTask extends AbstractTask { // receiving group list List receviersList = new ArrayList(); for(User user:users){ - receviersList.add(user.getEmail()); + receviersList.add(user.getEmail().trim()); } // custom receiver String receivers = sqlParameters.getReceivers(); if (StringUtils.isNotEmpty(receivers)){ String[] splits = receivers.split(Constants.COMMA); for (String receiver : splits){ - receviersList.add(receiver); + receviersList.add(receiver.trim()); } } @@ -365,7 +365,7 @@ public class SqlTask extends AbstractTask { if (StringUtils.isNotEmpty(receiversCc)){ String[] splits = receiversCc.split(Constants.COMMA); for (String receiverCc : splits){ - receviersCcList.add(receiverCc); + receviersCcList.add(receiverCc.trim()); } } From d7110c66151f820e0a4574684d9dfa6d3d410f14 Mon Sep 17 00:00:00 2001 From: lenboo Date: Tue, 4 Jun 2019 14:58:14 +0800 Subject: [PATCH 04/68] fix bug: failover process instance would change the state --- .../src/main/java/cn/escheduler/dao/ProcessDao.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java index 09e2149d88..d7dfa6225f 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java @@ -581,6 +581,8 @@ public class ProcessDao extends AbstractBaseDao { processInstance.setScheduleTime(command.getScheduleTime()); } processInstance.setHost(host); + + ExecutionStatus runStatus = ExecutionStatus.RUNNING_EXEUTION; int runTime = processInstance.getRunTimes(); switch (commandType){ case START_PROCESS: @@ -621,6 +623,7 @@ public class ProcessDao extends AbstractBaseDao { case RECOVER_TOLERANCE_FAULT_PROCESS: // recover tolerance fault process processInstance.setRecovery(Flag.YES); + runStatus = processInstance.getState(); break; case COMPLEMENT_DATA: // delete all the valid tasks when complement data @@ -652,7 +655,7 @@ public class ProcessDao extends AbstractBaseDao { default: break; } - processInstance.setState(ExecutionStatus.RUNNING_EXEUTION); + processInstance.setState(runStatus); return processInstance; } From 341333655386b5e21621a5176a6b1eac6be482a1 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Wed, 5 Jun 2019 16:32:36 +0800 Subject: [PATCH 05/68] kill task need kill sub pids --- .../cn/escheduler/common/utils/OSUtils.java | 2 +- .../escheduler/server/utils/ProcessUtils.java | 23 ++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/escheduler-common/src/main/java/cn/escheduler/common/utils/OSUtils.java b/escheduler-common/src/main/java/cn/escheduler/common/utils/OSUtils.java index d8dcc621a8..0cf06d3b02 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/utils/OSUtils.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/utils/OSUtils.java @@ -220,7 +220,7 @@ public class OSUtils { * @throws IOException */ public static String exeShell(String command) throws IOException { - return ShellExecutor.execCommand("groups"); + return ShellExecutor.execCommand(command); } /** diff --git a/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java b/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java index baf82de0df..5530e87e14 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java @@ -18,6 +18,7 @@ package cn.escheduler.server.utils; import cn.escheduler.common.Constants; import cn.escheduler.common.utils.CommonUtils; +import cn.escheduler.common.utils.OSUtils; import cn.escheduler.dao.model.TaskInstance; import cn.escheduler.server.rpc.LogClient; import org.apache.commons.io.FileUtils; @@ -33,6 +34,7 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; + /** * mainly used to get the start command line of a process */ @@ -139,6 +141,8 @@ public class ProcessUtils { {' ', '\t', '<', '>'}, {' ', '\t'}}; + private static Matcher matcher; + private static String createCommandLine(int verificationType, final String executablePath, final String[] cmd) { StringBuilder cmdbuf = new StringBuilder(80); @@ -256,7 +260,7 @@ public class ProcessUtils { return ; } - String cmd = String.format("sudo kill -9 %d", processId); + String cmd = String.format("sudo kill -9 %s", getPidsStr(processId)); logger.info("process id:{}, cmd:{}", processId, cmd); @@ -270,6 +274,23 @@ public class ProcessUtils { } } + /** + * get pids str + * @param processId + * @return + * @throws Exception + */ + private static String getPidsStr(int processId)throws Exception{ + StringBuilder sb = new StringBuilder(); + // pstree -p pid get sub pids + String pids = OSUtils.exeCmd("pstree -p " +processId+ ""); + Matcher mat = Pattern.compile("(\\d+)").matcher(pids); + while (mat.find()){ + sb.append(mat.group()+" "); + } + return sb.toString().trim(); + } + /** * find logs and kill yarn tasks * @param taskInstance From b997746e6257be4236e8451d17d8bf9e4737d990 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Wed, 5 Jun 2019 17:26:43 +0800 Subject: [PATCH 06/68] kill pid need kill sub process pids update --- .../src/main/java/cn/escheduler/server/utils/ProcessUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java b/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java index 5530e87e14..0c3aec602b 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java @@ -264,7 +264,7 @@ public class ProcessUtils { logger.info("process id:{}, cmd:{}", processId, cmd); - Runtime.getRuntime().exec(cmd); + OSUtils.exeCmd(cmd); // find log and kill yarn job killYarnJob(taskInstance); From 76662d31daa35c4c1111ac0faa4d622a0d0f2bfb Mon Sep 17 00:00:00 2001 From: xianhu Date: Thu, 6 Jun 2019 17:28:20 +0800 Subject: [PATCH 07/68] Solved the bugs that when editted file in resource center online, don't update the time and size of the file --- .../java/cn/escheduler/api/service/ResourcesService.java | 5 +++++ .../cn/escheduler/dao/mapper/ResourceMapperProvider.java | 1 + 2 files changed, 6 insertions(+) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java index 64b81d5ebc..f1c850a87a 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java @@ -569,6 +569,7 @@ public class ResourcesService extends BaseService { * @param resourceId * @return */ + @Transactional(value = "TransactionManager",rollbackFor = Exception.class) public Result updateResourceContent(int resourceId, String content) { Result result = new Result(); @@ -597,6 +598,10 @@ public class ResourcesService extends BaseService { } } + resource.setSize(content.getBytes().length); + resource.setUpdateTime(new Date()); + resourcesMapper.update(resource); + User user = userMapper.queryDetailsById(resource.getUserId()); String tenantCode = tenantMapper.queryById(user.getTenantId()).getTenantCode(); diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ResourceMapperProvider.java b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ResourceMapperProvider.java index 4122c7722c..4314b8f584 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ResourceMapperProvider.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ResourceMapperProvider.java @@ -118,6 +118,7 @@ public class ResourceMapperProvider { SET("`alias` = #{resource.alias}"); SET("`desc` = #{resource.desc}"); SET("`update_time` = #{resource.updateTime}"); + SET("`size` = #{resource.size}"); WHERE("`id` = #{resource.id}"); }}.toString(); } From d0ca95ead29da7755805997592152a91ef481faf Mon Sep 17 00:00:00 2001 From: lenboo Date: Sat, 8 Jun 2019 00:18:43 +0800 Subject: [PATCH 08/68] change the failover process. --- .../java/cn/escheduler/dao/ProcessDao.java | 2 +- .../escheduler/server/utils/ProcessUtils.java | 5 +- .../escheduler/server/zk/ZKMasterClient.java | 76 ++++++++++++------- 3 files changed, 52 insertions(+), 31 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java index d7dfa6225f..946bf3496b 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java @@ -58,6 +58,7 @@ public class ProcessDao extends AbstractBaseDao { private final int[] stateArray = new int[]{ExecutionStatus.SUBMITTED_SUCCESS.ordinal(), ExecutionStatus.RUNNING_EXEUTION.ordinal(), ExecutionStatus.READY_PAUSE.ordinal(), + ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal(), ExecutionStatus.READY_STOP.ordinal()}; @Autowired @@ -1569,7 +1570,6 @@ public class ProcessDao extends AbstractBaseDao { for (ProcessInstance processInstance:processInstanceList){ processNeedFailoverProcessInstances(processInstance); } - } @Transactional(value = "TransactionManager",rollbackFor = Exception.class) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java b/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java index baf82de0df..4980ef3b41 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java @@ -273,9 +273,8 @@ public class ProcessUtils { /** * find logs and kill yarn tasks * @param taskInstance - * @throws IOException */ - public static void killYarnJob(TaskInstance taskInstance) throws Exception { + public static void killYarnJob(TaskInstance taskInstance) { try { Thread.sleep(Constants.SLEEP_TIME_MILLIS); LogClient logClient = new LogClient(taskInstance.getHost(), Constants.RPC_PORT); @@ -295,7 +294,7 @@ public class ProcessUtils { } catch (Exception e) { logger.error("kill yarn job failed : " + e.getMessage(),e); - throw new RuntimeException("kill yarn job fail"); +// throw new RuntimeException("kill yarn job fail"); } } } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java index 85f9f6a7d2..fbabb2a82b 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java @@ -30,6 +30,7 @@ import cn.escheduler.dao.model.ProcessInstance; import cn.escheduler.dao.model.TaskInstance; import cn.escheduler.server.ResInfo; import cn.escheduler.server.utils.ProcessUtils; +import org.apache.commons.lang.StringUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.imps.CuratorFrameworkState; import org.apache.curator.framework.recipes.cache.PathChildrenCache; @@ -279,17 +280,9 @@ public class ZKMasterClient extends AbstractZKClient { for (int i = 0; i < Constants.ESCHEDULER_WARN_TIMES_FAILOVER;i++) { alertDao.sendServerStopedAlert(1, masterHost, "Master-Server"); } - - logger.info("start master failover ..."); - - List needFailoverProcessInstanceList = processDao.queryNeedFailoverProcessInstances(masterHost); - - //updateProcessInstance host is null and insert into command - for(ProcessInstance processInstance : needFailoverProcessInstanceList){ - processDao.processNeedFailoverProcessInstances(processInstance); + if(StringUtils.isNotEmpty(masterHost)){ + FailoverMaster(masterHost); } - - logger.info("master failover end"); }catch (Exception e){ logger.error("master failover failed : " + e.getMessage(),e); }finally { @@ -331,6 +324,8 @@ public class ZKMasterClient extends AbstractZKClient { } + + /** * monitor worker */ @@ -369,23 +364,9 @@ public class ZKMasterClient extends AbstractZKClient { alertDao.sendServerStopedAlert(1, workerHost, "Worker-Server"); } - logger.info("start worker failover ..."); - - - List needFailoverTaskInstanceList = processDao.queryNeedFailoverTaskInstances(workerHost); - for(TaskInstance taskInstance : needFailoverTaskInstanceList){ - ProcessInstance instance = processDao.findProcessInstanceDetailById(taskInstance.getProcessInstanceId()); - if(instance!=null){ - taskInstance.setProcessInstance(instance); - } - // only kill yarn job if exists , the local thread has exited - ProcessUtils.killYarnJob(taskInstance); - } - - //updateProcessInstance state value is NEED_FAULT_TOLERANCE - processDao.updateNeedFailoverTaskInstances(workerHost); - - logger.info("worker failover end"); + if(StringUtils.isNotEmpty(workerHost)){ + FailoverWorker(workerHost); + } }catch (Exception e){ logger.error("worker failover failed : " + e.getMessage(),e); } @@ -476,6 +457,46 @@ public class ZKMasterClient extends AbstractZKClient { } + /** + * failover worker tasks + * 1. kill yarn job if there are yarn jobs in tasks. + * 2. change task state from running to need failover. + * @param workerHost + */ + private void FailoverWorker(String workerHost){ + logger.info("start worker[{}] failover ...", workerHost); + + List needFailoverTaskInstanceList = processDao.queryNeedFailoverTaskInstances(workerHost); + for(TaskInstance taskInstance : needFailoverTaskInstanceList){ + ProcessInstance instance = processDao.findProcessInstanceDetailById(taskInstance.getProcessInstanceId()); + if(instance!=null){ + taskInstance.setProcessInstance(instance); + } + // only kill yarn job if exists , the local thread has exited + ProcessUtils.killYarnJob(taskInstance); + } + + //updateProcessInstance state value is NEED_FAULT_TOLERANCE + processDao.updateNeedFailoverTaskInstances(workerHost); + logger.info("end worker[{}] failover ...", workerHost); + } + + /** + * failover master tasks + * @param masterHost + */ + private void FailoverMaster(String masterHost) { + logger.info("start master failover ..."); + + List needFailoverProcessInstanceList = processDao.queryNeedFailoverProcessInstances(masterHost); + + //updateProcessInstance host is null and insert into command + for(ProcessInstance processInstance : needFailoverProcessInstanceList){ + processDao.processNeedFailoverProcessInstances(processInstance); + } + + logger.info("master failover end"); + } /** * get host ip @@ -488,6 +509,7 @@ public class ZKMasterClient extends AbstractZKClient { if(startIndex >= endIndex){ logger.error("parse ip error"); + return ""; } return path.substring(startIndex, endIndex); } From 963f28e31bac71f041a8d2ed1eeb6818bfd2e47a Mon Sep 17 00:00:00 2001 From: lenboo Date: Sat, 8 Jun 2019 00:32:50 +0800 Subject: [PATCH 09/68] update task state wait for yarn end. --- .../worker/task/AbstractCommandExecutor.java | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/AbstractCommandExecutor.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/AbstractCommandExecutor.java index 9e617e65c0..36c32c9713 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/AbstractCommandExecutor.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/AbstractCommandExecutor.java @@ -380,14 +380,22 @@ public abstract class AbstractCommandExecutor { boolean result = true; try { for (String appId : appIds) { - ExecutionStatus applicationStatus = HadoopUtils.getInstance().getApplicationStatus(appId); - logger.info("appId:{}, final state:{}",appId,applicationStatus.name()); - if (!applicationStatus.equals(ExecutionStatus.SUCCESS)) { - result = false; + while(true){ + ExecutionStatus applicationStatus = HadoopUtils.getInstance().getApplicationStatus(appId); + logger.info("appId:{}, final state:{}",appId,applicationStatus.name()); + if (applicationStatus.equals(ExecutionStatus.FAILURE) || + applicationStatus.equals(ExecutionStatus.KILL)) { + return false; + } + + if (applicationStatus.equals(ExecutionStatus.SUCCESS)){ + break; + } + Thread.sleep(Constants.SLEEP_TIME_MILLIS); } - } + } } catch (Exception e) { - logger.error(String.format("mapreduce applications: %s status failed : " + e.getMessage(), appIds.toString()),e); + logger.error(String.format("yarn applications: %s status failed : " + e.getMessage(), appIds.toString()),e); result = false; } return result; From ec600ffd018ab560b3054b6a9edc99b200d5bf4f Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Mon, 10 Jun 2019 11:20:43 +0800 Subject: [PATCH 10/68] Added checks on whether HDFS is configured to be true --- .../escheduler/api/service/TenantService.java | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java index 68fbc55348..c38f0f72ad 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java @@ -230,21 +230,29 @@ public class TenantService extends BaseService{ Tenant tenant = tenantMapper.queryById(id); - String tenantPath = HadoopUtils.getHdfsDataBasePath() + "/" + tenant.getTenantCode(); - - String resourcePath = HadoopUtils.getHdfsDir(tenant.getTenantCode()); - FileStatus[] fileStatus = HadoopUtils.getInstance().listFileStatus(resourcePath); - if (fileStatus.length > 0) { - putMsg(result, Status.HDFS_TERANT_RESOURCES_FILE_EXISTS); - return result; - } - fileStatus = HadoopUtils.getInstance().listFileStatus(HadoopUtils.getHdfsUdfDir(tenant.getTenantCode())); - if (fileStatus.length > 0) { - putMsg(result, Status.HDFS_TERANT_UDFS_FILE_EXISTS); + if (tenant == null){ + putMsg(result, Status.TENANT_NOT_EXIST); return result; } - HadoopUtils.getInstance().delete(tenantPath, true); + // if hdfs startup + if (PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ + String tenantPath = HadoopUtils.getHdfsDataBasePath() + "/" + tenant.getTenantCode(); + + String resourcePath = HadoopUtils.getHdfsDir(tenant.getTenantCode()); + FileStatus[] fileStatus = HadoopUtils.getInstance().listFileStatus(resourcePath); + if (fileStatus.length > 0) { + putMsg(result, Status.HDFS_TERANT_RESOURCES_FILE_EXISTS); + return result; + } + fileStatus = HadoopUtils.getInstance().listFileStatus(HadoopUtils.getHdfsUdfDir(tenant.getTenantCode())); + if (fileStatus.length > 0) { + putMsg(result, Status.HDFS_TERANT_UDFS_FILE_EXISTS); + return result; + } + + HadoopUtils.getInstance().delete(tenantPath, true); + } tenantMapper.deleteById(id); putMsg(result, Status.SUCCESS); From b7d0d4c952c42494a1e238d89b2d400a38acb9ae Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Mon, 10 Jun 2019 11:24:14 +0800 Subject: [PATCH 11/68] change Status.USER_NOT_EXIST to Status.TENANT_NOT_EXIST --- .../src/main/java/cn/escheduler/api/service/TenantService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java index c38f0f72ad..12624231a7 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java @@ -166,7 +166,7 @@ public class TenantService extends BaseService{ Tenant tenant = tenantMapper.queryById(id); if (tenant == null){ - putMsg(result, Status.USER_NOT_EXIST, id); + putMsg(result, Status.TENANT_NOT_EXIST); return result; } From 7f4f65db6e67e7865b99255662253ef35733d491 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Mon, 10 Jun 2019 16:15:50 +0800 Subject: [PATCH 12/68] Added the ability to sort by timing status --- .../escheduler/dao/mapper/ProcessDefinitionMapperProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ProcessDefinitionMapperProvider.java b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ProcessDefinitionMapperProvider.java index a619ee4ad5..48e8d0bdf0 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ProcessDefinitionMapperProvider.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ProcessDefinitionMapperProvider.java @@ -189,7 +189,7 @@ public class ProcessDefinitionMapperProvider { if(userId != null && 0 != Integer.parseInt(userId.toString())){ WHERE("td.user_id = #{userId}"); } - ORDER_BY(" td.update_time desc limit #{offset},#{pageSize} "); + ORDER_BY(" sc.schedule_release_state desc,td.update_time desc limit #{offset},#{pageSize} "); }}.toString(); } /** From b1fbd2170e1635788337ea05326afcb5e3d7ca91 Mon Sep 17 00:00:00 2001 From: lenboo Date: Wed, 12 Jun 2019 19:52:00 +0800 Subject: [PATCH 13/68] chang master/worker failover process. --- .../java/cn/escheduler/dao/ProcessDao.java | 17 ++- .../mapper/ProcessInstanceMapperProvider.java | 7 +- .../mapper/TaskInstanceMapperProvider.java | 7 +- .../dao/mapper/WorkerServerMapper.java | 17 +++ .../mapper/WorkerServerMapperProvider.java | 15 ++ .../cn/escheduler/dao/utils/DagHelper.java | 3 +- .../master/runner/MasterExecThread.java | 27 +++- .../escheduler/server/zk/ZKMasterClient.java | 134 ++++++++++++++---- 8 files changed, 189 insertions(+), 38 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java index 946bf3496b..0241016941 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java @@ -58,7 +58,7 @@ public class ProcessDao extends AbstractBaseDao { private final int[] stateArray = new int[]{ExecutionStatus.SUBMITTED_SUCCESS.ordinal(), ExecutionStatus.RUNNING_EXEUTION.ordinal(), ExecutionStatus.READY_PAUSE.ordinal(), - ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal(), +// ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal(), ExecutionStatus.READY_STOP.ordinal()}; @Autowired @@ -97,6 +97,9 @@ public class ProcessDao extends AbstractBaseDao { @Autowired private ErrorCommandMapper errorCommandMapper; + @Autowired + private WorkerServerMapper workerServerMapper; + /** * task queue impl */ @@ -122,6 +125,7 @@ public class ProcessDao extends AbstractBaseDao { udfFuncMapper = getMapper(UdfFuncMapper.class); resourceMapper = getMapper(ResourceMapper.class); workerGroupMapper = getMapper(WorkerGroupMapper.class); + workerServerMapper = getMapper(WorkerServerMapper.class); taskQueue = TaskQueueFactory.getTaskQueueInstance(); } @@ -1636,6 +1640,17 @@ public class ProcessDao extends AbstractBaseDao { return workerGroupMapper.queryById(workerGroupId); } + /** + * query worker server by host + * @param host + * @return + */ + public List queryWorkerServerByHost(String host){ + + return workerServerMapper.queryWorkerByHost(host); + + } + } diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ProcessInstanceMapperProvider.java b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ProcessInstanceMapperProvider.java index a6d09d431b..9ff57fc3c4 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ProcessInstanceMapperProvider.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ProcessInstanceMapperProvider.java @@ -402,7 +402,12 @@ public class ProcessInstanceMapperProvider { FROM(TABLE_NAME); - WHERE("`host` = #{host} and `state` in (" + strStates.toString() +")"); + Object host = parameter.get("host"); + if(host != null && StringUtils.isNotEmpty(host.toString())){ + + WHERE("`host` = #{host} "); + } + WHERE("`state` in (" + strStates.toString() +")"); ORDER_BY("`id` asc"); diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/TaskInstanceMapperProvider.java b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/TaskInstanceMapperProvider.java index 511b0970fe..ce1e69f197 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/TaskInstanceMapperProvider.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/TaskInstanceMapperProvider.java @@ -228,7 +228,12 @@ public class TaskInstanceMapperProvider { SELECT("*, UNIX_TIMESTAMP(end_time)-UNIX_TIMESTAMP(start_time) as duration"); FROM(TABLE_NAME); - WHERE("`host` = #{host} and `state` in (" + strStates.toString() +")"); + Object host = parameter.get("host"); + if(host != null && StringUtils.isNotEmpty(host.toString())){ + + WHERE("`host` = #{host} "); + } + WHERE("`state` in (" + strStates.toString() +")"); ORDER_BY("`id` asc"); } }.toString(); diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/WorkerServerMapper.java b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/WorkerServerMapper.java index b5ea3aa878..5e511a4edd 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/WorkerServerMapper.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/WorkerServerMapper.java @@ -42,6 +42,23 @@ public interface WorkerServerMapper { @SelectProvider(type = WorkerServerMapperProvider.class, method = "queryAllWorker") List queryAllWorker(); + /** + * query worker list + * + * @return + */ + @Results(value = { + @Result(property = "id", column = "id", javaType = Integer.class, jdbcType = JdbcType.INTEGER), + @Result(property = "host", column = "host", javaType = String.class, jdbcType = JdbcType.VARCHAR), + @Result(property = "port", column = "port", javaType = int.class, jdbcType = JdbcType.INTEGER), + @Result(property = "zkDirectory", column = "zk_directory", javaType = String.class, jdbcType = JdbcType.VARCHAR), + @Result(property = "resInfo", column = "res_info", javaType = String.class, jdbcType = JdbcType.VARCHAR), + @Result(property = "createTime", column = "create_time", javaType = Date.class, jdbcType = JdbcType.TIMESTAMP), + @Result(property = "lastHeartbeatTime", column = "last_heartbeat_time", javaType = Date.class, jdbcType = JdbcType.TIMESTAMP) + }) + @SelectProvider(type = WorkerServerMapperProvider.class, method = "queryWorkerByHost") + List queryWorkerByHost(@Param("host") String host); + /** * insert worker server * diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/WorkerServerMapperProvider.java b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/WorkerServerMapperProvider.java index bd5af7deda..15b330e077 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/WorkerServerMapperProvider.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/WorkerServerMapperProvider.java @@ -37,6 +37,21 @@ public class WorkerServerMapperProvider { }}.toString(); } + /** + * query worker list + * @return + */ + public String queryWorkerByHost(Map parameter) { + return new SQL() {{ + SELECT("*"); + + FROM(TABLE_NAME); + + WHERE("host = #{host}"); + }}.toString(); + } + + /** * insert worker server * @param parameter diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java b/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java index bc52e85062..a80a74d2d2 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java @@ -105,8 +105,7 @@ public class DagHelper { } for (TaskNode taskNode : tmpTaskNodeList) { - if ( !taskNode.isForbidden() - && null == findNodeByName(destTaskNodeList, taskNode.getName())) { + if (null == findNodeByName(destTaskNodeList, taskNode.getName())) { destTaskNodeList.add(taskNode); } } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java b/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java index f12726c3ab..359cff1ac2 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java @@ -411,6 +411,25 @@ public class MasterExecThread implements Runnable { return taskInstance; } + private Collection getStartVertex(String parentNodeName, DAG dag){ + Collection startVertex = null; + if(StringUtils.isNotEmpty(parentNodeName)){ + startVertex = dag.getSubsequentNodes(parentNodeName); + }else{ + startVertex = dag.getBeginNode(); + } + + for(String start : startVertex){ + TaskNode node = dag.getNode(start); + if(node.isForbidden()){ + + } + + } + + return startVertex; + } + /** * get post task instance by node * @@ -421,12 +440,8 @@ public class MasterExecThread implements Runnable { private List getPostTaskInstanceByNode(DAG dag, String parentNodeName){ List postTaskList = new ArrayList<>(); - Collection startVertex = null; - if(StringUtils.isNotEmpty(parentNodeName)){ - startVertex = dag.getSubsequentNodes(parentNodeName); - }else{ - startVertex = dag.getBeginNode(); - } + Collection startVertex = getStartVertex(parentNodeName, dag); + for (String nodeName : startVertex){ // encapsulation task instance diff --git a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java index fbabb2a82b..3066504a46 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java @@ -18,6 +18,7 @@ package cn.escheduler.server.zk; import cn.escheduler.common.Constants; import cn.escheduler.common.enums.ExecutionStatus; +import cn.escheduler.common.enums.ZKNodeType; import cn.escheduler.common.utils.CollectionUtils; import cn.escheduler.common.utils.DateUtils; import cn.escheduler.common.utils.OSUtils; @@ -28,11 +29,11 @@ import cn.escheduler.dao.ProcessDao; import cn.escheduler.dao.ServerDao; import cn.escheduler.dao.model.ProcessInstance; import cn.escheduler.dao.model.TaskInstance; +import cn.escheduler.dao.model.WorkerServer; import cn.escheduler.server.ResInfo; import cn.escheduler.server.utils.ProcessUtils; import org.apache.commons.lang.StringUtils; import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.imps.CuratorFrameworkState; import org.apache.curator.framework.recipes.cache.PathChildrenCache; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; @@ -135,7 +136,8 @@ public class ZKMasterClient extends AbstractZKClient { // check if fault tolerance is required，failure and tolerance if (getActiveMasterNum() == 1) { - processDao.masterStartupFaultTolerant(); +// processDao.masterStartupFaultTolerant(); + failoverMaster(null); } }catch (Exception e){ @@ -191,31 +193,20 @@ public class ZKMasterClient extends AbstractZKClient { Date now = new Date(); createTime = now ; try { + String osHost = OSUtils.getHost(); - // encapsulation master znnode - masterZNode = masterZNodeParentPath + "/" + OSUtils.getHost() + "_"; - List masterZNodeList = zkClient.getChildren().forPath(masterZNodeParentPath); - - if (CollectionUtils.isNotEmpty(masterZNodeList)){ - boolean flag = false; - for (String masterZNode : masterZNodeList){ - if (masterZNode.startsWith(OSUtils.getHost())){ - flag = true; - break; - } - } - - if (flag){ - logger.error("register failure , master already started on host : {}" , OSUtils.getHost()); - // exit system - System.exit(-1); - } + // zookeeper node exists, cannot start a new one. + if(checkZKNodeExists(osHost, ZKNodeType.MASTER)){ + logger.error("register failure , master already started on host : {}" , osHost); + // exit system + System.exit(-1); } // specify the format of stored data in ZK nodes String heartbeatZKInfo = getOsInfo(now); // create temporary sequence nodes for master znode - masterZNode = zkClient.create().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(masterZNode, heartbeatZKInfo.getBytes()); + masterZNode = zkClient.create().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath( + masterZNodeParentPath + "/" + OSUtils.getHost() + "_", heartbeatZKInfo.getBytes()); logger.info("register master node {} success" , masterZNode); @@ -239,6 +230,46 @@ public class ZKMasterClient extends AbstractZKClient { } + /** + * check the zookeeper node already exists + * @param host + * @param zkNodeType + * @return + * @throws Exception + */ + private boolean checkZKNodeExists(String host, ZKNodeType zkNodeType) throws Exception { + + String path = null; + switch (zkNodeType){ + case MASTER: + path = masterZNodeParentPath; + break; + case WORKER: + path = workerZNodeParentPath; + break; + case DEAD_SERVER: + path = deadServerZNodeParentPath; + break; + default: + break; + } + if(StringUtils.isEmpty(path)){ + logger.error("check zk node exists error, host:{}, zk node type:{}", host, zkNodeType.toString()); + return false; + } + + List masterZNodeList = null; + masterZNodeList = zkClient.getChildren().forPath(path); + if (CollectionUtils.isNotEmpty(masterZNodeList)){ + for (String masterZNode : masterZNodeList){ + if (masterZNode.startsWith(host)){ + return true; + } + } + } + return false; + } + /** * monitor master */ @@ -281,7 +312,7 @@ public class ZKMasterClient extends AbstractZKClient { alertDao.sendServerStopedAlert(1, masterHost, "Master-Server"); } if(StringUtils.isNotEmpty(masterHost)){ - FailoverMaster(masterHost); + failoverMaster(masterHost); } }catch (Exception e){ logger.error("master failover failed : " + e.getMessage(),e); @@ -365,7 +396,7 @@ public class ZKMasterClient extends AbstractZKClient { } if(StringUtils.isNotEmpty(workerHost)){ - FailoverWorker(workerHost); + failoverWorker(workerHost, true); } }catch (Exception e){ logger.error("worker failover failed : " + e.getMessage(),e); @@ -457,27 +488,76 @@ public class ZKMasterClient extends AbstractZKClient { } + /** + * task needs failover if task start before worker starts + * + * @param taskInstance + * @return + */ + private boolean checkTaskInstanceNeedFailover(TaskInstance taskInstance) throws Exception { + + boolean taskNeedFailover = true; + + // if the worker node exists in zookeeper, we must check the task starts after the worker + if(checkZKNodeExists(taskInstance.getHost(), ZKNodeType.WORKER)){ + //if task start after worker starts, there is no need to failover the task. + if(checkTaskAfterWorkerStart(taskInstance)){ + taskNeedFailover = false; + } + } + return taskNeedFailover; + } + + /** + * check task start after the worker server starts. + * @param taskInstance + * @return + */ + private boolean checkTaskAfterWorkerStart(TaskInstance taskInstance) { + Date workerServerStartDate = null; + List workerServers = processDao.queryWorkerServerByHost(taskInstance.getHost()); + if(workerServers.size() > 0){ + workerServerStartDate = workerServers.get(0).getCreateTime(); + } + + if(workerServerStartDate != null){ + return taskInstance.getStartTime().after(workerServerStartDate); + + }else{ + return false; + } + } + /** * failover worker tasks * 1. kill yarn job if there are yarn jobs in tasks. * 2. change task state from running to need failover. * @param workerHost */ - private void FailoverWorker(String workerHost){ + private void failoverWorker(String workerHost, boolean needCheckWorkerAlive) throws Exception { logger.info("start worker[{}] failover ...", workerHost); List needFailoverTaskInstanceList = processDao.queryNeedFailoverTaskInstances(workerHost); for(TaskInstance taskInstance : needFailoverTaskInstanceList){ + if(needCheckWorkerAlive){ + if(!checkTaskInstanceNeedFailover(taskInstance)){ + continue; + } + } + ProcessInstance instance = processDao.findProcessInstanceDetailById(taskInstance.getProcessInstanceId()); if(instance!=null){ taskInstance.setProcessInstance(instance); } // only kill yarn job if exists , the local thread has exited ProcessUtils.killYarnJob(taskInstance); + + taskInstance.setState(ExecutionStatus.NEED_FAULT_TOLERANCE); + processDao.saveTaskInstance(taskInstance); } - //updateProcessInstance state value is NEED_FAULT_TOLERANCE - processDao.updateNeedFailoverTaskInstances(workerHost); + //update task Instance state value is NEED_FAULT_TOLERANCE + // processDao.updateNeedFailoverTaskInstances(workerHost); logger.info("end worker[{}] failover ...", workerHost); } @@ -485,7 +565,7 @@ public class ZKMasterClient extends AbstractZKClient { * failover master tasks * @param masterHost */ - private void FailoverMaster(String masterHost) { + private void failoverMaster(String masterHost) { logger.info("start master failover ..."); List needFailoverProcessInstanceList = processDao.queryNeedFailoverProcessInstances(masterHost); From 5d837067c9e51ad05247e470eecb5d005b62d2b5 Mon Sep 17 00:00:00 2001 From: lenboo Date: Thu, 13 Jun 2019 10:16:40 +0800 Subject: [PATCH 14/68] update user doc --- docs/zh_CN/系统使用手册.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh_CN/系统使用手册.md b/docs/zh_CN/系统使用手册.md index 1e495c4ae5..fc0e999118 100644 --- a/docs/zh_CN/系统使用手册.md +++ b/docs/zh_CN/系统使用手册.md @@ -311,7 +311,7 @@ conf/common/hadoop.properties ## 安全中心（权限系统） - 安全中心是只有管理员账户才有权限的功能，有队列管理、租户管理、用户管理、告警组管理、worker分组、令牌管理等功能，还可以对资源、数据源、项目等授权 - - 管理员登录，默认用户名密码：admin/esheduler123 + - 管理员登录，默认用户名密码：admin/escheduler123 ### 创建队列 - 队列是在执行spark、mapreduce等程序，需要用到“队列”参数时使用的。 From efc2223c521d1f7f9647f228013a53473fce9aca Mon Sep 17 00:00:00 2001 From: lenboo Date: Thu, 13 Jun 2019 10:55:05 +0800 Subject: [PATCH 15/68] update user doc --- docs/zh_CN/前端部署文档.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh_CN/前端部署文档.md b/docs/zh_CN/前端部署文档.md index 96d1d48cf3..460134b858 100644 --- a/docs/zh_CN/前端部署文档.md +++ b/docs/zh_CN/前端部署文档.md @@ -5,9 +5,9 @@ ## 1、准备工作 #### 下载安装包 -目前最新安装包版本是1.0.2，下载地址： [码云下载](https://gitee.com/easyscheduler/EasyScheduler/attach_files/) +请下载最新版本的安装包，下载地址： [码云下载](https://gitee.com/easyscheduler/EasyScheduler/attach_files/) -下载 escheduler-ui-1.0.2.tar.gz 后，解压`tar -zxvf escheduler-ui-1.0.2.tar.gz ./`后，进入`escheduler-ui`目录 +下载 escheduler-ui-x.x.x.tar.gz 后，解压`tar -zxvf escheduler-ui-x.x.x.tar.gz ./`后，进入`escheduler-ui`目录 From dff5a26a2a260f72bc63eae8d735d11686a52339 Mon Sep 17 00:00:00 2001 From: lenboo Date: Thu, 13 Jun 2019 17:59:58 +0800 Subject: [PATCH 16/68] add zk node type --- .../cn/escheduler/common/enums/ZKNodeType.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 escheduler-common/src/main/java/cn/escheduler/common/enums/ZKNodeType.java diff --git a/escheduler-common/src/main/java/cn/escheduler/common/enums/ZKNodeType.java b/escheduler-common/src/main/java/cn/escheduler/common/enums/ZKNodeType.java new file mode 100644 index 0000000000..371231f727 --- /dev/null +++ b/escheduler-common/src/main/java/cn/escheduler/common/enums/ZKNodeType.java @@ -0,0 +1,15 @@ +package cn.escheduler.common.enums; + +/** + * zk node type + */ +public enum ZKNodeType { + + /** + * 0 do not send warning; + * 1 send if process success; + * 2 send if process failed; + * 3 send if process ending; + */ + MASTER, WORKER, DEAD_SERVER, TASK_QUEUE; +} From 597f518ed5d49219a80ffd5e859208f94c9865a7 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Fri, 14 Jun 2019 14:30:42 +0800 Subject: [PATCH 17/68] The SQL task fails if the mail delivery fails --- .../main/java/cn/escheduler/alert/utils/MailUtils.java | 1 + .../cn/escheduler/server/worker/task/sql/SqlTask.java | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/escheduler-alert/src/main/java/cn/escheduler/alert/utils/MailUtils.java b/escheduler-alert/src/main/java/cn/escheduler/alert/utils/MailUtils.java index d97d9d9d57..bd918fc645 100644 --- a/escheduler-alert/src/main/java/cn/escheduler/alert/utils/MailUtils.java +++ b/escheduler-alert/src/main/java/cn/escheduler/alert/utils/MailUtils.java @@ -160,6 +160,7 @@ public class MailUtils { return retMap; }catch (Exception e){ handleException(receivers, retMap, e); + return retMap; } } return retMap; diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java index bb57f60b6a..b5a9dd6f50 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java @@ -17,6 +17,7 @@ package cn.escheduler.server.worker.task.sql; import cn.escheduler.alert.utils.MailUtils; +import cn.escheduler.api.enums.Status; import cn.escheduler.common.Constants; import cn.escheduler.common.enums.DbType; import cn.escheduler.common.enums.ShowType; @@ -310,6 +311,7 @@ public class SqlTask extends AbstractTask { } } catch (Exception e) { logger.error(e.getMessage(),e); + throw new RuntimeException(e.getMessage()); } return connection; } @@ -371,9 +373,14 @@ public class SqlTask extends AbstractTask { String showTypeName = sqlParameters.getShowType().replace(Constants.COMMA,"").trim(); if(EnumUtils.isValidEnum(ShowType.class,showTypeName)){ - MailUtils.sendMails(receviersList,receviersCcList,title, content, ShowType.valueOf(showTypeName)); + Map mailResult = MailUtils.sendMails(receviersList, receviersCcList, title, content, ShowType.valueOf(showTypeName)); + Status status = (Status) mailResult.get(cn.escheduler.api.utils.Constants.STATUS); + if(status != Status.SUCCESS){ + throw new RuntimeException("send mail failed!"); + } }else{ logger.error("showType: {} is not valid " ,showTypeName); + throw new RuntimeException(String.format("showType: %s is not valid ",showTypeName)); } } From bb786b28c69f47e0d55142dba5c85c094567fd81 Mon Sep 17 00:00:00 2001 From: lenboo Date: Fri, 14 Jun 2019 15:23:51 +0800 Subject: [PATCH 18/68] change forbidden nodes run process. --- .../cn/escheduler/dao/utils/DagHelper.java | 112 ++++++++++++++++++ .../master/runner/MasterExecThread.java | 61 ++-------- .../server/master/MasterCommandTest.java | 40 +++++++ 3 files changed, 162 insertions(+), 51 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java b/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java index a80a74d2d2..58267f78db 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java @@ -18,16 +18,22 @@ package cn.escheduler.dao.utils; import cn.escheduler.common.enums.TaskDependType; +import cn.escheduler.common.graph.DAG; import cn.escheduler.common.model.TaskNode; import cn.escheduler.common.model.TaskNodeRelation; import cn.escheduler.common.process.ProcessDag; import cn.escheduler.common.utils.JSONUtils; import cn.escheduler.dao.model.ProcessData; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; /** * dag tools @@ -192,6 +198,24 @@ public class DagHelper { return processDag; } + /** + * parse the forbidden task nodes in process definition. + * @param processDefinitionJson + * @return + */ + public static Map getForbiddenTaskNodeMaps(String processDefinitionJson){ + Map forbidTaskNodeMap = new ConcurrentHashMap<>(); + ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); + + List taskNodeList = processData.getTasks(); + for(TaskNode node : taskNodeList){ + if(node.isForbidden()){ + forbidTaskNodeMap.putIfAbsent(node.getName(), node); + } + } + return forbidTaskNodeMap; + } + /** * find node by node name @@ -209,4 +233,92 @@ public class DagHelper { } return null; } + + + /** + * get start vertex in one dag + * it would find the post node if the start vertex is forbidden running + * @param parentNodeName the previous node + * @param dag + * @return + */ + public static Collection getStartVertex(String parentNodeName, DAG dag){ + Collection startVertexs = null; + if(StringUtils.isNotEmpty(parentNodeName)){ + startVertexs = dag.getSubsequentNodes(parentNodeName); + }else{ + startVertexs = dag.getBeginNode(); + } + + Collection tmpStartVertexs = new ArrayList<>(); + tmpStartVertexs.addAll(startVertexs); + + for(String start : tmpStartVertexs){ + TaskNode startNode = dag.getNode(start); + if(!startNode.isForbidden()){ + continue; + } + Collection postNodes = getStartVertex(start, dag); + + for(String post : postNodes){ + if(checkForbiddenPostCanSubmit(post, dag)){ + startVertexs.add(post); + } + } + startVertexs.remove(start); + } + + return startVertexs; + } + + /** + * + * @param postNodeName + * @param dag + * @return + */ + private static boolean checkForbiddenPostCanSubmit(String postNodeName, DAG dag){ + + TaskNode postNode = dag.getNode(postNodeName); + List dependList = postNode.getDepList(); + + for(String dependNodeName : dependList){ + TaskNode dependNode = dag.getNode(dependNodeName); + if(!dependNode.isForbidden()){ + return false; + } + } + return true; + } + + + + /*** + * generate dag graph + * @param processDag + * @return + */ + public static DAG buildDagGraph(ProcessDag processDag) { + + DAG dag = new DAG<>(); + + /** + * add vertex + */ + if (CollectionUtils.isNotEmpty(processDag.getNodes())){ + for (TaskNode node : processDag.getNodes()){ + dag.addNode(node.getName(),node); + } + } + + /** + * add edge + */ + if (CollectionUtils.isNotEmpty(processDag.getEdges())){ + for (TaskNodeRelation edge : processDag.getEdges()){ + dag.addEdge(edge.getStartNode(),edge.getEndNode()); + } + } + return dag; + } } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java b/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java index 359cff1ac2..d4997e48d5 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java @@ -79,6 +79,7 @@ public class MasterExecThread implements Runnable { private Map completeTaskList = new ConcurrentHashMap<>(); private Map readyToSubmitTaskList = new ConcurrentHashMap<>(); private Map dependFailedTask = new ConcurrentHashMap<>(); + private Map forbiddenTaskList = new ConcurrentHashMap<>(); private List recoverToleranceFaultTaskList = new ArrayList<>(); private AlertManager alertManager = new AlertManager(); @@ -269,6 +270,7 @@ public class MasterExecThread implements Runnable { private void buildFlowDag() throws Exception { recoverNodeIdList = getStartTaskInstanceList(processInstance.getCommandParam()); + forbiddenTaskList = DagHelper.getForbiddenTaskNodeMaps(processInstance.getProcessInstanceJson()); // generate process to get DAG info List recoveryNameList = getRecoveryNodeNameList(); List startNodeNameList = parseStartNodeName(processInstance.getCommandParam()); @@ -279,7 +281,8 @@ public class MasterExecThread implements Runnable { return; } // generate process dag - dag = buildDagGraph(processDag); + dag = DagHelper.buildDagGraph(processDag); + } private void initTaskQueue(){ @@ -411,24 +414,7 @@ public class MasterExecThread implements Runnable { return taskInstance; } - private Collection getStartVertex(String parentNodeName, DAG dag){ - Collection startVertex = null; - if(StringUtils.isNotEmpty(parentNodeName)){ - startVertex = dag.getSubsequentNodes(parentNodeName); - }else{ - startVertex = dag.getBeginNode(); - } - - for(String start : startVertex){ - TaskNode node = dag.getNode(start); - if(node.isForbidden()){ - } - - } - - return startVertex; - } /** * get post task instance by node @@ -440,10 +426,12 @@ public class MasterExecThread implements Runnable { private List getPostTaskInstanceByNode(DAG dag, String parentNodeName){ List postTaskList = new ArrayList<>(); - Collection startVertex = getStartVertex(parentNodeName, dag); + Collection startVertex = DagHelper.getStartVertex(parentNodeName, dag); + if(startVertex == null){ + return postTaskList; + } for (String nodeName : startVertex){ - // encapsulation task instance TaskInstance taskInstance = createTaskInstance(processInstance, nodeName , dag.getNode(nodeName),parentNodeName); @@ -532,8 +520,8 @@ public class MasterExecThread implements Runnable { List depsNameList = taskNode.getDepList(); for(String depsNode : depsNameList ){ - // dependencies must be all complete - if(!completeTaskList.containsKey(depsNode)){ + // dependencies must be fully completed or run prohibited + if(!completeTaskList.containsKey(depsNode) || !forbiddenTaskList.containsKey(depsNode)){ return DependResult.WAITING; } ExecutionStatus taskState = completeTaskList.get(depsNode).getState(); @@ -919,35 +907,6 @@ public class MasterExecThread implements Runnable { } } - /*** - * generate dag graph - * @param processDag - * @return - */ - public DAG buildDagGraph(ProcessDag processDag) { - - DAG dag = new DAG<>(); - - /** - * add vertex - */ - if (CollectionUtils.isNotEmpty(processDag.getNodes())){ - for (TaskNode node : processDag.getNodes()){ - dag.addNode(node.getName(),node); - } - } - - /** - * add edge - */ - if (CollectionUtils.isNotEmpty(processDag.getEdges())){ - for (TaskNodeRelation edge : processDag.getEdges()){ - dag.addEdge(edge.getStartNode(),edge.getEndNode()); - } - } - return dag; - } - /** * whether the retry interval is timed out * @param taskInstance diff --git a/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java b/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java index 3aed212618..cfa723105d 100644 --- a/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java +++ b/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java @@ -18,15 +18,27 @@ package cn.escheduler.server.master; import cn.escheduler.common.enums.CommandType; import cn.escheduler.common.enums.FailureStrategy; +import cn.escheduler.common.enums.TaskDependType; import cn.escheduler.common.enums.WarningType; +import cn.escheduler.common.graph.DAG; +import cn.escheduler.common.model.TaskNode; +import cn.escheduler.common.model.TaskNodeRelation; +import cn.escheduler.common.process.ProcessDag; import cn.escheduler.dao.datasource.ConnectionFactory; import cn.escheduler.dao.mapper.CommandMapper; +import cn.escheduler.dao.mapper.ProcessDefinitionMapper; import cn.escheduler.dao.model.Command; +import cn.escheduler.dao.model.ProcessDefinition; +import cn.escheduler.dao.utils.DagHelper; +import org.codehaus.jackson.SerializableString; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.Collection; + /** * master test */ @@ -36,9 +48,14 @@ public class MasterCommandTest { private CommandMapper commandMapper; + private ProcessDefinitionMapper processDefinitionMapper; + + @Before public void before(){ + commandMapper = ConnectionFactory.getSqlSession().getMapper(CommandMapper.class); + processDefinitionMapper = ConnectionFactory.getSqlSession().getMapper(ProcessDefinitionMapper.class); } @@ -104,4 +121,27 @@ public class MasterCommandTest { } + @Test + public void testDagHelper(){ + + ProcessDefinition processDefinition = processDefinitionMapper.queryByDefineId(19); + + try { + ProcessDag processDag = DagHelper.generateFlowDag(processDefinition.getProcessDefinitionJson(), + new ArrayList<>(), new ArrayList<>(), TaskDependType.TASK_POST); + + DAG dag = DagHelper.buildDagGraph(processDag); + Collection start = DagHelper.getStartVertex("1", dag); + + System.out.println(start.toString()); + + } catch (Exception e) { + e.printStackTrace(); + } + + } + + + + } From a7db4c3f2479c6af25d77247cddde7a69e9ec691 Mon Sep 17 00:00:00 2001 From: lenboo Date: Fri, 14 Jun 2019 16:33:17 +0800 Subject: [PATCH 19/68] change forbidden task process. --- .../escheduler/server/master/runner/MasterExecThread.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java b/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java index d4997e48d5..118922c822 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java @@ -520,8 +520,11 @@ public class MasterExecThread implements Runnable { List depsNameList = taskNode.getDepList(); for(String depsNode : depsNameList ){ - // dependencies must be fully completed or run prohibited - if(!completeTaskList.containsKey(depsNode) || !forbiddenTaskList.containsKey(depsNode)){ + if(forbiddenTaskList.containsKey(depsNode)){ + continue; + } + // dependencies must be fully completed + if(!completeTaskList.containsKey(depsNode)){ return DependResult.WAITING; } ExecutionStatus taskState = completeTaskList.get(depsNode).getState(); From 79b05a440573e401f22aafa3f233f1626248b02b Mon Sep 17 00:00:00 2001 From: lenboo Date: Fri, 14 Jun 2019 16:58:48 +0800 Subject: [PATCH 20/68] change forbidden task process --- .../main/java/cn/escheduler/dao/utils/DagHelper.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java b/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java index 58267f78db..c101d2791c 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java @@ -250,10 +250,10 @@ public class DagHelper { startVertexs = dag.getBeginNode(); } - Collection tmpStartVertexs = new ArrayList<>(); + List tmpStartVertexs = new ArrayList<>(); tmpStartVertexs.addAll(startVertexs); - for(String start : tmpStartVertexs){ + for(String start : startVertexs){ TaskNode startNode = dag.getNode(start); if(!startNode.isForbidden()){ continue; @@ -262,13 +262,13 @@ public class DagHelper { for(String post : postNodes){ if(checkForbiddenPostCanSubmit(post, dag)){ - startVertexs.add(post); + tmpStartVertexs.add(post); } } - startVertexs.remove(start); + tmpStartVertexs.remove(start); } - return startVertexs; + return tmpStartVertexs; } /** From c768445322b48c7dbb63a9fbe6e1ccd199e372f7 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Fri, 14 Jun 2019 17:27:32 +0800 Subject: [PATCH 21/68] The SQL task fails if the mail delivery fails --- .../java/cn/escheduler/server/worker/task/sql/SqlTask.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java index b5a9dd6f50..8b2f84c4a8 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java @@ -374,8 +374,7 @@ public class SqlTask extends AbstractTask { String showTypeName = sqlParameters.getShowType().replace(Constants.COMMA,"").trim(); if(EnumUtils.isValidEnum(ShowType.class,showTypeName)){ Map mailResult = MailUtils.sendMails(receviersList, receviersCcList, title, content, ShowType.valueOf(showTypeName)); - Status status = (Status) mailResult.get(cn.escheduler.api.utils.Constants.STATUS); - if(status != Status.SUCCESS){ + if(!(Boolean) mailResult.get(cn.escheduler.api.utils.Constants.STATUS)){ throw new RuntimeException("send mail failed!"); } }else{ From 0865aeca39f866f05856c011c88e05afacdc95eb Mon Sep 17 00:00:00 2001 From: lenboo Date: Fri, 14 Jun 2019 18:01:43 +0800 Subject: [PATCH 22/68] change forbidden task process. --- .../src/main/java/cn/escheduler/dao/utils/DagHelper.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java b/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java index c101d2791c..d811cf78bc 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java @@ -251,7 +251,9 @@ public class DagHelper { } List tmpStartVertexs = new ArrayList<>(); - tmpStartVertexs.addAll(startVertexs); + if(startVertexs!= null){ + tmpStartVertexs.addAll(startVertexs); + } for(String start : startVertexs){ TaskNode startNode = dag.getNode(start); From 4f09ff36ddea487bd54ada8b4b5167aca57bd191 Mon Sep 17 00:00:00 2001 From: lenboo Date: Fri, 14 Jun 2019 18:06:54 +0800 Subject: [PATCH 23/68] update --- .../cn/escheduler/server/master/runner/MasterExecThread.java | 3 +++ .../src/main/java/cn/escheduler/server/zk/ZKMasterClient.java | 1 + .../java/cn/escheduler/server/master/MasterCommandTest.java | 3 +++ 3 files changed, 7 insertions(+) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java b/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java index 118922c822..cb38c7a722 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java @@ -432,6 +432,9 @@ public class MasterExecThread implements Runnable { } for (String nodeName : startVertex){ + if(completeTaskList.containsKey(nodeName)){ + continue; + } // encapsulation task instance TaskInstance taskInstance = createTaskInstance(processInstance, nodeName , dag.getNode(nodeName),parentNodeName); diff --git a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java index 3066504a46..85c805e2fc 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/zk/ZKMasterClient.java @@ -136,6 +136,7 @@ public class ZKMasterClient extends AbstractZKClient { // check if fault tolerance is required，failure and tolerance if (getActiveMasterNum() == 1) { + failoverWorker(null, true); // processDao.masterStartupFaultTolerant(); failoverMaster(null); } diff --git a/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java b/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java index cfa723105d..114d9d3a84 100644 --- a/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java +++ b/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java @@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Collection; +import java.util.Map; /** * master test @@ -135,6 +136,8 @@ public class MasterCommandTest { System.out.println(start.toString()); + Map forbidden = DagHelper.getForbiddenTaskNodeMaps(processDefinition.getProcessDefinitionJson()); + System.out.println(forbidden); } catch (Exception e) { e.printStackTrace(); } From 1cac85f2effc928a853f5ab2f8d878d710837394 Mon Sep 17 00:00:00 2001 From: lenboo Date: Fri, 14 Jun 2019 18:31:16 +0800 Subject: [PATCH 24/68] update --- .../cn/escheduler/dao/utils/DagHelper.java | 18 +++++++++++------- .../server/master/runner/MasterExecThread.java | 5 +---- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java b/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java index d811cf78bc..bc509b1e99 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/utils/DagHelper.java @@ -24,15 +24,13 @@ import cn.escheduler.common.model.TaskNodeRelation; import cn.escheduler.common.process.ProcessDag; import cn.escheduler.common.utils.JSONUtils; import cn.escheduler.dao.model.ProcessData; +import cn.escheduler.dao.model.TaskInstance; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; /** @@ -240,9 +238,15 @@ public class DagHelper { * it would find the post node if the start vertex is forbidden running * @param parentNodeName the previous node * @param dag + * @param completeTaskList * @return */ - public static Collection getStartVertex(String parentNodeName, DAG dag){ + public static Collection getStartVertex(String parentNodeName, DAG dag, + Map completeTaskList){ + + if(completeTaskList == null){ + completeTaskList = new HashMap<>(); + } Collection startVertexs = null; if(StringUtils.isNotEmpty(parentNodeName)){ startVertexs = dag.getSubsequentNodes(parentNodeName); @@ -257,10 +261,10 @@ public class DagHelper { for(String start : startVertexs){ TaskNode startNode = dag.getNode(start); - if(!startNode.isForbidden()){ + if(!startNode.isForbidden() && !completeTaskList.containsKey(start)){ continue; } - Collection postNodes = getStartVertex(start, dag); + Collection postNodes = getStartVertex(start, dag, completeTaskList); for(String post : postNodes){ if(checkForbiddenPostCanSubmit(post, dag)){ diff --git a/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java b/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java index cb38c7a722..9ce08d691a 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/master/runner/MasterExecThread.java @@ -426,15 +426,12 @@ public class MasterExecThread implements Runnable { private List getPostTaskInstanceByNode(DAG dag, String parentNodeName){ List postTaskList = new ArrayList<>(); - Collection startVertex = DagHelper.getStartVertex(parentNodeName, dag); + Collection startVertex = DagHelper.getStartVertex(parentNodeName, dag, completeTaskList); if(startVertex == null){ return postTaskList; } for (String nodeName : startVertex){ - if(completeTaskList.containsKey(nodeName)){ - continue; - } // encapsulation task instance TaskInstance taskInstance = createTaskInstance(processInstance, nodeName , dag.getNode(nodeName),parentNodeName); From 1a09f56d5011b31280be1db34763ab9d549a8e51 Mon Sep 17 00:00:00 2001 From: lenboo Date: Mon, 17 Jun 2019 10:28:03 +0800 Subject: [PATCH 25/68] update --- .../src/main/resources/dao/data_source.properties | 6 +++--- .../java/cn/escheduler/server/master/MasterCommandTest.java | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/escheduler-dao/src/main/resources/dao/data_source.properties b/escheduler-dao/src/main/resources/dao/data_source.properties index cac3aa5e20..3c89dd1fd2 100644 --- a/escheduler-dao/src/main/resources/dao/data_source.properties +++ b/escheduler-dao/src/main/resources/dao/data_source.properties @@ -1,9 +1,9 @@ # base spring data source configuration spring.datasource.type=com.alibaba.druid.pool.DruidDataSource spring.datasource.driver-class-name=com.mysql.jdbc.Driver -spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/escheduler?characterEncoding=UTF-8 -spring.datasource.username=xx -spring.datasource.password=xx +spring.datasource.url=jdbc:mysql://192.168.220.188:3306/escheduler_new?characterEncoding=UTF-8 +spring.datasource.username=root +spring.datasource.password=root@123 # connection configuration spring.datasource.initialSize=5 diff --git a/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java b/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java index 114d9d3a84..9c71795179 100644 --- a/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java +++ b/escheduler-server/src/test/java/cn/escheduler/server/master/MasterCommandTest.java @@ -30,7 +30,6 @@ import cn.escheduler.dao.mapper.ProcessDefinitionMapper; import cn.escheduler.dao.model.Command; import cn.escheduler.dao.model.ProcessDefinition; import cn.escheduler.dao.utils.DagHelper; -import org.codehaus.jackson.SerializableString; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; @@ -132,7 +131,7 @@ public class MasterCommandTest { new ArrayList<>(), new ArrayList<>(), TaskDependType.TASK_POST); DAG dag = DagHelper.buildDagGraph(processDag); - Collection start = DagHelper.getStartVertex("1", dag); + Collection start = DagHelper.getStartVertex("1", dag, null); System.out.println(start.toString()); From 25ff540ff2bf2e1f3ea43f988ccff5613879759f Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Wed, 19 Jun 2019 16:47:32 +0800 Subject: [PATCH 26/68] Modified SQL task custom variable replacement rules --- .../server/worker/task/sql/SqlTask.java | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java index 8b2f84c4a8..4eb567d8c8 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java @@ -17,7 +17,6 @@ package cn.escheduler.server.worker.task.sql; import cn.escheduler.alert.utils.MailUtils; -import cn.escheduler.api.enums.Status; import cn.escheduler.common.Constants; import cn.escheduler.common.enums.DbType; import cn.escheduler.common.enums.ShowType; @@ -197,7 +196,7 @@ public class SqlTask extends AbstractTask { } // special characters need to be escaped, ${} needs to be escaped - String rgex = "'?\\$\\{(.*?)\\}'?"; + String rgex = "['\"]*\\$\\{(.*?)\\}['\"]*"; setSqlParamsMap(sql,rgex,sqlParamsMap,paramsMap); // replace the ${} of the SQL statement with the Placeholder @@ -328,6 +327,7 @@ public class SqlTask extends AbstractTask { ParameterUtils.setInParameter(key,stmt,prop.getType(),prop.getValue()); } } + logger.info("prepare statement replace sql:{}",stmt.toString()); return stmt; } @@ -417,19 +417,5 @@ public class SqlTask extends AbstractTask { logPrint.append(sqlParamsMap.get(i).getValue()+"("+sqlParamsMap.get(i).getType()+")"); } logger.info(logPrint.toString()); - - //direct print style - Pattern pattern = Pattern.compile(rgex); - Matcher m = pattern.matcher(content); - int index = 1; - StringBuffer sb = new StringBuffer("replaced sql , direct:"); - while (m.find()) { - - m.appendReplacement(sb, sqlParamsMap.get(index).getValue()); - - index ++; - } - m.appendTail(sb); - logger.info(sb.toString()); } } From f72d647fb2dcf90a1e302fd390bd5e97802c3cff Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Thu, 20 Jun 2019 18:35:28 +0800 Subject: [PATCH 27/68] add RESOURCE_FILE_EXIST --- escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java | 1 + 1 file changed, 1 insertion(+) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java b/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java index 6e25d91825..d165fb6c16 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java @@ -174,6 +174,7 @@ public enum Status { RESOURCE_SUFFIX_FORBID_CHANGE(20008, "resource suffix not allowed to be modified"), UDF_RESOURCE_SUFFIX_NOT_JAR(20009, "UDF resource suffix name must be jar"), HDFS_COPY_FAIL(20009, "hdfs copy {0} -> {1} fail"), + RESOURCE_FILE_EXIST(20010, "resource file {0} already exists in hdfs,please delete it or change name!"), From 4a69289c41871cdbb827122a2ff4d725e89d430c Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Thu, 20 Jun 2019 18:42:29 +0800 Subject: [PATCH 28/68] add verify resource file is exists in hdfs --- .../api/service/ResourcesService.java | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java index f1c850a87a..7b56d927ad 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java @@ -420,6 +420,41 @@ public class ResourcesService extends BaseService { return result; } + /** + * verify resource by name and type + * @param name + * @param type + * @param loginUser + * @return + */ + public Result verifyResourceName(String name, ResourceType type,User loginUser) { + Result result = new Result(); + putMsg(result, Status.SUCCESS); + Resource resource = resourcesMapper.queryResourceByNameAndType(name, type.ordinal()); + if (resource != null) { + logger.error("resource type:{} name:{} has exist, can't create again.", type, name); + putMsg(result, Status.RESOURCE_EXIST); + } else { + // query tenant + String tenantCode = tenantMapper.queryById(loginUser.getTenantId()).getTenantCode(); + + try { + String hdfsFilename = getHdfsFileName(type,tenantCode,name); + if(HadoopUtils.getInstance().exists(hdfsFilename)){ + logger.error("resource type:{} name:{} has exist in hdfs {}, can't create again.", type, name,hdfsFilename); + putMsg(result, Status.RESOURCE_FILE_EXIST,hdfsFilename); + } + + } catch (Exception e) { + logger.error(e.getMessage(),e); + putMsg(result,Status.HDFS_OPERATION_ERROR); + } + } + + + return result; + } + /** * verify resource by name and type * @@ -815,6 +850,23 @@ public class ResourcesService extends BaseService { return hdfsFileName; } + /** + * get hdfs file name + * + * @param resourceType + * @param tenantCode + * @param hdfsFileName + * @return + */ + private String getHdfsFileName(ResourceType resourceType, String tenantCode, String hdfsFileName) { + if (resourceType.equals(ResourceType.FILE)) { + hdfsFileName = HadoopUtils.getHdfsFilename(tenantCode, hdfsFileName); + } else if (resourceType.equals(ResourceType.UDF)) { + hdfsFileName = HadoopUtils.getHdfsUdfFilename(tenantCode, hdfsFileName); + } + return hdfsFileName; + } + /** * get authorized resource list * From 33be5ff9dd19493fbfa99dc50f65df8b4f418caf Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Thu, 20 Jun 2019 18:44:58 +0800 Subject: [PATCH 29/68] update verifyResourceName --- .../cn/escheduler/api/controller/ResourcesController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/controller/ResourcesController.java b/escheduler-api/src/main/java/cn/escheduler/api/controller/ResourcesController.java index 1574012d19..947eace7d9 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/controller/ResourcesController.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/controller/ResourcesController.java @@ -195,9 +195,9 @@ public class ResourcesController extends BaseController{ ) { try { logger.info("login user {}, verfiy resource alias: {},resource type: {}", - loginUser.getUserName(), alias); + loginUser.getUserName(), alias,type); - return resourceService.verifyResourceName(alias, type); + return resourceService.verifyResourceName(alias,type,loginUser); } catch (Exception e) { logger.error(VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR.getMsg(), e); return error(Status.VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR.getCode(), Status.VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR.getMsg()); From 5c1b3f0ed580bb6330232627b8015e5c69ceeff5 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Fri, 21 Jun 2019 10:58:02 +0800 Subject: [PATCH 30/68] qianfan task result add judge --- .../java/cn/escheduler/common/Constants.java | 5 ++ .../common/enums/TaskRecordStatus.java | 35 ++++++++++++++ .../cn/escheduler/common/enums/TaskType.java | 8 +++- .../java/cn/escheduler/dao/TaskRecordDao.java | 47 ++++++++++++++++++- .../worker/runner/TaskScheduleThread.java | 20 ++++++++ 5 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 escheduler-common/src/main/java/cn/escheduler/common/enums/TaskRecordStatus.java diff --git a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java index b12145973e..e5137ef8fe 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java @@ -252,6 +252,11 @@ public final class Constants { */ public static final String YYYY_MM_DD_HH_MM_SS = "yyyy-MM-dd HH:mm:ss"; + /** + * date format of yyyyMMdd + */ + public static final String YYYYMMDD = "yyyyMMdd"; + /** * date format of yyyyMMddHHmmss */ diff --git a/escheduler-common/src/main/java/cn/escheduler/common/enums/TaskRecordStatus.java b/escheduler-common/src/main/java/cn/escheduler/common/enums/TaskRecordStatus.java new file mode 100644 index 0000000000..a0fb8f1dfc --- /dev/null +++ b/escheduler-common/src/main/java/cn/escheduler/common/enums/TaskRecordStatus.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package cn.escheduler.common.enums; + + +/** + * task record status + * + */ +public enum TaskRecordStatus { + + /** + * status： + * 0 sucess + * 1 failure + * 2 exception + */ + SUCCESS,FAILURE,EXCEPTION + + +} diff --git a/escheduler-common/src/main/java/cn/escheduler/common/enums/TaskType.java b/escheduler-common/src/main/java/cn/escheduler/common/enums/TaskType.java index 64c2e951eb..1d589167e3 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/enums/TaskType.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/enums/TaskType.java @@ -30,5 +30,11 @@ public enum TaskType { * 6 PYTHON * 7 DEPENDENT */ - SHELL,SQL, SUB_PROCESS,PROCEDURE,MR,SPARK,PYTHON,DEPENDENT + SHELL,SQL, SUB_PROCESS,PROCEDURE,MR,SPARK,PYTHON,DEPENDENT; + + public static boolean typeIsNormalTask(String typeName) { + TaskType taskType = TaskType.valueOf(typeName); + return !(taskType == TaskType.SUB_PROCESS || taskType == TaskType.DEPENDENT); + } + } diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/TaskRecordDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/TaskRecordDao.java index 4378e06410..58d54589f6 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/TaskRecordDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/TaskRecordDao.java @@ -17,6 +17,8 @@ package cn.escheduler.dao; import cn.escheduler.common.Constants; +import cn.escheduler.common.enums.TaskRecordStatus; +import cn.escheduler.common.utils.CollectionUtils; import cn.escheduler.common.utils.DateUtils; import cn.escheduler.dao.model.TaskRecord; import org.apache.commons.configuration.Configuration; @@ -28,6 +30,7 @@ import org.slf4j.LoggerFactory; import java.sql.*; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -43,7 +46,7 @@ public class TaskRecordDao { /** - * 加载配置文件 + * load conf file */ private static Configuration conf; @@ -56,6 +59,14 @@ public class TaskRecordDao { } } + + /** + * get task record flag + * @return + */ + public static boolean getTaskRecordFlag(){ + return conf.getBoolean(Constants.TASK_RECORD_FLAG); + } /** * create connection * @return @@ -253,4 +264,38 @@ public class TaskRecordDao { } return recordList; } + + /** + * according to procname and procdate query task record + * @param procName + * @param procDate + * @return + */ + public static TaskRecordStatus getTaskRecordState(String procName,String procDate){ + String sql = String.format("SELECT * FROM eamp_hive_log_hd WHERE PROC_NAME='%s' and PROC_DATE='%s'" + ,procName,procDate); + List taskRecordList = getQueryResult(sql); + + // contains no record and sql exception + if (CollectionUtils.isEmpty(taskRecordList)){ + // exception + return TaskRecordStatus.EXCEPTION; + }else if (taskRecordList.size() > 1){ + return TaskRecordStatus.EXCEPTION; + }else { + TaskRecord taskRecord = taskRecordList.get(0); + if (taskRecord == null){ + return TaskRecordStatus.EXCEPTION; + } + Long targetRowCount = taskRecord.getTargetRowCount(); + if (targetRowCount <= 0){ + return TaskRecordStatus.FAILURE; + }else { + return TaskRecordStatus.SUCCESS; + } + + } + } + + } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java index f381b9ef66..60edad5606 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java @@ -19,15 +19,18 @@ package cn.escheduler.server.worker.runner; import cn.escheduler.common.Constants; import cn.escheduler.common.enums.ExecutionStatus; +import cn.escheduler.common.enums.TaskRecordStatus; import cn.escheduler.common.enums.TaskType; import cn.escheduler.common.model.TaskNode; import cn.escheduler.common.process.Property; import cn.escheduler.common.task.AbstractParameters; import cn.escheduler.common.task.TaskTimeoutParameter; import cn.escheduler.common.utils.CommonUtils; +import cn.escheduler.common.utils.DateUtils; import cn.escheduler.common.utils.HadoopUtils; import cn.escheduler.common.utils.TaskParametersUtils; import cn.escheduler.dao.ProcessDao; +import cn.escheduler.dao.TaskRecordDao; import cn.escheduler.dao.model.ProcessInstance; import cn.escheduler.dao.model.TaskInstance; import cn.escheduler.server.utils.LoggerUtils; @@ -199,6 +202,23 @@ public class TaskScheduleThread implements Callable { if (task.getExitStatusCode() == Constants.EXIT_CODE_SUCCESS){ status = ExecutionStatus.SUCCESS; + // task recor flat : if true , start up qianfan + if (TaskRecordDao.getTaskRecordFlag() + && TaskType.typeIsNormalTask(taskInstance.getTaskType())){ + Date scheduleTime = processInstance.getScheduleTime(); + if(scheduleTime == null){ + scheduleTime = processInstance.getStartTime(); + } + + // process exec time : yyyyMMdd format + String scheduleDate = DateUtils.format(scheduleTime, Constants.YYYYMMDD); + TaskRecordStatus taskRecordState = TaskRecordDao.getTaskRecordState(taskInstance.getName(), scheduleDate); + logger.info("task record status : {}",taskRecordState); + if (taskRecordState == TaskRecordStatus.FAILURE){ + status = ExecutionStatus.FAILURE; + } + } + }else if (task.getExitStatusCode() == Constants.EXIT_CODE_KILL){ status = ExecutionStatus.KILL; }else { From b27a080baae1abcbd3574c8264b9f27ef961cadb Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Fri, 21 Jun 2019 11:32:14 +0800 Subject: [PATCH 31/68] qianfan task result add judge update --- .../src/main/java/cn/escheduler/dao/TaskRecordDao.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/TaskRecordDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/TaskRecordDao.java index 58d54589f6..a4e317a1dd 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/TaskRecordDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/TaskRecordDao.java @@ -272,8 +272,8 @@ public class TaskRecordDao { * @return */ public static TaskRecordStatus getTaskRecordState(String procName,String procDate){ - String sql = String.format("SELECT * FROM eamp_hive_log_hd WHERE PROC_NAME='%s' and PROC_DATE='%s'" - ,procName,procDate); + String sql = String.format("SELECT * FROM eamp_hive_log_hd WHERE PROC_NAME='%s' and PROC_DATE like '%s'" + ,procName,procDate + "%"); List taskRecordList = getQueryResult(sql); // contains no record and sql exception @@ -297,5 +297,4 @@ public class TaskRecordDao { } } - } From 9c369ccde492025c550cc40ee2fbd47763248344 Mon Sep 17 00:00:00 2001 From: lenboo Date: Fri, 21 Jun 2019 16:42:00 +0800 Subject: [PATCH 32/68] move queue count and command count to statistics. --- .../home/pages/projects/pages/index/index.vue | 24 ------------------- .../src/js/conf/home/router/index.js | 8 +++++++ .../components/secondaryMenu/_source/menu.js | 18 +++++++++++++- .../src/js/module/i18n/locale/en_US.js | 8 ++++++- .../src/js/module/i18n/locale/zh_CN.js | 6 +++++ 5 files changed, 38 insertions(+), 26 deletions(-) diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue index 28f61f9e65..f695f3a678 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue @@ -33,30 +33,6 @@ -
-
-
-
-
- {{$t('Queue statistics')}} -
-
- - -
-
-
-
-
-
- {{$t('Command status statistics')}} -
-
- - -
-
-
diff --git a/escheduler-ui/src/js/conf/home/router/index.js b/escheduler-ui/src/js/conf/home/router/index.js index 97a7e81a10..c1aa86d6ec 100644 --- a/escheduler-ui/src/js/conf/home/router/index.js +++ b/escheduler-ui/src/js/conf/home/router/index.js @@ -439,6 +439,14 @@ const router = new Router({ meta: { title: `Mysql` } + }, + { + path: '/monitor/servers/statistics', + name: 'statistics', + component: resolve => require(['../pages/monitor/pages/servers/statistics'], resolve), + meta: { + title: `statistics` + } } ] } diff --git a/escheduler-ui/src/js/module/components/secondaryMenu/_source/menu.js b/escheduler-ui/src/js/module/components/secondaryMenu/_source/menu.js index 20182ce0b1..1c0aefbec7 100644 --- a/escheduler-ui/src/js/module/components/secondaryMenu/_source/menu.js +++ b/escheduler-ui/src/js/module/components/secondaryMenu/_source/menu.js @@ -193,7 +193,7 @@ let menu = { monitor: [ { name: `${i18n.$t('Servers manage')}`, - id: 0, + id: 1, path: '', isOpen: true, disabled: true, @@ -242,6 +242,22 @@ let menu = { disabled: true } ] + }, + { + name: `${i18n.$t('Statistics manage')}`, + id: 0, + path: '', + isOpen: true, + disabled: true, + icon: 'fa-server', + children: [ + { + name: "Statistics", + path: 'statistics', + id: 0, + disabled: true + } + ] } ] } diff --git a/escheduler-ui/src/js/module/i18n/locale/en_US.js b/escheduler-ui/src/js/module/i18n/locale/en_US.js index fbc621e18b..cf81d9c95c 100644 --- a/escheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/escheduler-ui/src/js/module/i18n/locale/en_US.js @@ -458,5 +458,11 @@ export default { 'Post Statement': 'Post Statement', 'Statement cannot be empty': 'Statement cannot be empty', 'Process Define Count': 'Process Define Count', - 'Process Instance Running Count': 'Process Instance Running Count' + 'Process Instance Running Count': 'Process Instance Running Count', + 'process number of waiting for running': 'process number of waiting for running', + 'failure command number': 'failure command number', + 'tasks number of waiting running': 'tasks number of waiting running', + 'task number of ready to kill': '待杀死任务数', + 'Statistics manage': 'Statistics manage', + 'statistics': 'statistics', } diff --git a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js index d77c55715a..78ef8323aa 100644 --- a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -460,4 +460,10 @@ export default { 'Process Define Count': '流程定义个数', 'Process Instance Running Count': '运行流程实例个数', 'Please select a queue': '请选择队列', + 'process number of waiting for running': '待执行的流程数', + 'failure command number': '执行失败的命令数', + 'tasks number of waiting running': '待运行任务数', + 'task number of ready to kill': '待杀死任务数', + 'Statistics manage': '统计管理', + 'statistics': '统计', } From ef4c36d256f7b1a4d2439d5e5cf94f84fcb4816b Mon Sep 17 00:00:00 2001 From: lenboo Date: Fri, 21 Jun 2019 16:48:35 +0800 Subject: [PATCH 33/68] update --- .../monitor/pages/servers/statistics.vue | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 escheduler-ui/src/js/conf/home/pages/monitor/pages/servers/statistics.vue diff --git a/escheduler-ui/src/js/conf/home/pages/monitor/pages/servers/statistics.vue b/escheduler-ui/src/js/conf/home/pages/monitor/pages/servers/statistics.vue new file mode 100644 index 0000000000..e6302fdd1f --- /dev/null +++ b/escheduler-ui/src/js/conf/home/pages/monitor/pages/servers/statistics.vue @@ -0,0 +1,115 @@ + + + + From 48389f1822d38ddb65e3a2bef51aa1676e745a01 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Mon, 24 Jun 2019 11:49:35 +0800 Subject: [PATCH 34/68] add verifyResourceName test --- .../controller/ResourcesControllerTest.java | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/escheduler-api/src/test/java/cn/escheduler/api/controller/ResourcesControllerTest.java b/escheduler-api/src/test/java/cn/escheduler/api/controller/ResourcesControllerTest.java index 0d4ac945dd..1dc7855186 100644 --- a/escheduler-api/src/test/java/cn/escheduler/api/controller/ResourcesControllerTest.java +++ b/escheduler-api/src/test/java/cn/escheduler/api/controller/ResourcesControllerTest.java @@ -34,6 +34,8 @@ import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.MvcResult; import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; import org.springframework.web.context.WebApplicationContext; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; @@ -43,7 +45,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @RunWith(SpringRunner.class) @SpringBootTest public class ResourcesControllerTest { - private static Logger logger = LoggerFactory.getLogger(QueueControllerTest.class); + private static Logger logger = LoggerFactory.getLogger(ResourcesControllerTest.class); private MockMvc mockMvc; @@ -71,4 +73,24 @@ public class ResourcesControllerTest { Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); } + + @Test + public void verifyResourceName() throws Exception { + + MultiValueMap paramsMap = new LinkedMultiValueMap<>(); + paramsMap.add("name","list_resources_1.sh"); + paramsMap.add("type","FILE"); + + MvcResult mvcResult = mockMvc.perform(get("/resources/verify-name") + .header("sessionId", "c24ed9d9-1c20-48a0-bd9c-5cfca14a4dcb") + .params(paramsMap)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) + .andReturn(); + + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + + Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); + logger.info(mvcResult.getResponse().getContentAsString()); + } } \ No newline at end of file From cc3c7e911e6d59381d06fbd502fa390b9d7d3688 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Mon, 24 Jun 2019 13:51:31 +0800 Subject: [PATCH 35/68] change soft_version to 1.0.4 --- sql/soft_version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/soft_version b/sql/soft_version index e6d5cb833c..a6a3a43c3a 100644 --- a/sql/soft_version +++ b/sql/soft_version @@ -1 +1 @@ -1.0.2 \ No newline at end of file +1.0.4 \ No newline at end of file From 3f59730282943bb8080c54c8f614357df752b46b Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Mon, 24 Jun 2019 14:05:05 +0800 Subject: [PATCH 36/68] Update the latest version number --- docs/zh_CN/前端部署文档.md | 4 ++-- docs/zh_CN/后端部署文档.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/zh_CN/前端部署文档.md b/docs/zh_CN/前端部署文档.md index 96d1d48cf3..c5bfa2f833 100644 --- a/docs/zh_CN/前端部署文档.md +++ b/docs/zh_CN/前端部署文档.md @@ -5,9 +5,9 @@ ## 1、准备工作 #### 下载安装包 -目前最新安装包版本是1.0.2，下载地址： [码云下载](https://gitee.com/easyscheduler/EasyScheduler/attach_files/) +目前最新安装包版本是1.0.4，下载地址： [码云下载](https://gitee.com/easyscheduler/EasyScheduler/attach_files/) -下载 escheduler-ui-1.0.2.tar.gz 后，解压`tar -zxvf escheduler-ui-1.0.2.tar.gz ./`后，进入`escheduler-ui`目录 +下载 escheduler-ui-1.0.4.tar.gz 后，解压`tar -zxvf escheduler-ui-1.0.4.tar.gz ./`后，进入`escheduler-ui`目录 diff --git a/docs/zh_CN/后端部署文档.md b/docs/zh_CN/后端部署文档.md index f2df8a6989..273f34dd22 100644 --- a/docs/zh_CN/后端部署文档.md +++ b/docs/zh_CN/后端部署文档.md @@ -4,7 +4,7 @@ ## 1、准备工作 -目前最新安装包版本是1.0.3，下载地址： [码云下载](https://gitee.com/easyscheduler/EasyScheduler/attach_files/) ，下载escheduler-backend-1.0.3.tar.gz(后端简称escheduler-backend),escheduler-ui-1.0.3.tar.gz(前端简称escheduler-ui) +目前最新安装包版本是1.0.4，下载地址： [码云下载](https://gitee.com/easyscheduler/EasyScheduler/attach_files/) ，下载escheduler-backend-1.0.4.tar.gz(后端简称escheduler-backend),escheduler-ui-1.0.4.tar.gz(前端简称escheduler-ui) #### 准备一: 基础软件安装(必装项请自行安装) @@ -149,7 +149,7 @@ install.sh : 一键部署脚本 ### 2.2 编译源码来部署 -将源码包release版本1.0.3下载后，解压进入根目录 +将源码包release版本1.0.4下载后，解压进入根目录 * 执行编译命令： From b49d218df46ce2d60847a46fadeaadd833a7132a Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Mon, 24 Jun 2019 14:15:03 +0800 Subject: [PATCH 37/68] user list display admin --- .../main/java/cn/escheduler/dao/mapper/UserMapperProvider.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/UserMapperProvider.java b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/UserMapperProvider.java index 4a7df12a0f..363cd38f6a 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/UserMapperProvider.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/UserMapperProvider.java @@ -187,7 +187,6 @@ public class UserMapperProvider { return new SQL() {{ SELECT("count(0)"); FROM(TABLE_NAME); - WHERE("user_type = 1"); Object searchVal = parameter.get("searchVal"); if(searchVal != null && StringUtils.isNotEmpty(searchVal.toString())){ WHERE( " user_name like concat('%', #{searchVal}, '%') "); @@ -209,7 +208,6 @@ public class UserMapperProvider { FROM(TABLE_NAME + " u "); LEFT_OUTER_JOIN("t_escheduler_tenant t on u.tenant_id = t.id"); LEFT_OUTER_JOIN("t_escheduler_queue q on t.queue_id = q.id"); - WHERE("u.user_type = 1"); Object searchVal = parameter.get("searchVal"); if(searchVal != null && StringUtils.isNotEmpty(searchVal.toString())){ WHERE( " u.user_name like concat('%', #{searchVal}, '%') "); From 7a33a5f1901db9e8a4b0dc0dd00958f4afd67cd4 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Mon, 24 Jun 2019 14:58:44 +0800 Subject: [PATCH 38/68] datasource project resource for admin operate --- .../cn/escheduler/api/service/DataSourceService.java | 3 ++- .../java/cn/escheduler/api/service/ProjectService.java | 9 --------- .../java/cn/escheduler/api/service/ResourcesService.java | 3 ++- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java index 36b9b45861..5e6fac13e0 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java @@ -21,6 +21,7 @@ import cn.escheduler.api.utils.Constants; import cn.escheduler.api.utils.PageInfo; import cn.escheduler.api.utils.Result; import cn.escheduler.common.enums.DbType; +import cn.escheduler.common.enums.UserType; import cn.escheduler.common.job.db.*; import cn.escheduler.dao.mapper.DataSourceMapper; import cn.escheduler.dao.mapper.DatasourceUserMapper; @@ -537,7 +538,7 @@ public class DataSourceService extends BaseService{ putMsg(result, Status.RESOURCE_NOT_EXIST); return result; } - if(loginUser.getId() != dataSource.getUserId()){ + if(loginUser.getId() != dataSource.getUserId() || loginUser.getUserType() != UserType.ADMIN_USER){ putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ProjectService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ProjectService.java index c90d4da779..30d8f827aa 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ProjectService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ProjectService.java @@ -76,15 +76,6 @@ public class ProjectService extends BaseService{ return descCheck; } - /** - * only general users can create projects. administrators have no corresponding tenants and can only view - * 管理员没有对应的租户,只能查看,只有普通用户才可以创建项目 - */ - if (!userService.isGeneral(loginUser)) { - putMsg(result, Status.USER_NO_OPERATION_PERM); - return result; - } - Project project = projectMapper.queryByName(name); if (project != null) { putMsg(result, Status.PROJECT_ALREADY_EXISTS, name); diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java index 291c59ea22..0dea2e00d3 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java @@ -21,6 +21,7 @@ import cn.escheduler.api.utils.Constants; import cn.escheduler.api.utils.PageInfo; import cn.escheduler.api.utils.Result; import cn.escheduler.common.enums.ResourceType; +import cn.escheduler.common.enums.UserType; import cn.escheduler.common.utils.FileUtils; import cn.escheduler.common.utils.HadoopUtils; import cn.escheduler.common.utils.PropertyUtils; @@ -399,7 +400,7 @@ public class ResourcesService extends BaseService { putMsg(result, Status.RESOURCE_NOT_EXIST); return result; } - if (loginUser.getId() != resource.getUserId()) { + if (loginUser.getId() != resource.getUserId() || loginUser.getUserType() != UserType.ADMIN_USER) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } From ab05b7edcaeab716c6bbff471874af341b3406b5 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Mon, 24 Jun 2019 15:20:11 +0800 Subject: [PATCH 39/68] datasource and resource for admin operation --- .../main/java/cn/escheduler/api/service/DataSourceService.java | 2 +- .../main/java/cn/escheduler/api/service/ResourcesService.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java index 5e6fac13e0..9081a436cf 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java @@ -538,7 +538,7 @@ public class DataSourceService extends BaseService{ putMsg(result, Status.RESOURCE_NOT_EXIST); return result; } - if(loginUser.getId() != dataSource.getUserId() || loginUser.getUserType() != UserType.ADMIN_USER){ + if(loginUser.getId() != dataSource.getUserId() && loginUser.getUserType() != UserType.ADMIN_USER){ putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java index 0dea2e00d3..b59e43387b 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java @@ -400,7 +400,7 @@ public class ResourcesService extends BaseService { putMsg(result, Status.RESOURCE_NOT_EXIST); return result; } - if (loginUser.getId() != resource.getUserId() || loginUser.getUserType() != UserType.ADMIN_USER) { + if (loginUser.getId() != resource.getUserId() && loginUser.getUserType() != UserType.ADMIN_USER) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } From 15ce0733df2002984c8ac79fcbb480f653994cfd Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Mon, 24 Jun 2019 18:11:53 +0800 Subject: [PATCH 40/68] admin support delete process define --- .../cn/escheduler/api/service/ProcessDefinitionService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessDefinitionService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessDefinitionService.java index 45ff487f5e..76c7173f02 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessDefinitionService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ProcessDefinitionService.java @@ -24,6 +24,7 @@ import cn.escheduler.api.utils.PageInfo; import cn.escheduler.common.enums.Flag; import cn.escheduler.common.enums.ReleaseState; import cn.escheduler.common.enums.TaskType; +import cn.escheduler.common.enums.UserType; import cn.escheduler.common.graph.DAG; import cn.escheduler.common.model.TaskNode; import cn.escheduler.common.model.TaskNodeRelation; @@ -365,7 +366,7 @@ public class ProcessDefinitionService extends BaseDAGService { } // Determine if the login user is the owner of the process definition - if (loginUser.getId() != processDefinition.getUserId()) { + if (loginUser.getId() != processDefinition.getUserId() && loginUser.getUserType() != UserType.ADMIN_USER) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } From 947f3ade4c82ea4a6d984e8c2a878746428bd366 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Mon, 24 Jun 2019 18:18:11 +0800 Subject: [PATCH 41/68] =?UTF-8?q?admin=20support=20project=EF=BC=8Cdatasou?= =?UTF-8?q?rce=20and=20resource=20operation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../js/conf/home/pages/dag/_source/config.js | 2 +- .../js/conf/home/pages/dag/_source/dag.vue | 22 +++++++++---------- .../pages/dag/_source/formModel/formModel.vue | 2 +- .../pages/dag/_source/plugIn/jsPlumbHandle.js | 2 +- .../conf/home/pages/dag/_source/udp/udp.vue | 2 +- .../datasource/pages/list/_source/list.vue | 4 +--- .../pages/datasource/pages/list/index.vue | 2 +- .../definition/pages/list/_source/list.vue | 15 ++++++------- .../definition/pages/list/_source/start.vue | 2 +- .../pages/definition/pages/list/index.vue | 2 +- .../instance/pages/list/_source/list.vue | 8 +------ .../projects/pages/list/_source/list.vue | 6 ++--- .../home/pages/projects/pages/list/index.vue | 4 ++-- .../pages/file/pages/create/index.vue | 2 +- .../pages/file/pages/list/_source/list.vue | 14 +++++------- .../resource/pages/file/pages/list/index.vue | 6 ++--- .../pages/udf/pages/function/_source/list.vue | 8 +++---- .../pages/udf/pages/function/index.vue | 2 +- .../pages/udf/pages/resource/_source/list.vue | 9 +++----- .../pages/udf/pages/resource/index.vue | 2 +- .../pages/users/_source/createUser.vue | 11 +++++++--- .../security/pages/users/_source/list.vue | 10 ++++++++- .../src/js/module/mixin/disabledState.js | 4 ++-- 23 files changed, 67 insertions(+), 74 deletions(-) diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/config.js b/escheduler-ui/src/js/conf/home/pages/dag/_source/config.js index 740846890c..9c1065870f 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/config.js +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/config.js @@ -26,7 +26,7 @@ import Permissions from '@/module/permissions' * @desc tooltip */ const toolOper = (dagThis) => { - let disabled = Permissions.getAuth() === false ? false : !dagThis.$store.state.dag.isDetails + let disabled =!dagThis.$store.state.dag.isDetails// Permissions.getAuth() === false ? false : !dagThis.$store.state.dag.isDetails return [ { code: 'pointer', diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue b/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue index 37c4119676..2c0422c701 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue @@ -8,7 +8,7 @@ :id="v" v-for="(item,v) in tasksTypeList" @mousedown="_getDagId(v)"> -
+
@@ -68,10 +68,9 @@ type="primary" size="xsmall" :loading="spinnerLoading" - v-ps="['GENERAL_USER']" @click="_saveChart" icon="fa fa-save" - :disabled="isDetails"> + > {{spinnerLoading ? 'Loading...' : $t('Save')}}
@@ -205,9 +204,9 @@ * @param item */ _getDagId (v) { - if (this.isDetails) { - return - } + // if (this.isDetails) { + // return + // } this.dagBarId = v }, /** @@ -239,11 +238,12 @@ }) }, _operationClass (item) { - if (item.disable) { - return this.toolOperCode === item.code ? 'active' : '' - } else { - return 'disable' - } + return this.toolOperCode === item.code ? 'active' : '' + // if (item.disable) { + // return this.toolOperCode === item.code ? 'active' : '' + // } else { + // return 'disable' + // } }, /** * Storage interface diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index 7f36417a3a..863a44abf5 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -165,7 +165,7 @@
{{$t('Cancel')}} - {{spinnerLoading ? 'Loading...' : $t('Confirm add')}} + {{spinnerLoading ? 'Loading...' : $t('Confirm add')}}
diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js b/escheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js index e2c0c677a9..440deb48fb 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js @@ -71,7 +71,7 @@ JSP.prototype.init = function ({ dag, instance }) { this.setConfig({ isDrag: !store.state.dag.isDetails, isAttachment: false, - isNewNodes: Permissions.getAuth() === false ? false : !store.state.dag.isDetails, + isNewNodes: !store.state.dag.isDetails,//Permissions.getAuth() === false ? false : !store.state.dag.isDetails, isDblclick: true, isContextmenu: true, isClick: false diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue b/escheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue index 8e11dca356..f7de034db5 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue @@ -62,7 +62,7 @@
{{$t('Cancel')}} - {{$t('Add')}} + {{$t('Add')}} diff --git a/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/list.vue b/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/list.vue index b32aeeb6bc..a08051bc35 100644 --- a/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/list.vue +++ b/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/list.vue @@ -56,7 +56,6 @@ shape="circle" size="xsmall" data-toggle="tooltip" - v-ps="['GENERAL_USER']" :title="$t('Edit')" icon="iconfont icon-bianjixiugai" @click="_edit(item)"> @@ -77,8 +76,7 @@ size="xsmall" icon="iconfont icon-shanchu" data-toggle="tooltip" - :title="$t('delete')" - v-ps="['GENERAL_USER']"> + :title="$t('delete')"> diff --git a/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/index.vue b/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/index.vue index de94e0ee21..8d5012700f 100644 --- a/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/index.vue +++ b/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/index.vue @@ -3,7 +3,7 @@ diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue index aa2be8ef8e..f9e8dba231 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue @@ -58,12 +58,12 @@ - - - - - - - + + + + + + + :title="$t('delete')"> diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue index ddb6b0a156..c2e3c33728 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue @@ -137,7 +137,7 @@
{{$t('Cancel')}} - {{spinnerLoading ? 'Loading...' : $t('Start')}} + {{spinnerLoading ? 'Loading...' : $t('Start')}}
diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/index.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/index.vue index bc63896c17..bf8612dd98 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/index.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/index.vue @@ -3,7 +3,7 @@ diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue index f98383c558..2bc1cad066 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue @@ -73,7 +73,6 @@ data-toggle="tooltip" :title="$t('Edit')" @click="_reEdit(item)" - v-ps="['GENERAL_USER']" icon="iconfont icon-bianjixiugai" :disabled="item.state !== 'SUCCESS' && item.state !== 'PAUSE' && item.state !== 'FAILURE' && item.state !== 'STOP'"> + :title="$t('delete')"> diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/list/_source/list.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/list/_source/list.vue index 087b50032a..21a58fc1ea 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/list/_source/list.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/list/_source/list.vue @@ -63,8 +63,7 @@ data-toggle="tooltip" :title="$t('Edit')" @click="_edit(item)" - icon="iconfont icon-bianjixiugai" - v-ps="['GENERAL_USER']"> + icon="iconfont icon-bianjixiugai"> + icon="iconfont icon-shanchu"> diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/list/index.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/list/index.vue index 6031b590e0..7b2f555192 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/list/index.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/list/index.vue @@ -3,7 +3,7 @@ @@ -113,4 +113,4 @@ }, components: { mListConstruction, mSpin, mConditions, mList, mCreateProject, mNoData } } - \ No newline at end of file + diff --git a/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue b/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue index d08ed7f8dd..bf3ebe044d 100644 --- a/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue +++ b/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue @@ -49,7 +49,7 @@ diff --git a/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/_source/list.vue b/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/_source/list.vue index 40c03ff7a0..89acc74dd6 100644 --- a/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/_source/list.vue +++ b/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/_source/list.vue @@ -51,8 +51,7 @@ :title="$t('Edit')" :disabled="_rtDisb(item)" @click="_edit(item,$index)" - icon="iconfont icon-bianjixiugai" - v-ps="['GENERAL_USER']"> + icon="iconfont icon-bianjixiugai"> + @click="_rename(item,$index)"> + icon="iconfont icon-download"> + :title="$t('delete')"> @@ -210,4 +206,4 @@ }, components: { } } - \ No newline at end of file + diff --git a/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/index.vue b/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/index.vue index 6555d7a6d4..7434772899 100644 --- a/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/index.vue +++ b/escheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/index.vue @@ -4,8 +4,8 @@ @@ -98,4 +98,4 @@ }, components: { mListConstruction, mConditions, mList, mSpin, mNoData } } - \ No newline at end of file + diff --git a/escheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/list.vue b/escheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/list.vue index 8d96ede359..ad0c510c87 100644 --- a/escheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/list.vue +++ b/escheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/list.vue @@ -1,4 +1,4 @@ - diff --git a/escheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/resource/index.vue b/escheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/resource/index.vue index ed9a81a705..228501dbed 100644 --- a/escheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/resource/index.vue +++ b/escheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/resource/index.vue @@ -3,7 +3,7 @@ diff --git a/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue b/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue index 9d3ab042d8..b02db7848e 100644 --- a/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue +++ b/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue @@ -98,7 +98,9 @@ userName: '', userPassword: '', tenantId: {}, - queueName: {}, + queueName: { + id:'' + }, email: '', phone: '', tenantList: [], @@ -197,6 +199,7 @@ }, _submit () { this.$refs['popup'].spinnerLoading = true + console.log(this.tenantId.id) let param = { userName: this.userName, userPassword: this.userPassword, @@ -205,9 +208,11 @@ queue: this.queueName.code, phone: this.phone } + if (this.item) { param.id = this.item.id } + this.store.dispatch(`security/${this.item ? 'updateUser' : 'createUser'}`, param).then(res => { setTimeout(() => { this.$refs['popup'].spinnerLoading = false @@ -232,7 +237,7 @@ this.phone = this.item.phone this.tenantId = _.find(this.tenantList, ['id', this.item.tenantId]) this.$nextTick(() => { - this.queueName = _.find(this.queueList, ['code', this.item.queue]) + this.queueName = _.find(this.queueList, ['code', this.item.queue])||{id:''} }) } }) @@ -243,7 +248,7 @@ this.email = this.item.email this.phone = this.item.phone this.tenantId.id = this.item.tenantId - this.queueName = { queue: this.item.queue } + this.queueName = { queue: this.item.queue} } } }, diff --git a/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/list.vue b/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/list.vue index 125a3dfd93..e97886a61b 100644 --- a/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/list.vue +++ b/escheduler-ui/src/js/conf/home/pages/security/pages/users/_source/list.vue @@ -9,6 +9,9 @@ {{$t('User Name')}} + + 用户类型 + {{$t('Tenant')}} @@ -21,6 +24,7 @@ {{$t('Phone')}} + {{$t('Create Time')}} @@ -40,6 +44,9 @@ {{item.userName || '-'}} + + {{item.userType === 'GENERAL_USER' ? `${$t('Ordinary users')}` : `${$t('Administrator')}`}} + {{item.tenantName || '-'}} {{item.queue || '-'}} @@ -62,7 +69,7 @@ {{$t('UDF Function')}} @@ -84,6 +91,7 @@ size="xsmall" data-toggle="tooltip" :title="$t('delete')" + :disabled="item.userType === 'ADMIN_USER'" icon="iconfont icon-shanchu"> diff --git a/escheduler-ui/src/js/module/mixin/disabledState.js b/escheduler-ui/src/js/module/mixin/disabledState.js index 7c0b1f8e92..4b814a1908 100644 --- a/escheduler-ui/src/js/module/mixin/disabledState.js +++ b/escheduler-ui/src/js/module/mixin/disabledState.js @@ -28,11 +28,11 @@ export default { } }, created () { - this.isDetails = Permissions.getAuth() ? this.store.state.dag.isDetails : true + this.isDetails =this.store.state.dag.isDetails// Permissions.getAuth() ? this.store.state.dag.isDetails : true }, computed: { _isDetails () { - return this.isDetails ? 'icon-disabled' : '' + return ''// this.isDetails ? 'icon-disabled' : '' } } } From 9135b1420e30aedaf3ba49ecff6ef53d8868784b Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Mon, 24 Jun 2019 19:33:17 +0800 Subject: [PATCH 42/68] add resource upload s3 --- .../api/service/ResourcesService.java | 40 ++++----- .../escheduler/api/service/TenantService.java | 4 +- .../api/service/UdfFuncService.java | 12 +-- .../escheduler/api/service/UsersService.java | 6 +- .../java/cn/escheduler/common/Constants.java | 21 ++++- .../common/enums/ResUploadType.java | 29 +++++++ .../escheduler/common/utils/HadoopUtils.java | 87 +++++++++++-------- .../common/utils/PropertyUtils.java | 14 ++- .../main/resources/common/common.properties | 4 +- .../resources/common/hadoop/hadoop.properties | 12 ++- install.sh | 24 +++-- 11 files changed, 168 insertions(+), 85 deletions(-) create mode 100644 escheduler-common/src/main/java/cn/escheduler/common/enums/ResUploadType.java diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java index b59e43387b..4f2c62271c 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java @@ -86,8 +86,8 @@ public class ResourcesService extends BaseService { Result result = new Result(); // if hdfs not startup - if (!PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ - logger.error("hdfs startup state: {}", PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)); + if (!PropertyUtils.getResUploadStartupState()){ + logger.error("resource upload startup state: {}", PropertyUtils.getResUploadStartupState()); putMsg(result, Status.HDFS_NOT_STARTUP); return result; } @@ -185,9 +185,9 @@ public class ResourcesService extends BaseService { ResourceType type) { Result result = new Result(); - // if hdfs not startup - if (!PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ - logger.error("hdfs startup state: {}", PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)); + // if resource upload startup + if (!PropertyUtils.getResUploadStartupState()){ + logger.error("resource upload startup state: {}", PropertyUtils.getResUploadStartupState()); putMsg(result, Status.HDFS_NOT_STARTUP); return result; } @@ -386,9 +386,9 @@ public class ResourcesService extends BaseService { public Result delete(User loginUser, int resourceId) throws Exception { Result result = new Result(); - // if hdfs not startup - if (!PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ - logger.error("hdfs startup state: {}", PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)); + // if resource upload startup + if (!PropertyUtils.getResUploadStartupState()){ + logger.error("resource upload startup state: {}", PropertyUtils.getResUploadStartupState()); putMsg(result, Status.HDFS_NOT_STARTUP); return result; } @@ -449,9 +449,9 @@ public class ResourcesService extends BaseService { public Result readResource(int resourceId, int skipLineNum, int limit) { Result result = new Result(); - // if hdfs not startup - if (!PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ - logger.error("hdfs startup state: {}", PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)); + // if resource upload startup + if (!PropertyUtils.getResUploadStartupState()){ + logger.error("resource upload startup state: {}", PropertyUtils.getResUploadStartupState()); putMsg(result, Status.HDFS_NOT_STARTUP); return result; } @@ -510,9 +510,9 @@ public class ResourcesService extends BaseService { @Transactional(value = "TransactionManager",rollbackFor = Exception.class) public Result onlineCreateResource(User loginUser, ResourceType type, String fileName, String fileSuffix, String desc, String content) { Result result = new Result(); - // if hdfs not startup - if (!PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ - logger.error("hdfs startup state: {}", PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)); + // if resource upload startup + if (!PropertyUtils.getResUploadStartupState()){ + logger.error("resource upload startup state: {}", PropertyUtils.getResUploadStartupState()); putMsg(result, Status.HDFS_NOT_STARTUP); return result; } @@ -573,9 +573,9 @@ public class ResourcesService extends BaseService { public Result updateResourceContent(int resourceId, String content) { Result result = new Result(); - // if hdfs not startup - if (!PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ - logger.error("hdfs startup state: {}", PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)); + // if resource upload startup + if (!PropertyUtils.getResUploadStartupState()){ + logger.error("resource upload startup state: {}", PropertyUtils.getResUploadStartupState()); putMsg(result, Status.HDFS_NOT_STARTUP); return result; } @@ -663,9 +663,9 @@ public class ResourcesService extends BaseService { * @return */ public org.springframework.core.io.Resource downloadResource(int resourceId) throws Exception { - // if hdfs not startup - if (!PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ - logger.error("hdfs startup state: {}", PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)); + // if resource upload startup + if (!PropertyUtils.getResUploadStartupState()){ + logger.error("resource upload startup state: {}", PropertyUtils.getResUploadStartupState()); throw new RuntimeException("hdfs not startup"); } diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java index 68fbc55348..7538071e3a 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java @@ -96,7 +96,7 @@ public class TenantService extends BaseService{ tenantMapper.insert(tenant); // if hdfs startup - if (PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ + if (PropertyUtils.getResUploadStartupState()){ String resourcePath = HadoopUtils.getHdfsDataBasePath() + "/" + tenantCode + "/resources"; String udfsPath = HadoopUtils.getHdfsUdfDir(tenantCode); /** @@ -178,7 +178,7 @@ public class TenantService extends BaseService{ Tenant newTenant = tenantMapper.queryByTenantCode(tenantCode); if (newTenant == null){ // if hdfs startup - if (PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ + if (PropertyUtils.getResUploadStartupState()){ String resourcePath = HadoopUtils.getHdfsDataBasePath() + "/" + tenantCode + "/resources"; String udfsPath = HadoopUtils.getHdfsUdfDir(tenantCode); //init hdfs resource diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/UdfFuncService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/UdfFuncService.java index 52e605f711..8ca399d17f 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/UdfFuncService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/UdfFuncService.java @@ -80,9 +80,9 @@ public class UdfFuncService extends BaseService{ int resourceId) { Result result = new Result(); - // if hdfs not startup - if (!PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ - logger.error("hdfs startup state: {}", PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)); + // if resource upload startup + if (!PropertyUtils.getResUploadStartupState()){ + logger.error("resource upload startup state: {}", PropertyUtils.getResUploadStartupState()); putMsg(result, Status.HDFS_NOT_STARTUP); return result; } @@ -167,9 +167,9 @@ public class UdfFuncService extends BaseService{ // verify udfFunc is exist UdfFunc udf = udfFuncMapper.queryUdfById(udfFuncId); - // if hdfs not startup - if (!PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ - logger.error("hdfs startup state: {}", PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)); + // if resource upload startup + if (!PropertyUtils.getResUploadStartupState()){ + logger.error("resource upload startup state: {}", PropertyUtils.getResUploadStartupState()); putMsg(result, Status.HDFS_NOT_STARTUP); return result; } diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java index 5db8662c07..c0dab22b69 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/UsersService.java @@ -125,7 +125,7 @@ public class UsersService extends BaseService { Tenant tenant = tenantMapper.queryById(tenantId); // if hdfs startup - if (PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ + if (PropertyUtils.getResUploadStartupState()){ String userPath = HadoopUtils.getHdfsDataBasePath() + "/" + tenant.getTenantCode() + "/home/" + user.getId(); HadoopUtils.getInstance().mkdir(userPath); @@ -245,7 +245,7 @@ public class UsersService extends BaseService { Tenant newTenant = tenantMapper.queryById(tenantId); if (newTenant != null) { // if hdfs startup - if (PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ + if (PropertyUtils.getResUploadStartupState()){ String newTenantCode = newTenant.getTenantCode(); String oldResourcePath = HadoopUtils.getHdfsDataBasePath() + "/" + oldTenant.getTenantCode() + "/resources"; String oldUdfsPath = HadoopUtils.getHdfsUdfDir(oldTenant.getTenantCode()); @@ -308,7 +308,7 @@ public class UsersService extends BaseService { User user = userMapper.queryTenantCodeByUserId(id); - if (PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ + if (PropertyUtils.getResUploadStartupState()){ String userPath = HadoopUtils.getHdfsDataBasePath() + "/" + user.getTenantCode() + "/home/" + id; HadoopUtils.getInstance().delete(userPath, true); diff --git a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java index ddd27170db..30d2e26ded 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/Constants.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/Constants.java @@ -60,6 +60,23 @@ public final class Constants { */ public static final String FS_DEFAULTFS = "fs.defaultFS"; + + /** + * fs s3a endpoint + */ + public static final String FS_S3A_ENDPOINT = "fs.s3a.endpoint"; + + /** + * fs s3a access key + */ + public static final String FS_S3A_ACCESS_KEY = "fs.s3a.access.key"; + + /** + * fs s3a secret key + */ + public static final String FS_S3A_SECRET_KEY = "fs.s3a.secret.key"; + + /** * yarn.resourcemanager.ha.rm.idsfs.defaultFS */ @@ -123,9 +140,9 @@ public final class Constants { public static final String DEVELOPMENT_STATE = "development.state"; /** - * hdfs.startup.state + * res.upload.startup.type */ - public static final String HDFS_STARTUP_STATE = "hdfs.startup.state"; + public static final String RES_UPLOAD_STARTUP_TYPE = "res.upload.startup.type"; /** * zookeeper quorum diff --git a/escheduler-common/src/main/java/cn/escheduler/common/enums/ResUploadType.java b/escheduler-common/src/main/java/cn/escheduler/common/enums/ResUploadType.java new file mode 100644 index 0000000000..65d8be8f92 --- /dev/null +++ b/escheduler-common/src/main/java/cn/escheduler/common/enums/ResUploadType.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package cn.escheduler.common.enums; + +/** + * data base types + */ +public enum ResUploadType { + /** + * 0 hdfs + * 1 s3 + * 2 none + */ + HDFS,S3,NONE +} diff --git a/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java b/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java index 85716bec81..6f3e5e2198 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java @@ -18,6 +18,7 @@ package cn.escheduler.common.utils; import cn.escheduler.common.Constants; import cn.escheduler.common.enums.ExecutionStatus; +import cn.escheduler.common.enums.ResUploadType; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONException; import com.alibaba.fastjson.JSONObject; @@ -40,6 +41,7 @@ import java.util.stream.Stream; import static cn.escheduler.common.Constants.*; import static cn.escheduler.common.utils.PropertyUtils.*; +import static cn.escheduler.common.utils.PropertyUtils.getString; /** * hadoop utils @@ -94,48 +96,61 @@ public class HadoopUtils implements Closeable { try { configuration = new Configuration(); - if (getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE)){ - System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, - getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)); - configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION,"kerberos"); - UserGroupInformation.setConfiguration(configuration); - UserGroupInformation.loginUserFromKeytab(getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), - getString(Constants.LOGIN_USER_KEY_TAB_PATH)); - } + String resUploadStartupType = PropertyUtils.getString(Constants.RES_UPLOAD_STARTUP_TYPE); + ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); + + if (resUploadType == ResUploadType.HDFS){ + if (getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE)){ + System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, + getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)); + configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION,"kerberos"); + UserGroupInformation.setConfiguration(configuration); + UserGroupInformation.loginUserFromKeytab(getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), + getString(Constants.LOGIN_USER_KEY_TAB_PATH)); + } - String defaultFS = configuration.get(FS_DEFAULTFS); - //first get key from core-site.xml hdfs-site.xml ,if null ,then try to get from properties file - // the default is the local file system - if(defaultFS.startsWith("file")){ - String defaultFSProp = getString(FS_DEFAULTFS); - if(StringUtils.isNotBlank(defaultFSProp)){ - Map fsRelatedProps = getPrefixedProperties("fs."); - configuration.set(FS_DEFAULTFS,defaultFSProp); - fsRelatedProps.entrySet().stream().forEach(entry -> configuration.set(entry.getKey(), entry.getValue())); + String defaultFS = configuration.get(FS_DEFAULTFS); + //first get key from core-site.xml hdfs-site.xml ,if null ,then try to get from properties file + // the default is the local file system + if(defaultFS.startsWith("file")){ + String defaultFSProp = getString(FS_DEFAULTFS); + if(StringUtils.isNotBlank(defaultFSProp)){ + Map fsRelatedProps = getPrefixedProperties("fs."); + configuration.set(FS_DEFAULTFS,defaultFSProp); + fsRelatedProps.entrySet().stream().forEach(entry -> configuration.set(entry.getKey(), entry.getValue())); + }else{ + logger.error("property:{} can not to be empty, please set!"); + throw new RuntimeException("property:{} can not to be empty, please set!"); + } }else{ - logger.error("property:{} can not to be empty, please set!"); - throw new RuntimeException("property:{} can not to be empty, please set!"); + logger.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", FS_DEFAULTFS, defaultFS); } - }else{ - logger.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", FS_DEFAULTFS, defaultFS); - } - if (fs == null) { - if(StringUtils.isNotEmpty(hdfsUser)){ - //UserGroupInformation ugi = UserGroupInformation.createProxyUser(hdfsUser,UserGroupInformation.getLoginUser()); - UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser); - ugi.doAs(new PrivilegedExceptionAction() { - @Override - public Boolean run() throws Exception { - fs = FileSystem.get(configuration); - return true; - } - }); - }else{ - logger.warn("hdfs.root.user is not set value!"); - fs = FileSystem.get(configuration); + if (fs == null) { + if(StringUtils.isNotEmpty(hdfsUser)){ + //UserGroupInformation ugi = UserGroupInformation.createProxyUser(hdfsUser,UserGroupInformation.getLoginUser()); + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser); + ugi.doAs(new PrivilegedExceptionAction() { + @Override + public Boolean run() throws Exception { + fs = FileSystem.get(configuration); + return true; + } + }); + }else{ + logger.warn("hdfs.root.user is not set value!"); + fs = FileSystem.get(configuration); + } } + }else if (resUploadType == ResUploadType.S3){ + configuration.set(FS_DEFAULTFS,getString(FS_DEFAULTFS)); + configuration.set(FS_S3A_ENDPOINT,getString(FS_S3A_ENDPOINT)); + configuration.set(FS_S3A_ACCESS_KEY,getString(FS_S3A_ACCESS_KEY)); + configuration.set(FS_S3A_SECRET_KEY,getString(FS_S3A_SECRET_KEY)); + fs = FileSystem.get(configuration); } + + String rmHaIds = getString(YARN_RESOURCEMANAGER_HA_RM_IDS); String appAddress = getString(Constants.YARN_APPLICATION_STATUS_ADDRESS); if (!StringUtils.isEmpty(rmHaIds)) { diff --git a/escheduler-common/src/main/java/cn/escheduler/common/utils/PropertyUtils.java b/escheduler-common/src/main/java/cn/escheduler/common/utils/PropertyUtils.java index f5dab12618..475cbfb72e 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/utils/PropertyUtils.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/utils/PropertyUtils.java @@ -16,6 +16,8 @@ */ package cn.escheduler.common.utils; +import cn.escheduler.common.Constants; +import cn.escheduler.common.enums.ResUploadType; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,11 +67,15 @@ public class PropertyUtils { } } -/* - public static PropertyUtils getInstance(){ - return propertyUtils; + /** + * judge whether resource upload startup + * @return + */ + public static Boolean getResUploadStartupState(){ + String resUploadStartupType = PropertyUtils.getString(Constants.RES_UPLOAD_STARTUP_TYPE); + ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); + return resUploadType == ResUploadType.HDFS || resUploadType == ResUploadType.S3; } -*/ /** * get property value diff --git a/escheduler-common/src/main/resources/common/common.properties b/escheduler-common/src/main/resources/common/common.properties index 357ba2cbc6..874d3d0f0b 100644 --- a/escheduler-common/src/main/resources/common/common.properties +++ b/escheduler-common/src/main/resources/common/common.properties @@ -16,8 +16,8 @@ hdfs.root.user=hdfs # data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/escheduler" is recommended data.store2hdfs.basepath=/escheduler -# whether hdfs starts -hdfs.startup.state=false +# resource upload startup type : HDFS,S3,NONE +res.upload.startup.type=NONE # whether kerberos starts hadoop.security.authentication.startup.state=false diff --git a/escheduler-common/src/main/resources/common/hadoop/hadoop.properties b/escheduler-common/src/main/resources/common/hadoop/hadoop.properties index f210ae7533..81452a83a2 100644 --- a/escheduler-common/src/main/resources/common/hadoop/hadoop.properties +++ b/escheduler-common/src/main/resources/common/hadoop/hadoop.properties @@ -1,6 +1,16 @@ -# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml to the conf directory +# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml +# to the conf directory，support s3，for example : s3a://escheduler fs.defaultFS=hdfs://mycluster:8020 +# s3 need，s3 endpoint +fs.s3a.endpoint=http://192.168.199.91:9010 + +# s3 need，s3 access key +fs.s3a.access.key=A3DXS30FO22544RE + +# s3 need，s3 secret key +fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK + #resourcemanager ha note this need ips , this empty if single yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx diff --git a/install.sh b/install.sh index 2245409986..dc26cc59ef 100644 --- a/install.sh +++ b/install.sh @@ -110,14 +110,17 @@ xlsFilePath="/tmp/xls" #是否启动监控自启动脚本 monitorServerState="false" -# hadoop 配置 -# 是否启动hdfs,如果启动则为true,需要配置以下hadoop相关参数; -# 不启动设置为false,如果为false,以下配置不需要修改 -# 特别注意：如果启动hdfs，需要自行创建hdfs根路径，也就是install.sh中的 hdfsPath -hdfsStartupSate="false" +# 资源中心上传选择存储方式：HDFS,S3,NONE +resUploadStartupType="NONE" -# namenode地址，支持HA,需要将core-site.xml和hdfs-site.xml放到conf目录下 -namenodeFs="hdfs://mycluster:8020" +# 如果resUploadStartupType为HDFS，defaultFS写namenode地址，支持HA,需要将core-site.xml和hdfs-site.xml放到conf目录下 +# 如果是S3，则写S3地址，比如说：s3a://escheduler，注意，一定要创建根目录/escheduler +defaultFS="hdfs://mycluster:8020" + +# 如果配置了S3，则需要有以下配置 +s3Endpoint="http://192.168.199.91:9010" +s3AccessKey="A3DXS30FO22544RE" +s3SecretKey="OloCLq3n+8+sdPHUhJ21XrSxTC+JK" # resourcemanager HA配置，如果是单resourcemanager,这里为空即可 yarnHaIps="192.168.xx.xx,192.168.xx.xx" @@ -273,7 +276,10 @@ sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.us sed -i ${txt} "s#org.quartz.dataSource.myDs.password.*#org.quartz.dataSource.myDs.password=${mysqlPassword}#g" conf/quartz.properties -sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${namenodeFs}#g" conf/common/hadoop/hadoop.properties +sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g" conf/common/hadoop/hadoop.properties +sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g" conf/common/hadoop/hadoop.properties +sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g" conf/common/hadoop/hadoop.properties +sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g" conf/common/hadoop/hadoop.properties sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common/hadoop/hadoop.properties sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common/hadoop/hadoop.properties @@ -283,7 +289,7 @@ sed -i ${txt} "s#data.download.basedir.path.*#data.download.basedir.path=${downl sed -i ${txt} "s#process.exec.basepath.*#process.exec.basepath=${execPath}#g" conf/common/common.properties sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common/common.properties sed -i ${txt} "s#data.store2hdfs.basepath.*#data.store2hdfs.basepath=${hdfsPath}#g" conf/common/common.properties -sed -i ${txt} "s#hdfs.startup.state.*#hdfs.startup.state=${hdfsStartupSate}#g" conf/common/common.properties +sed -i ${txt} "s#res.upload.startup.type.*#res.upload.startup.type=${resUploadStartupType}#g" conf/common/common.properties sed -i ${txt} "s#escheduler.env.path.*#escheduler.env.path=${shellEnvPath}#g" conf/common/common.properties sed -i ${txt} "s#resource.view.suffixs.*#resource.view.suffixs=${resSuffixs}#g" conf/common/common.properties sed -i ${txt} "s#development.state.*#development.state=${devState}#g" conf/common/common.properties From bb6cb00878f2d5ba1c283a3362e2b80baf73284a Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Mon, 24 Jun 2019 20:27:35 +0800 Subject: [PATCH 43/68] determine if the file exists in hdfs when read file --- .../java/cn/escheduler/api/enums/Status.java | 1 + .../api/service/ResourcesService.java | 27 ++++++++++--------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java b/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java index d165fb6c16..a76a61517c 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java @@ -175,6 +175,7 @@ public enum Status { UDF_RESOURCE_SUFFIX_NOT_JAR(20009, "UDF resource suffix name must be jar"), HDFS_COPY_FAIL(20009, "hdfs copy {0} -> {1} fail"), RESOURCE_FILE_EXIST(20010, "resource file {0} already exists in hdfs,please delete it or change name!"), + RESOURCE_FILE_NOT_EXIST(20011, "resource file {0} not exists in hdfs!"), diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java index 7b56d927ad..f5ed65359a 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java @@ -515,13 +515,19 @@ public class ResourcesService extends BaseService { String hdfsFileName = HadoopUtils.getHdfsFilename(tenantCode, resource.getAlias()); logger.info("resource hdfs path is {} ", hdfsFileName); try { - List content = HadoopUtils.getInstance().catFile(hdfsFileName, skipLineNum, limit); + if(HadoopUtils.getInstance().exists(hdfsFileName)){ + List content = HadoopUtils.getInstance().catFile(hdfsFileName, skipLineNum, limit); + + putMsg(result, Status.SUCCESS); + Map map = new HashMap<>(); + map.put(ALIAS, resource.getAlias()); + map.put(CONTENT, StringUtils.join(content.toArray(), "\n")); + result.setData(map); + }else{ + logger.error("read file {} not exist in hdfs", hdfsFileName); + putMsg(result, Status.RESOURCE_FILE_NOT_EXIST); + } - putMsg(result, Status.SUCCESS); - Map map = new HashMap<>(); - map.put(ALIAS, resource.getAlias()); - map.put(CONTENT, StringUtils.join(content.toArray(), "\n")); - result.setData(map); } catch (Exception e) { logger.error(String.format("Resource %s read failed", hdfsFileName), e); putMsg(result, Status.HDFS_OPERATION_ERROR); @@ -565,17 +571,14 @@ public class ResourcesService extends BaseService { String name = fileName.trim() + "." + nameSuffix; - //check file already exists - Resource resource = resourcesMapper.queryResourceByNameAndType(name, type.ordinal()); - if (resource != null) { - logger.error("resource {} has exist, can't recreate .", name); - putMsg(result, Status.RESOURCE_EXIST); + result = verifyResourceName(name,type,loginUser); + if (!result.getCode().equals(Status.SUCCESS.getCode())) { return result; } // save data Date now = new Date(); - resource = new Resource(name,name,desc,loginUser.getId(),type,content.getBytes().length,now,now); + Resource resource = new Resource(name,name,desc,loginUser.getId(),type,content.getBytes().length,now,now); resourcesMapper.insert(resource); From 996dace03db00438623bf9e00283294f2360a612 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Mon, 24 Jun 2019 20:58:18 +0800 Subject: [PATCH 44/68] determine if the file exists in hdfs when read file --- .../main/java/cn/escheduler/api/service/ResourcesService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java index f5ed65359a..aeb7e18096 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java @@ -525,7 +525,7 @@ public class ResourcesService extends BaseService { result.setData(map); }else{ logger.error("read file {} not exist in hdfs", hdfsFileName); - putMsg(result, Status.RESOURCE_FILE_NOT_EXIST); + putMsg(result, Status.RESOURCE_FILE_NOT_EXIST,hdfsFileName); } } catch (Exception e) { From e8a41b552295aaf00aa848163dd7b89bb6b55161 Mon Sep 17 00:00:00 2001 From: lenboo Date: Tue, 25 Jun 2019 10:49:59 +0800 Subject: [PATCH 45/68] update documents --- docs/zh_CN/后端部署文档.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh_CN/后端部署文档.md b/docs/zh_CN/后端部署文档.md index 273f34dd22..e7f936fb7b 100644 --- a/docs/zh_CN/后端部署文档.md +++ b/docs/zh_CN/后端部署文档.md @@ -4,7 +4,7 @@ ## 1、准备工作 -目前最新安装包版本是1.0.4，下载地址： [码云下载](https://gitee.com/easyscheduler/EasyScheduler/attach_files/) ，下载escheduler-backend-1.0.4.tar.gz(后端简称escheduler-backend),escheduler-ui-1.0.4.tar.gz(前端简称escheduler-ui) +请下载最新版本的安装包，下载地址： [码云下载](https://gitee.com/easyscheduler/EasyScheduler/attach_files/) ，下载escheduler-backend-x.x.x.tar.gz(后端简称escheduler-backend),escheduler-ui-x.x.x.tar.gz(前端简称escheduler-ui) #### 准备一: 基础软件安装(必装项请自行安装) From adac6cc8e349f70683f9d1385e68b28e2de6d7bc Mon Sep 17 00:00:00 2001 From: bao liang <29528966+lenboo@users.noreply.github.com> Date: Tue, 25 Jun 2019 11:05:18 +0800 Subject: [PATCH 46/68] =?UTF-8?q?Update=20=E5=90=8E=E7=AB=AF=E9=83=A8?= =?UTF-8?q?=E7=BD=B2=E6=96=87=E6=A1=A3.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/zh_CN/后端部署文档.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh_CN/后端部署文档.md b/docs/zh_CN/后端部署文档.md index e7f936fb7b..ea1090a1d4 100644 --- a/docs/zh_CN/后端部署文档.md +++ b/docs/zh_CN/后端部署文档.md @@ -149,7 +149,7 @@ install.sh : 一键部署脚本 ### 2.2 编译源码来部署 -将源码包release版本1.0.4下载后，解压进入根目录 +将源码包release版本下载后，解压进入根目录 * 执行编译命令： From 63dd1f8c0786f989235e50d0fff1fc708d9d3e37 Mon Sep 17 00:00:00 2001 From: ligang Date: Tue, 25 Jun 2019 13:51:38 +0800 Subject: [PATCH 47/68] [maven-release-plugin] prepare release 1.0.4 --- escheduler-alert/pom.xml | 2 +- escheduler-api/pom.xml | 2 +- escheduler-common/pom.xml | 2 +- escheduler-dao/pom.xml | 2 +- escheduler-rpc/pom.xml | 2 +- escheduler-server/pom.xml | 2 +- pom.xml | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/escheduler-alert/pom.xml b/escheduler-alert/pom.xml index 13958c1b37..54925b6cc8 100644 --- a/escheduler-alert/pom.xml +++ b/escheduler-alert/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.0.3-SNAPSHOT + 1.0.4 escheduler-alert jar diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index 759942dbbe..3d15bad31e 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -3,7 +3,7 @@ cn.analysys escheduler - 1.0.3-SNAPSHOT + 1.0.4 escheduler-api jar diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index 2b92694564..020bf5c3fd 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.0.3-SNAPSHOT + 1.0.4 escheduler-common escheduler-common diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index b2898adce3..eb9184279a 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.0.3-SNAPSHOT + 1.0.4 escheduler-dao escheduler-dao diff --git a/escheduler-rpc/pom.xml b/escheduler-rpc/pom.xml index 416f1495ed..e11d808e03 100644 --- a/escheduler-rpc/pom.xml +++ b/escheduler-rpc/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.0.3-SNAPSHOT + 1.0.4 4.0.0 diff --git a/escheduler-server/pom.xml b/escheduler-server/pom.xml index f6390264e0..51c7923db5 100644 --- a/escheduler-server/pom.xml +++ b/escheduler-server/pom.xml @@ -3,7 +3,7 @@ escheduler cn.analysys - 1.0.3-SNAPSHOT + 1.0.4 escheduler-server escheduler-server diff --git a/pom.xml b/pom.xml index 89d7de8acf..708180afd1 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 cn.analysys escheduler - 1.0.3-SNAPSHOT + 1.0.4 pom escheduler http://maven.apache.org @@ -390,7 +390,7 @@ scm:git:https://github.com/analysys/EasyScheduler.git scm:git:https://github.com/analysys/EasyScheduler.git https://github.com/analysys/EasyScheduler.git - HEAD + 1.0.4 From 8f070a924050aa7bed8b3353f404788d12cfbb14 Mon Sep 17 00:00:00 2001 From: ligang Date: Tue, 25 Jun 2019 13:52:05 +0800 Subject: [PATCH 48/68] [maven-release-plugin] prepare for next development iteration --- escheduler-alert/pom.xml | 2 +- escheduler-api/pom.xml | 2 +- escheduler-common/pom.xml | 2 +- escheduler-dao/pom.xml | 2 +- escheduler-rpc/pom.xml | 2 +- escheduler-server/pom.xml | 2 +- pom.xml | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/escheduler-alert/pom.xml b/escheduler-alert/pom.xml index 54925b6cc8..58c9df4a51 100644 --- a/escheduler-alert/pom.xml +++ b/escheduler-alert/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.0.4 + 1.0.4-SNAPSHOT escheduler-alert jar diff --git a/escheduler-api/pom.xml b/escheduler-api/pom.xml index 3d15bad31e..1d0431ea52 100644 --- a/escheduler-api/pom.xml +++ b/escheduler-api/pom.xml @@ -3,7 +3,7 @@ cn.analysys escheduler - 1.0.4 + 1.0.4-SNAPSHOT escheduler-api jar diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml index 020bf5c3fd..cf5ba6048b 100644 --- a/escheduler-common/pom.xml +++ b/escheduler-common/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.0.4 + 1.0.4-SNAPSHOT escheduler-common escheduler-common diff --git a/escheduler-dao/pom.xml b/escheduler-dao/pom.xml index eb9184279a..42e2f8730a 100644 --- a/escheduler-dao/pom.xml +++ b/escheduler-dao/pom.xml @@ -4,7 +4,7 @@ cn.analysys escheduler - 1.0.4 + 1.0.4-SNAPSHOT escheduler-dao escheduler-dao diff --git a/escheduler-rpc/pom.xml b/escheduler-rpc/pom.xml index e11d808e03..4ec4ea2260 100644 --- a/escheduler-rpc/pom.xml +++ b/escheduler-rpc/pom.xml @@ -4,7 +4,7 @@ escheduler cn.analysys - 1.0.4 + 1.0.4-SNAPSHOT 4.0.0 diff --git a/escheduler-server/pom.xml b/escheduler-server/pom.xml index 51c7923db5..cd33d49006 100644 --- a/escheduler-server/pom.xml +++ b/escheduler-server/pom.xml @@ -3,7 +3,7 @@ escheduler cn.analysys - 1.0.4 + 1.0.4-SNAPSHOT escheduler-server escheduler-server diff --git a/pom.xml b/pom.xml index 708180afd1..ea4545d787 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 cn.analysys escheduler - 1.0.4 + 1.0.4-SNAPSHOT pom escheduler http://maven.apache.org @@ -390,7 +390,7 @@ scm:git:https://github.com/analysys/EasyScheduler.git scm:git:https://github.com/analysys/EasyScheduler.git https://github.com/analysys/EasyScheduler.git - 1.0.4 + HEAD From 602591aa8c0ded9d002389d4ed73211f602b860c Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Wed, 26 Jun 2019 15:42:30 +0800 Subject: [PATCH 49/68] qianfan task result add judge update --- .../worker/runner/TaskScheduleThread.java | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java index 60edad5606..75b1e01a60 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java @@ -25,15 +25,14 @@ import cn.escheduler.common.model.TaskNode; import cn.escheduler.common.process.Property; import cn.escheduler.common.task.AbstractParameters; import cn.escheduler.common.task.TaskTimeoutParameter; -import cn.escheduler.common.utils.CommonUtils; -import cn.escheduler.common.utils.DateUtils; -import cn.escheduler.common.utils.HadoopUtils; -import cn.escheduler.common.utils.TaskParametersUtils; +import cn.escheduler.common.task.shell.ShellParameters; +import cn.escheduler.common.utils.*; import cn.escheduler.dao.ProcessDao; import cn.escheduler.dao.TaskRecordDao; import cn.escheduler.dao.model.ProcessInstance; import cn.escheduler.dao.model.TaskInstance; import cn.escheduler.server.utils.LoggerUtils; +import cn.escheduler.server.utils.ParamUtils; import cn.escheduler.server.worker.log.TaskLogger; import cn.escheduler.server.worker.task.AbstractTask; import cn.escheduler.server.worker.task.TaskManager; @@ -144,6 +143,7 @@ public class TaskScheduleThread implements Callable { TaskNode taskNode = JSONObject.parseObject(taskJson, TaskNode.class); + List projectRes = createProjectResFiles(taskNode); // copy hdfs file to local @@ -205,17 +205,25 @@ public class TaskScheduleThread implements Callable { // task recor flat : if true , start up qianfan if (TaskRecordDao.getTaskRecordFlag() && TaskType.typeIsNormalTask(taskInstance.getTaskType())){ - Date scheduleTime = processInstance.getScheduleTime(); - if(scheduleTime == null){ - scheduleTime = processInstance.getStartTime(); - } - // process exec time : yyyyMMdd format - String scheduleDate = DateUtils.format(scheduleTime, Constants.YYYYMMDD); - TaskRecordStatus taskRecordState = TaskRecordDao.getTaskRecordState(taskInstance.getName(), scheduleDate); - logger.info("task record status : {}",taskRecordState); - if (taskRecordState == TaskRecordStatus.FAILURE){ - status = ExecutionStatus.FAILURE; + AbstractParameters params = JSONUtils.parseObject(taskProps.getTaskParams(), AbstractParameters.class); + + // replace placeholder + Map paramsMap = ParamUtils.convert(taskProps.getUserDefParamsMap(), + taskProps.getDefinedParams(), + params.getLocalParametersMap(), + processInstance.getCmdTypeIfComplement(), + processInstance.getScheduleTime()); + if (paramsMap != null && !paramsMap.isEmpty() + && paramsMap.containsKey("v_proc_date")){ + String vProcDate = paramsMap.get("v_proc_date").getValue(); + if (!StringUtils.isEmpty(vProcDate)){ + TaskRecordStatus taskRecordState = TaskRecordDao.getTaskRecordState(taskInstance.getName(), vProcDate); + logger.info("task record status : {}",taskRecordState); + if (taskRecordState == TaskRecordStatus.FAILURE){ + status = ExecutionStatus.FAILURE; + } + } } } From 8b88cd0b3d71ee0503f60245f46595eb3eff5220 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Wed, 26 Jun 2019 16:09:03 +0800 Subject: [PATCH 50/68] qianfan task result add judge update --- .../worker/runner/TaskScheduleThread.java | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java index 75b1e01a60..c21a3a2f1b 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/TaskScheduleThread.java @@ -25,7 +25,12 @@ import cn.escheduler.common.model.TaskNode; import cn.escheduler.common.process.Property; import cn.escheduler.common.task.AbstractParameters; import cn.escheduler.common.task.TaskTimeoutParameter; +import cn.escheduler.common.task.mr.MapreduceParameters; +import cn.escheduler.common.task.procedure.ProcedureParameters; +import cn.escheduler.common.task.python.PythonParameters; import cn.escheduler.common.task.shell.ShellParameters; +import cn.escheduler.common.task.spark.SparkParameters; +import cn.escheduler.common.task.sql.SqlParameters; import cn.escheduler.common.utils.*; import cn.escheduler.dao.ProcessDao; import cn.escheduler.dao.TaskRecordDao; @@ -206,7 +211,7 @@ public class TaskScheduleThread implements Callable { if (TaskRecordDao.getTaskRecordFlag() && TaskType.typeIsNormalTask(taskInstance.getTaskType())){ - AbstractParameters params = JSONUtils.parseObject(taskProps.getTaskParams(), AbstractParameters.class); + AbstractParameters params = (AbstractParameters) JSONUtils.parseObject(taskProps.getTaskParams(), getCurTaskParamsClass()); // replace placeholder Map paramsMap = ParamUtils.convert(taskProps.getUserDefParamsMap(), @@ -279,6 +284,39 @@ public class TaskScheduleThread implements Callable { } + /** + * get current task parameter class + * @return + */ + private Class getCurTaskParamsClass(){ + Class paramsClass = null; + TaskType taskType = TaskType.valueOf(taskInstance.getTaskType()); + switch (taskType){ + case SHELL: + paramsClass = ShellParameters.class; + break; + case SQL: + paramsClass = SqlParameters.class; + break; + case PROCEDURE: + paramsClass = ProcedureParameters.class; + break; + case MR: + paramsClass = MapreduceParameters.class; + break; + case SPARK: + paramsClass = SparkParameters.class; + break; + case PYTHON: + paramsClass = PythonParameters.class; + break; + default: + logger.error("not support this task type: {}", taskType); + throw new IllegalArgumentException("not support this task type"); + } + return paramsClass; + } + /** * kill task */ From 455d82a7400d3f5845f56ce37fb4f16066ed4fef Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Thu, 27 Jun 2019 11:52:39 +0800 Subject: [PATCH 51/68] sqlTask update --- .../main/java/cn/escheduler/server/worker/task/sql/SqlTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java index 4eb567d8c8..dd10d05ddf 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java @@ -374,7 +374,7 @@ public class SqlTask extends AbstractTask { String showTypeName = sqlParameters.getShowType().replace(Constants.COMMA,"").trim(); if(EnumUtils.isValidEnum(ShowType.class,showTypeName)){ Map mailResult = MailUtils.sendMails(receviersList, receviersCcList, title, content, ShowType.valueOf(showTypeName)); - if(!(Boolean) mailResult.get(cn.escheduler.api.utils.Constants.STATUS)){ + if(!(Boolean) mailResult.get(Constants.STATUS)){ throw new RuntimeException("send mail failed!"); } }else{ From 729963fc63caddf5451ad0c2b10ca402375e7779 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Thu, 27 Jun 2019 13:53:51 +0800 Subject: [PATCH 52/68] tenant service update --- .../main/java/cn/escheduler/api/service/TenantService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java index ffaa4de7ad..35bb35a2b4 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java @@ -235,8 +235,8 @@ public class TenantService extends BaseService{ return result; } - // if hdfs startup - if (PropertyUtils.getBoolean(cn.escheduler.common.Constants.HDFS_STARTUP_STATE)){ + // if resource upload startup + if (PropertyUtils.getResUploadStartupState()){ String tenantPath = HadoopUtils.getHdfsDataBasePath() + "/" + tenant.getTenantCode(); String resourcePath = HadoopUtils.getHdfsDir(tenant.getTenantCode()); From ce8645bc69b8ef57fc90e60d0e1154444fa1d0f0 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Thu, 27 Jun 2019 16:09:22 +0800 Subject: [PATCH 53/68] add kerberos interface --- .../api/controller/DataSourceController.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/controller/DataSourceController.java b/escheduler-api/src/main/java/cn/escheduler/api/controller/DataSourceController.java index 5a0b911581..4d1aaa6c28 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/controller/DataSourceController.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/controller/DataSourceController.java @@ -34,9 +34,11 @@ import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.*; import springfox.documentation.annotations.ApiIgnore; +import java.util.HashMap; import java.util.Map; import static cn.escheduler.api.enums.Status.*; +import static cn.escheduler.common.utils.PropertyUtils.getBoolean; /** @@ -429,4 +431,24 @@ public class DataSourceController extends BaseController { return error(AUTHORIZED_DATA_SOURCE.getCode(), AUTHORIZED_DATA_SOURCE.getMsg()); } } + + /** + * get user info + * + * @param loginUser + * @return + */ + @ApiOperation(value = "getKerberosStartupState", notes= "GET_USER_INFO_NOTES") + @GetMapping(value="/kerberos-startup-state") + @ResponseStatus(HttpStatus.OK) + public Result getKerberosStartupState(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser){ + logger.info("login user {},get user info : {}", loginUser.getUserName()); + try{ + Boolean kerberosStartupState = getBoolean(cn.escheduler.common.Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE); + return success(Status.SUCCESS.getMsg(),kerberosStartupState); + }catch (Exception e){ + logger.error(KERBEROS_STARTUP_STATE.getMsg(),e); + return error(Status.KERBEROS_STARTUP_STATE.getCode(), Status.KERBEROS_STARTUP_STATE.getMsg()); + } + } } From a0e1495f27f3ad05ece5167957546932fc0e740f Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Thu, 27 Jun 2019 16:12:04 +0800 Subject: [PATCH 54/68] add kerberos interface --- escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java | 1 + 1 file changed, 1 insertion(+) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java b/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java index 7d4a3c6381..bd1d5e91b3 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java @@ -233,6 +233,7 @@ public enum Status { QUEUE_COUNT_ERROR(90001,"queue count error"), + KERBEROS_STARTUP_STATE(100001,"get kerberos startup state error"), ; private int code; From 3f217f5d00f21eac83320d2241448ca49a3701ed Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Fri, 28 Jun 2019 14:39:02 +0800 Subject: [PATCH 55/68] add datasource kerberos auth and FAQ modify --- docs/zh_CN/EasyScheduler-FAQ.md | 267 +++++++++++++++--- .../api/controller/DataSourceController.java | 30 +- .../api/service/DataSourceService.java | 39 ++- .../cn/escheduler/api/utils/CheckUtils.java | 14 + .../cn/escheduler/api/utils/Constants.java | 1 + .../common/job/db/BaseDataSource.java | 20 +- .../common/job/db/HiveDataSource.java | 13 +- .../common/job/db/SparkDataSource.java | 3 +- .../pages/list/_source/createDataSource.vue | 41 ++- .../js/conf/home/store/datasource/actions.js | 9 + .../src/js/module/i18n/locale/en_US.js | 3 +- .../src/js/module/i18n/locale/zh_CN.js | 1 + install.sh | 2 +- 13 files changed, 372 insertions(+), 71 deletions(-) diff --git a/docs/zh_CN/EasyScheduler-FAQ.md b/docs/zh_CN/EasyScheduler-FAQ.md index e9f9d5d7ab..e6af99cae4 100644 --- a/docs/zh_CN/EasyScheduler-FAQ.md +++ b/docs/zh_CN/EasyScheduler-FAQ.md @@ -1,96 +1,283 @@ -Q:单机运行服务老挂，应该是内存不够，测试机器4核8G。生产环境需要分布式，如果单机的话建议的配置是？ +## Q：EasyScheduler服务介绍及建议运行内存 -A: Easy Scheduler有5个服务组成，这些服务本身需要的内存和cpu不多， +A： EasyScheduler由5个服务组成，MasterServer、WorkerServer、ApiServer、AlertServer、LoggerServer和UI。 -| 服务 | 内存 | cpu核数 | -| ------------ | ---- | ------- | -| MasterServer | 2G | 2核 | -| WorkerServer | 2G | 2核 | -| ApiServer | 512M | 1核 | -| AlertServer | 512M | 1核 | -| LoggerServer | 512M | 1核 | +| 服务 | 说明 | +| ------------------------- | ------------------------------------------------------------ | +| MasterServer | 主要负责 **DAG** 的切分和任务状态的监控 | +| WorkerServer/LoggerServer | 主要负责任务的提交、执行和任务状态的更新。LoggerServer用于Rest Api通过 **RPC** 查看日志 | +| ApiServer | 提供Rest Api服务，供UI进行调用 | +| AlertServer | 提供告警服务 | +| UI | 前端页面展示 | -注意：由于如果任务较多，WorkServer所在机器建议物理内存在16G以上 +注意：**由于服务比较多，建议单机部署最好是4核16G以上** +--- + +## Q： 管理员为什么不能创建项目 + +A：管理员目前属于"**纯管理**", 没有租户，即没有linux上对应的用户，所以没有执行权限, **故没有所属的项目、资源及数据源**，所以没有创建权限。**但是有所有的查看权限**。如果需要创建项目等业务操作，**请使用管理员创建租户和普通用户，然后使用普通用户登录进行操作**。我们将会在1.1.0版本中将管理员的创建和执行权限放开，管理员将会有所有的权限 + +--- + +## Q：系统支持哪些邮箱？ + +A：支持绝大多数邮箱，qq、163、126、139、outlook、aliyun等皆支持。支持**TLS和SSL**协议，可以在alert.properties中选择性配置 + +--- + +## Q：常用的系统变量时间参数有哪些，如何使用？ + +A：请参考 https://analysys.github.io/easyscheduler_docs_cn/%E7%B3%BB%E7%BB%9F%E4%BD%BF%E7%94%A8%E6%89%8B%E5%86%8C.html#%E7%B3%BB%E7%BB%9F%E5%8F%82%E6%95%B0 + +--- +## Q：pip install kazoo 这个安装报错。是必须安装的吗？ + +A： 这个是python连接zookeeper需要使用到的，必须要安装 --- -Q: 管理员为什么不能创建项目？ +## Q: 怎么指定机器运行任务 -A: 管理员目前属于"纯管理", 没有租户，即没有linux上对应的用户，所以没有执行权限, 但是有所有的查看权限。如果需要创建项目等业务操作，请使用管理员创建租户和普通用户，然后使用普通用户登录进行操作 +A：使用 **管理员** 创建Worker分组，在 **流程定义启动** 的时候可**指定Worker分组**或者在**任务节点上指定Worker分组**。如果不指定，则使用Default，**Default默认是使用的集群里所有的Worker中随机选取一台来进行任务提交、执行** --- -Q: 系统支持哪些邮箱？ +## Q：任务的优先级 + +A：我们同时 **支持流程和任务的优先级**。优先级我们有 **HIGHEST、HIGH、MEDIUM、LOW和LOWEST** 五种级别。**可以设置不同流程实例之间的优先级，也可以设置同一个流程实例中不同任务实例的优先级**。详细内容请参考任务优先级设计 https://analysys.github.io/easyscheduler_docs_cn/%E7%B3%BB%E7%BB%9F%E6%9E%B6%E6%9E%84%E8%AE%BE%E8%AE%A1.html#%E7%B3%BB%E7%BB%9F%E6%9E%B6%E6%9E%84%E8%AE%BE%E8%AE%A1 + +---- + +## Q：escheduler-grpc报错 + +A：在根目录下执行：mvn -U clean package assembly:assembly -Dmaven.test.skip=true ， 然后刷新下整个项目 + +---- + +## Q：EasyScheduler支持windows上运行么 + +A： 理论上只有**Worker是需要在Linux上运行的**，其它的服务都是可以在windows上正常运行的。但是还是建议最好能在linux上部署使用 + +----- + +## Q：UI 在 linux 编译node-sass提示：Error：EACCESS:permission denied，mkdir xxxx -A: 支持绝大多数邮箱，qq、163、126、139、outlook、aliyun等皆可支持 +A：单独安装 **npm install node-sass --unsafe-perm**，之后再 **npm install** --- -Q:常用的系统变量时间参数有哪些,如何使用？ +## Q：UI 不能正常登陆访问 -A: 请参考使用手册中的系统参数 +A： 1，如果是node启动的查看escheduler-ui下的.env API_BASE配置是否是Api Server服务地址 + +​ 2，如果是nginx启动的并且是通过 **install-escheduler-ui.sh** 安装的，查看 **/etc/nginx/conf.d/escheduler.conf** 中的proxy_pass配置是否是Api Server服务地址 + +​ 3，如果以上配置都是正确的，那么请查看Api Server服务是否是正常的，curl http://192.168.xx.xx:12345/escheduler/users/get-user-info，查看Api Server日志，如果提示 cn.escheduler.api.interceptor.LoginHandlerInterceptor:[76] - session info is null，则证明Api Server服务是正常的 + +​ 4，如果以上都没有问题，需要查看一下 **application.properties** 中的 **server.context-path 和 server.port 配置**是否正确 --- -Q：pip install kazoo 这个安装报错。是必须安装的吗？ +## Q： 流程定义手动启动或调度启动之后，没有流程实例生成 + +A： 1，首先通过**jps 查看MasterServer服务是否存在**，或者从服务监控直接查看zk中是否存在master服务 -A: 这个是python连接zookeeper需要使用到的 +​ 2，如果存在master服务，查看 **命令状态统计** 或者 **t_escheduler_error_command** 中是否增加的新记录，如果增加了，**请查看 message 字段定位启动异常原因** --- -Q: 如果alert、api、logger服务任意一个宕机，任何还会正常执行吧 +## Q ： 任务状态一直处于提交成功状态 -A: 不影响，影响正在运行中的任务的服务有Master和Worker服务 +A： 1，首先通过**jps 查看WorkerServer服务是否存在**，或者从服务监控直接查看zk中是否存在worker服务 + +​ 2，如果 **WorkerServer** 服务正常，需要 **查看MasterServer是否把task任务放到zk队列中** ，**需要查看MasterServer日志及zk队列中是否有任务阻塞** + +​ 3，如果以上都没有问题，需要定位是否指定了Worker分组，但是 **Worker分组的机器不是在线状态** --- -Q: 这个怎么指定机器运行任务的啊 」 +## Q： 是否提供Docker镜像及Dockerfile + +A： 提供Docker镜像及Dockerfile。 -A: 通过worker分组： 这个流程只能在指定的机器组里执行。默认是Default，可以在任一worker上执行。 +Docker镜像地址：https://hub.docker.com/r/escheduler/escheduler_images + +Dockerfile地址：https://github.com/qiaozhanwei/escheduler_dockerfile/tree/master/docker_escheduler --- -Q: 跨用户的任务依赖怎么实现呢， 比如A用户写了一个任务，B用户需要依赖这个任务 +## Q ： install.sh 中需要注意问题 + +A： 1，如果替换变量中包含特殊字符，**请用 \ 转移符进行转移** + +​ 2，installPath="/data1_1T/escheduler"，**这个目录不能和当前要一键安装的install.sh目录是一样的** -就比如说 我们数仓组 写了一个 中间宽表的任务， 其他业务部门想要使用这个中间表的时候，他们应该是另外一个用户，怎么依赖这个中间表呢 +​ 3，deployUser="escheduler"，**部署用户必须具有sudo权限**，因为worker是通过sudo -u 租户 sh xxx.command进行执行的 -A: 有两种情况，一个是要运行这个宽表任务，可以使用子工作流把宽表任务放到自己的工作流里面。另一个是检查这个宽表任务有没有完成，可以使用依赖节点来检查这个宽表任务在指定的时间周期有没有完成。 +​ 4，monitorServerState="false"，服务监控脚本是否启动，默认是不启动服务监控脚本的。**如果启动服务监控脚本，则每5分钟定时来监控master和worker的服务是否down机，如果down机则会自动重启** + +​ 5，hdfsStartupSate="false"，是否开启HDFS资源上传功能。默认是不开启的，**如果不开启则资源中心是不能使用的**。如果开启，需要conf/common/hadoop/hadoop.properties中配置fs.defaultFS和yarn的相关配置，如果使用namenode HA，需要将core-site.xml和hdfs-site.xml复制到conf根目录下 + +​ 注意：**1.0.x版本是不会自动创建hdfs根目录的，需要自行创建，并且需要部署用户有hdfs的操作权限** --- -Q: 启动WorkerServer服务时不能正常启动，报以下信息是什么原因？ +## Q ： 流程定义和流程实例下线异常 + +A ： 对于 **1.0.4 以前的版本中**，修改escheduler-api cn.escheduler.api.quartz包下的代码即可 ``` -[INFO] 2019-05-06 16:39:31.492 cn.escheduler.server.zk.ZKWorkerClient:[155] - register failure , worker already started on : 127.0.0.1, please wait for a moment and try again +public boolean deleteJob(String jobName, String jobGroupName) { + lock.writeLock().lock(); + try { + JobKey jobKey = new JobKey(jobName,jobGroupName); + if(scheduler.checkExists(jobKey)){ + logger.info("try to delete job, job name: {}, job group name: {},", jobName, jobGroupName); + return scheduler.deleteJob(jobKey); + }else { + return true; + } + + } catch (SchedulerException e) { + logger.error(String.format("delete job : %s failed",jobName), e); + } finally { + lock.writeLock().unlock(); + } + return false; + } ``` -A：Worker/Master Server在启动时，会向Zookeeper注册自己的启动信息，是Zookeeper的临时节点，如果两次启动时间间隔较短的情况，上次启动的Worker/Master Server在Zookeeper的会话还未过期，会出现上述信息，处理办法是等待session过期，一般是1分钟左右 +--- ----- +## Q ： HDFS启动之前创建的租户，能正常使用资源中心吗 -Q: 编译时escheduler-grpc模块一直报错：Information:java: Errors occurred while compiling module 'escheduler-rpc'， 找不到LogParameter、RetStrInfo、RetByteInfo等class类 +A： 不能。因为在未启动HDFS创建的租户，不会在HDFS中注册租户目录。所以上次资源会报错 -A: 这是因为rpc源码包是google Grpc实现的，需要使用maven进行编译，在根目录下执行：mvn -U clean package assembly:assembly -Dmaven.test.skip=true ， 然后刷新下整个项目 +## Q : 多Master和多Worker状态下，服务掉了，怎么容错 ----- +A： **注意：Master监控Master及Worker服务。** -Q：EasyScheduler支持windows上运行么？ +​ 1，如果Master服务掉了，其它的Master会接管挂掉的Master的流程，继续监控Worker task状态 -A: 建议在Ubuntu、Centos上运行，暂不支持windows上运行，不过windows上可以进行编译。开发调试的话建议Ubuntu或者mac上进行。 +​ 2，如果Worker服务掉，Master会监控到Worker服务掉了，如果存在Yarn任务，Kill Yarn任务之后走重试 + +具体请看容错设计：https://analysys.github.io/easyscheduler_docs_cn/%E7%B3%BB%E7%BB%9F%E6%9E%B6%E6%9E%84%E8%AE%BE%E8%AE%A1.html#%E7%B3%BB%E7%BB%9F%E6%9E%B6%E6%9E%84%E8%AE%BE%E8%AE%A1 + +--- + +## Q ： 对于Master和Worker一台机器伪分布式下的容错 + +A ： 1.0.3 版本只实现了Master启动流程容错，不走Worker容错。也就是说如果Worker挂掉的时候，没有Master存在。这流程将会出现问题。我们会在 **1.1.0** 版本中增加Master和Worker启动自容错，修复这个问题。如果想手动修改这个问题，需要针对 **跨重启正在运行流程** **并且已经掉的正在运行的Worker任务，需要修改为失败**，**同时跨重启正在运行流程设置为失败状态**。然后从失败节点进行流程恢复即可 + +--- + +## Q ： 定时容易设置成每秒执行 + +A ： 设置定时的时候需要注意，如果第一位（* * * * * ? *）设置成 \* ，则表示每秒执行。**我们将会在1.1.0版本中加入显示最近调度的时间列表** ，使用http://cron.qqe2.com/ 可以在线看近5次运行时间 ------ -Q：任务为什么不执行？ -A: 不执行的原因： +## Q： 定时有有效时间范围吗 -查看command表里有没有内容？ +A：有的，**如果定时的起止时间是同一个时间，那么此定时将是无效的定时**。**如果起止时间的结束时间比当前的时间小，很有可能定时会被自动删除** -查看Master server的运行日志： -查看Worker Server的运行日志 +## Q ： 任务依赖有几种实现 +A： 1，**DAG** 之间的任务依赖关系，是从 **入度为零** 进行DAG切分的 + +​ 2，有 **任务依赖节点** ，可以实现跨流程的任务或者流程依赖，具体请参考 依赖(DEPENDENT)节点：https://analysys.github.io/easyscheduler_docs_cn/%E7%B3%BB%E7%BB%9F%E4%BD%BF%E7%94%A8%E6%89%8B%E5%86%8C.html#%E4%BB%BB%E5%8A%A1%E8%8A%82%E7%82%B9%E7%B1%BB%E5%9E%8B%E5%92%8C%E5%8F%82%E6%95%B0%E8%AE%BE%E7%BD%AE + +​ 注意：**不支持跨项目的流程或任务依赖** + +## Q： 流程定义有几种启动方式 + +A： 1，在 **流程定义列表**，点击 **启动** 按钮 + +​ 2，**流程定义列表添加定时器**，调度启动流程定义 + +​ 3，流程定义 **查看或编辑** DAG 页面，任意 **任务节点右击** 启动流程定义 + +​ 4，可以对流程定义 DAG 编辑，设置某些任务的运行标志位 **禁止运行**，则在启动流程定义的时候，将该节点的连线将从DAG中去掉 + +## Q ： Python任务设置Python版本 + +A： 1，对于1**.0.3之后的版本**只需要修改 conf/env/.escheduler_env.sh中的PYTHON_HOME + +``` +export PYTHON_HOME=/bin/python +``` + +注意：这了 **PYTHON_HOME** ，是python命令的绝对路径，而不是单纯的 PYTHON_HOME，还需要注意的是 export PATH 的时候，需要直接 + +``` +export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH +``` + +​ 2，对 1.0.3 之前的版本，Python任务只能支持系统的Python版本，不支持指定Python版本 + +## Q： Worker Task 通过sudo -u 租户 sh xxx.command会产生子进程，在kill的时候，是否会杀掉 + +A： 我们会在1.0.4中增加kill任务同时，kill掉任务产生的各种所有子进程 + + + +## Q ： EasyScheduler中的队列怎么用，用户队列和租户队列是什么意思 + +A ： EasyScheduler 中的队列可以在用户或者租户上指定队列，**用户指定的队列优先级是高于租户队列的优先级的。**，例如：对MR任务指定队列，是通过 mapreduce.job.queuename 来指定队列的。 + +注意：MR在用以上方法指定队列的时候，传递参数请使用如下方式： + +``` + Configuration conf = new Configuration(); + GenericOptionsParser optionParser = new GenericOptionsParser(conf, args); + String[] remainingArgs = optionParser.getRemainingArgs(); +``` + + + +如果是Spark任务 --queue 方式指定队列 + + + +## Q : Master 后者 Worker报如下告警 + +![1560847965302](C:\Users\Administrator\Desktop\FAQ\1560847965302.png) + + + +A ： 修改conf下的 master.properties **master.reserved.memory** 的值为更小的值，比如说0.1 或者 + +worker.properties **worker.reserved.memory** 的值为更小的值，比如说0.1 + + + +## Q : hive版本是1.1.0+cdh5.15.0，SQL hive任务连接报错 + +![EF4DA613-5129-4c7a-A0AB-61E5A866A919](C:\Users\Administrator\Desktop\FAQ\EF4DA613-5129-4c7a-A0AB-61E5A866A919.png) + + + +A ： 将 hive pom + +``` + + org.apache.hive + hive-jdbc + 2.1.0 + +``` + +修改为 + +``` + + org.apache.hive + hive-jdbc + 1.1.0 + +``` diff --git a/escheduler-api/src/main/java/cn/escheduler/api/controller/DataSourceController.java b/escheduler-api/src/main/java/cn/escheduler/api/controller/DataSourceController.java index 4d1aaa6c28..51e0c93bd6 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/controller/DataSourceController.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/controller/DataSourceController.java @@ -18,10 +18,13 @@ package cn.escheduler.api.controller; import cn.escheduler.api.enums.Status; import cn.escheduler.api.service.DataSourceService; +import cn.escheduler.api.utils.CheckUtils; import cn.escheduler.api.utils.Constants; import cn.escheduler.api.utils.Result; import cn.escheduler.common.enums.DbType; +import cn.escheduler.common.enums.ResUploadType; import cn.escheduler.common.utils.ParameterUtils; +import cn.escheduler.common.utils.PropertyUtils; import cn.escheduler.dao.model.User; import io.swagger.annotations.Api; import io.swagger.annotations.ApiImplicitParam; @@ -56,12 +59,16 @@ public class DataSourceController extends BaseController { /** * create data source - * 创建数据源 - * * @param loginUser * @param name * @param note * @param type + * @param host + * @param port + * @param database + * @param principal + * @param userName + * @param password * @param other * @return */ @@ -86,13 +93,14 @@ public class DataSourceController extends BaseController { @RequestParam(value = "host") String host, @RequestParam(value = "port") String port, @RequestParam(value = "database") String database, + @RequestParam(value = "principal") String principal, @RequestParam(value = "userName") String userName, @RequestParam(value = "password") String password, @RequestParam(value = "other") String other) { - logger.info("login user {} create datasource ame: {}, note: {}, type: {}, other: {}", - loginUser.getUserName(), name, note, type, other); + logger.info("login user {} create datasource name: {}, note: {}, type: {}, host: {},port: {},database : {},principal: {},userName : {} other: {}", + loginUser.getUserName(), name, note, type, host,port,database,principal,userName,other); try { - String parameter = dataSourceService.buildParameter(name, note, type, host, port, database, userName, password, other); + String parameter = dataSourceService.buildParameter(name, note, type, host, port, database,principal,userName, password, other); Map result = dataSourceService.createDataSource(loginUser, name, note, type, parameter); return returnDataList(result); @@ -136,13 +144,14 @@ public class DataSourceController extends BaseController { @RequestParam(value = "host") String host, @RequestParam(value = "port") String port, @RequestParam(value = "database") String database, + @RequestParam(value = "principal") String principal, @RequestParam(value = "userName") String userName, @RequestParam(value = "password") String password, @RequestParam(value = "other") String other) { logger.info("login user {} updateProcessInstance datasource name: {}, note: {}, type: {}, other: {}", loginUser.getUserName(), name, note, type, other); try { - String parameter = dataSourceService.buildParameter(name, note, type, host, port, database, userName, password, other); + String parameter = dataSourceService.buildParameter(name, note, type, host, port, database,principal, userName, password, other); Map dataSource = dataSourceService.updateDataSource(id, loginUser, name, note, type, parameter); return returnDataList(dataSource); } catch (Exception e) { @@ -271,13 +280,14 @@ public class DataSourceController extends BaseController { @RequestParam(value = "host") String host, @RequestParam(value = "port") String port, @RequestParam(value = "database") String database, + @RequestParam(value = "principal") String principal, @RequestParam(value = "userName") String userName, @RequestParam(value = "password") String password, @RequestParam(value = "other") String other) { logger.info("login user {}, connect datasource: {} failure, note: {}, type: {}, other: {}", loginUser.getUserName(), name, note, type, other); try { - String parameter = dataSourceService.buildParameter(name, note, type, host, port, database, userName, password, other); + String parameter = dataSourceService.buildParameter(name, note, type, host, port, database,principal,userName, password, other); Boolean isConnection = dataSourceService.checkConnection(type, parameter); Result result = new Result(); @@ -442,10 +452,10 @@ public class DataSourceController extends BaseController { @GetMapping(value="/kerberos-startup-state") @ResponseStatus(HttpStatus.OK) public Result getKerberosStartupState(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser){ - logger.info("login user {},get user info : {}", loginUser.getUserName()); + logger.info("login user {},get kerberos startup state : {}", loginUser.getUserName()); try{ - Boolean kerberosStartupState = getBoolean(cn.escheduler.common.Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE); - return success(Status.SUCCESS.getMsg(),kerberosStartupState); + // if upload resource is HDFS and kerberos startup is true , else false + return success(Status.SUCCESS.getMsg(), CheckUtils.getKerberosStartupState()); }catch (Exception e){ logger.error(KERBEROS_STARTUP_STATE.getMsg(),e); return error(Status.KERBEROS_STARTUP_STATE.getCode(), Status.KERBEROS_STARTUP_STATE.getMsg()); diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java index 9081a436cf..02164f971b 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/DataSourceService.java @@ -17,12 +17,15 @@ package cn.escheduler.api.service; import cn.escheduler.api.enums.Status; +import cn.escheduler.api.utils.CheckUtils; import cn.escheduler.api.utils.Constants; import cn.escheduler.api.utils.PageInfo; import cn.escheduler.api.utils.Result; import cn.escheduler.common.enums.DbType; +import cn.escheduler.common.enums.ResUploadType; import cn.escheduler.common.enums.UserType; import cn.escheduler.common.job.db.*; +import cn.escheduler.common.utils.PropertyUtils; import cn.escheduler.dao.mapper.DataSourceMapper; import cn.escheduler.dao.mapper.DatasourceUserMapper; import cn.escheduler.dao.mapper.ProjectMapper; @@ -31,6 +34,8 @@ import cn.escheduler.dao.model.Resource; import cn.escheduler.dao.model.User; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.TypeReference; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -42,6 +47,9 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.util.*; +import static cn.escheduler.common.utils.PropertyUtils.getBoolean; +import static cn.escheduler.common.utils.PropertyUtils.getString; + /** * datasource service */ @@ -55,6 +63,7 @@ public class DataSourceService extends BaseService{ public static final String TYPE = "type"; public static final String HOST = "host"; public static final String PORT = "port"; + public static final String PRINCIPAL = "principal"; public static final String DATABASE = "database"; public static final String USER_NAME = "userName"; public static final String PASSWORD = "password"; @@ -240,6 +249,7 @@ public class DataSourceService extends BaseService{ map.put(TYPE, dataSourceType); map.put(HOST, host); map.put(PORT, port); + map.put(PRINCIPAL, datasourceForm.getPrincipal()); map.put(DATABASE, database); map.put(USER_NAME, datasourceForm.getUser()); map.put(PASSWORD, datasourceForm.getPassword()); @@ -363,11 +373,21 @@ public class DataSourceService extends BaseService{ Class.forName(Constants.COM_MYSQL_JDBC_DRIVER); break; case HIVE: - datasource = JSONObject.parseObject(parameter, HiveDataSource.class); - Class.forName(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER); - break; case SPARK: - datasource = JSONObject.parseObject(parameter, SparkDataSource.class); + if (CheckUtils.getKerberosStartupState()) { + System.setProperty(cn.escheduler.common.Constants.JAVA_SECURITY_KRB5_CONF, + getString(cn.escheduler.common.Constants.JAVA_SECURITY_KRB5_CONF_PATH)); + Configuration configuration = new Configuration(); + configuration.set(cn.escheduler.common.Constants.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + UserGroupInformation.setConfiguration(configuration); + UserGroupInformation.loginUserFromKeytab(getString(cn.escheduler.common.Constants.LOGIN_USER_KEY_TAB_USERNAME), + getString(cn.escheduler.common.Constants.LOGIN_USER_KEY_TAB_PATH)); + } + if (dbType == DbType.HIVE){ + datasource = JSONObject.parseObject(parameter, HiveDataSource.class); + }else if (dbType == DbType.SPARK){ + datasource = JSONObject.parseObject(parameter, SparkDataSource.class); + } Class.forName(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER); break; case CLICKHOUSE: @@ -443,10 +463,18 @@ public class DataSourceService extends BaseService{ * @param other * @return */ - public String buildParameter(String name, String desc, DbType type, String host, String port, String database, String userName, String password, String other) { + public String buildParameter(String name, String desc, DbType type, String host, + String port, String database,String principal,String userName, + String password, String other) { String address = buildAddress(type, host, port); + String jdbcUrl = address + "/" + database; + if (CheckUtils.getKerberosStartupState() && + (type == DbType.HIVE || type == DbType.SPARK)){ + jdbcUrl += ";principal=" + principal; + } + String separator = ""; if (Constants.MYSQL.equals(type.name()) || Constants.POSTGRESQL.equals(type.name()) @@ -465,6 +493,7 @@ public class DataSourceService extends BaseService{ parameterMap.put(Constants.JDBC_URL, jdbcUrl); parameterMap.put(Constants.USER, userName); parameterMap.put(Constants.PASSWORD, password); + parameterMap.put(Constants.PRINCIPAL,principal); if (other != null && !"".equals(other)) { Map map = JSONObject.parseObject(other, new TypeReference>() { }); diff --git a/escheduler-api/src/main/java/cn/escheduler/api/utils/CheckUtils.java b/escheduler-api/src/main/java/cn/escheduler/api/utils/CheckUtils.java index 6a8c627d7b..f6330b79de 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/utils/CheckUtils.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/utils/CheckUtils.java @@ -18,8 +18,10 @@ package cn.escheduler.api.utils; import cn.escheduler.api.enums.Status; +import cn.escheduler.common.enums.ResUploadType; import cn.escheduler.common.task.AbstractParameters; import cn.escheduler.common.utils.JSONUtils; +import cn.escheduler.common.utils.PropertyUtils; import cn.escheduler.common.utils.TaskParametersUtils; import org.apache.commons.lang.StringUtils; @@ -28,6 +30,7 @@ import java.util.HashMap; import java.util.Map; import java.util.regex.Pattern; +import static cn.escheduler.common.utils.PropertyUtils.getBoolean; /** @@ -158,4 +161,15 @@ public class CheckUtils { return pattern.matcher(str).matches(); } + /** + * if upload resource is HDFS and kerberos startup is true , else false + * @return + */ + public static boolean getKerberosStartupState(){ + String resUploadStartupType = PropertyUtils.getString(cn.escheduler.common.Constants.RES_UPLOAD_STARTUP_TYPE); + ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); + Boolean kerberosStartupState = getBoolean(cn.escheduler.common.Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE); + return resUploadType == ResUploadType.HDFS && kerberosStartupState; + } + } diff --git a/escheduler-api/src/main/java/cn/escheduler/api/utils/Constants.java b/escheduler-api/src/main/java/cn/escheduler/api/utils/Constants.java index 79cf3e5b3f..1dfe3ac470 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/utils/Constants.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/utils/Constants.java @@ -111,6 +111,7 @@ public class Constants { public static final String ADDRESS = "address"; public static final String DATABASE = "database"; public static final String JDBC_URL = "jdbcUrl"; + public static final String PRINCIPAL = "principal"; public static final String USER = "user"; public static final String PASSWORD = "password"; public static final String OTHER = "other"; diff --git a/escheduler-common/src/main/java/cn/escheduler/common/job/db/BaseDataSource.java b/escheduler-common/src/main/java/cn/escheduler/common/job/db/BaseDataSource.java index af0624091a..f215d3e8c9 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/job/db/BaseDataSource.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/job/db/BaseDataSource.java @@ -45,6 +45,18 @@ public abstract class BaseDataSource { */ private String other; + /** + * principal + */ + private String principal; + + public String getPrincipal() { + return principal; + } + + public void setPrincipal(String principal) { + this.principal = principal; + } /** * test whether the data source can be connected successfully * @throws Exception @@ -73,14 +85,14 @@ public abstract class BaseDataSource { this.password = password; } - public String getAddress() { - return address; - } - public void setAddress(String address) { this.address = address; } + public String getAddress() { + return address; + } + public String getDatabase() { return database; } diff --git a/escheduler-common/src/main/java/cn/escheduler/common/job/db/HiveDataSource.java b/escheduler-common/src/main/java/cn/escheduler/common/job/db/HiveDataSource.java index 28e37991d7..719c5eb300 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/job/db/HiveDataSource.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/job/db/HiveDataSource.java @@ -17,12 +17,12 @@ package cn.escheduler.common.job.db; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; +import java.sql.*; /** * data source of hive @@ -32,6 +32,8 @@ public class HiveDataSource extends BaseDataSource { private static final Logger logger = LoggerFactory.getLogger(HiveDataSource.class); + + /** * gets the JDBC url for the data source connection * @return @@ -43,7 +45,7 @@ public class HiveDataSource extends BaseDataSource { jdbcUrl += "/"; } - jdbcUrl += getDatabase(); + jdbcUrl += getDatabase() + ";principal=" + getPrincipal(); if (StringUtils.isNotEmpty(getOther())) { jdbcUrl += ";" + getOther(); @@ -67,11 +69,10 @@ public class HiveDataSource extends BaseDataSource { try { con.close(); } catch (SQLException e) { - logger.error("Postgre datasource try conn close conn error", e); + logger.error("hive datasource try conn close conn error", e); throw e; } } } - } } diff --git a/escheduler-common/src/main/java/cn/escheduler/common/job/db/SparkDataSource.java b/escheduler-common/src/main/java/cn/escheduler/common/job/db/SparkDataSource.java index d9a24eef22..13aa06eaae 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/job/db/SparkDataSource.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/job/db/SparkDataSource.java @@ -31,7 +31,6 @@ public class SparkDataSource extends BaseDataSource { private static final Logger logger = LoggerFactory.getLogger(SparkDataSource.class); - /** * gets the JDBC url for the data source connection * @return @@ -43,7 +42,7 @@ public class SparkDataSource extends BaseDataSource { jdbcUrl += "/"; } - jdbcUrl += getDatabase(); + jdbcUrl += getDatabase() + ";principal=" + getPrincipal(); if (StringUtils.isNotEmpty(getOther())) { jdbcUrl += ";" + getOther(); diff --git a/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue b/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue index 8b09570862..cc6d25e7de 100644 --- a/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue +++ b/escheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue @@ -63,6 +63,17 @@ + + + + +
+ {{$t('select tenant')}} + +
超时告警 @@ -73,6 +77,7 @@ import mLocalParams from '../formModel/tasks/_source/localParams' import disabledState from '@/module/mixin/disabledState' import Affirm from '../jumpAffirm' + import FormTenant from "./_source/selectTenant"; export default { name: 'udp', @@ -90,6 +95,8 @@ syncDefine: true, // Timeout alarm timeout: 0, + + tenantId: -1, // checked Timeout alarm checkedTimeout: true } @@ -116,6 +123,7 @@ this.store.commit('dag/setGlobalParams', _.cloneDeep(this.udpList)) this.store.commit('dag/setName', _.cloneDeep(this.name)) this.store.commit('dag/setTimeout', _.cloneDeep(this.timeout)) + this.store.commit('dag/setTenantId', _.cloneDeep(this.tenantId)) this.store.commit('dag/setDesc', _.cloneDeep(this.desc)) this.store.commit('dag/setSyncDefine', this.syncDefine) }, @@ -181,9 +189,10 @@ this.syncDefine = dag.syncDefine this.timeout = dag.timeout || 0 this.checkedTimeout = this.timeout !== 0 + this.tenantId = dag.tenantId || -1 }, mounted () {}, - components: { mLocalParams } + components: {FormTenant, mLocalParams } } diff --git a/escheduler-ui/src/js/conf/home/pages/dag/definitionDetails.vue b/escheduler-ui/src/js/conf/home/pages/dag/definitionDetails.vue index e377ebfa7f..c8849d594e 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/definitionDetails.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/definitionDetails.vue @@ -26,7 +26,7 @@ methods: { ...mapMutations('dag', ['resetParams', 'setIsDetails']), ...mapActions('dag', ['getProcessList', 'getResourcesList', 'getProcessDetails']), - ...mapActions('security', ['getWorkerGroupsAll']), + ...mapActions('security', ['getTenantList','getWorkerGroupsAll']), /** * init */ @@ -43,7 +43,8 @@ // get resource this.getResourcesList(), // get worker group list - this.getWorkerGroupsAll() + this.getWorkerGroupsAll(), + this.getTenantList() ]).then((data) => { let item = data[0] this.setIsDetails(item.releaseState === 'ONLINE') diff --git a/escheduler-ui/src/js/conf/home/pages/dag/index.vue b/escheduler-ui/src/js/conf/home/pages/dag/index.vue index b9aee791dd..7a3429f3b1 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/index.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/index.vue @@ -25,7 +25,7 @@ methods: { ...mapMutations('dag', ['resetParams']), ...mapActions('dag', ['getProcessList', 'getResourcesList']), - ...mapActions('security', ['getWorkerGroupsAll']), + ...mapActions('security', ['getTenantList','getWorkerGroupsAll']), /** * init */ @@ -40,7 +40,8 @@ // get resource this.getResourcesList(), // get worker group list - this.getWorkerGroupsAll() + this.getWorkerGroupsAll(), + this.getTenantList() ]).then((data) => { this.isLoading = false // Whether to pop up the box? @@ -65,4 +66,4 @@ }, components: { mDag, mSpin } } - \ No newline at end of file + diff --git a/escheduler-ui/src/js/conf/home/pages/dag/instanceDetails.vue b/escheduler-ui/src/js/conf/home/pages/dag/instanceDetails.vue index 3e6c49c1f6..cbe220ad36 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/instanceDetails.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/instanceDetails.vue @@ -26,7 +26,7 @@ methods: { ...mapMutations('dag', ['setIsDetails', 'resetParams']), ...mapActions('dag', ['getProcessList', 'getResourcesList', 'getInstancedetail']), - ...mapActions('security', ['getWorkerGroupsAll']), + ...mapActions('security', ['getTenantList','getWorkerGroupsAll']), /** * init */ @@ -43,7 +43,8 @@ // get resources this.getResourcesList(), // get worker group list - this.getWorkerGroupsAll() + this.getWorkerGroupsAll(), + this.getTenantList() ]).then((data) => { let item = data[0] let flag = false @@ -92,4 +93,4 @@ }, components: { mDag, mSpin, mVariable } } - \ No newline at end of file + diff --git a/escheduler-ui/src/js/conf/home/store/dag/actions.js b/escheduler-ui/src/js/conf/home/store/dag/actions.js index e41e4be760..e756cfdff4 100644 --- a/escheduler-ui/src/js/conf/home/store/dag/actions.js +++ b/escheduler-ui/src/js/conf/home/store/dag/actions.js @@ -115,6 +115,7 @@ export default { // timeout state.timeout = processDefinitionJson.timeout + state.tenantId = processDefinitionJson.tenantId resolve(res.data) }).catch(res => { reject(res) @@ -146,6 +147,8 @@ export default { // timeout state.timeout = processInstanceJson.timeout + state.tenantId = processInstanceJson.tenantId + resolve(res.data) }).catch(res => { reject(res) @@ -160,6 +163,7 @@ export default { let data = { globalParams: state.globalParams, tasks: state.tasks, + tenantId: state.tenantId, timeout: state.timeout } io.post(`projects/${state.projectName}/process/save`, { @@ -183,6 +187,7 @@ export default { let data = { globalParams: state.globalParams, tasks: state.tasks, + tenantId: state.tenantId, timeout: state.timeout } io.post(`projects/${state.projectName}/process/update`, { @@ -207,6 +212,7 @@ export default { let data = { globalParams: state.globalParams, tasks: state.tasks, + tenantId: state.tenantId, timeout: state.timeout } io.post(`projects/${state.projectName}/instance/update`, { diff --git a/escheduler-ui/src/js/conf/home/store/dag/mutations.js b/escheduler-ui/src/js/conf/home/store/dag/mutations.js index 15f87fab30..d3386bc76a 100644 --- a/escheduler-ui/src/js/conf/home/store/dag/mutations.js +++ b/escheduler-ui/src/js/conf/home/store/dag/mutations.js @@ -58,6 +58,12 @@ export default { setTimeout (state, payload) { state.timeout = payload }, + /** + * set tenantId + */ + setTenantId (state, payload) { + state.tenantId = payload + }, /** * set global params */ @@ -100,6 +106,7 @@ export default { state.name = payload && payload.name || '' state.desc = payload && payload.desc || '' state.timeout = payload && payload.timeout || 0 + state.tenantId = payload && payload.tenantId || -1 state.processListS = payload && payload.processListS || [] state.resourcesListS = payload && payload.resourcesListS || [] state.isDetails = payload && payload.isDetails || false diff --git a/escheduler-ui/src/js/conf/home/store/dag/state.js b/escheduler-ui/src/js/conf/home/store/dag/state.js index 9679893e60..37df7b8b86 100644 --- a/escheduler-ui/src/js/conf/home/store/dag/state.js +++ b/escheduler-ui/src/js/conf/home/store/dag/state.js @@ -31,6 +31,8 @@ export default { tasks: [], // Timeout alarm timeout: 0, + // tenant id + tenantId:-1, // Node location information locations: {}, // Node-to-node connection diff --git a/escheduler-ui/src/js/conf/home/store/security/actions.js b/escheduler-ui/src/js/conf/home/store/security/actions.js index 9fda7663ab..ff96adccf9 100644 --- a/escheduler-ui/src/js/conf/home/store/security/actions.js +++ b/escheduler-ui/src/js/conf/home/store/security/actions.js @@ -240,7 +240,13 @@ export default { getTenantList ({ state }, payload) { return new Promise((resolve, reject) => { io.get(`tenant/list`, payload, res => { - resolve(res.data) + let list=res.data + list.unshift({ + id: -1, + tenantName: 'Default' + }) + state.tenantAllList = list + resolve(list) }).catch(e => { reject(e) }) diff --git a/escheduler-ui/src/js/conf/home/store/security/state.js b/escheduler-ui/src/js/conf/home/store/security/state.js index be52d7838c..cbb67a1823 100644 --- a/escheduler-ui/src/js/conf/home/store/security/state.js +++ b/escheduler-ui/src/js/conf/home/store/security/state.js @@ -15,5 +15,6 @@ * limitations under the License. */ export default { - workerGroupsListAll: [] + workerGroupsListAll: [], + tenantAllList : [] } diff --git a/escheduler-ui/src/js/module/i18n/locale/en_US.js b/escheduler-ui/src/js/module/i18n/locale/en_US.js index cf81d9c95c..933b6be99d 100644 --- a/escheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/escheduler-ui/src/js/module/i18n/locale/en_US.js @@ -465,4 +465,5 @@ export default { 'task number of ready to kill': '待杀死任务数', 'Statistics manage': 'Statistics manage', 'statistics': 'statistics', + 'select tenant':'select tenant', } diff --git a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js index 78ef8323aa..a5ab26dfcb 100644 --- a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -466,4 +466,5 @@ export default { 'task number of ready to kill': '待杀死任务数', 'Statistics manage': '统计管理', 'statistics': '统计', + 'select tenant':'选择租户', } From ec7406ddababf2858ccfc76607dca34c43f9a18c Mon Sep 17 00:00:00 2001 From: lenboo Date: Mon, 1 Jul 2019 10:14:04 +0800 Subject: [PATCH 63/68] add tenant id in process definition --- .../src/main/java/cn/escheduler/dao/ProcessDao.java | 6 +----- .../cn/escheduler/dao/model/ProcessInstance.java | 9 ++++++++- .../server/worker/runner/FetchTaskThread.java | 12 ++++++++---- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java index 387c61635a..2aa59c7013 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/ProcessDao.java @@ -494,11 +494,7 @@ public class ProcessDao extends AbstractBaseDao { processInstance.setProcessInstancePriority(command.getProcessInstancePriority()); processInstance.setWorkerGroupId(command.getWorkerGroupId()); processInstance.setTimeout(processDefinition.getTimeout()); - Tenant tenant = getTenantForProcess(processDefinition.getTenantId(), - processDefinition.getUserId()); - if(tenant!= null){ - processInstance.setTenantCode(tenant.getTenantCode()); - } + processInstance.setTenantId(processDefinition.getTenantId()); return processInstance; } diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/model/ProcessInstance.java b/escheduler-dao/src/main/java/cn/escheduler/dao/model/ProcessInstance.java index 0519118404..24902c0121 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/model/ProcessInstance.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/model/ProcessInstance.java @@ -192,7 +192,7 @@ public class ProcessInstance { /** * tenant id */ - private String tenantId; + private int tenantId; public ProcessInstance(){ @@ -553,4 +553,11 @@ public class ProcessInstance { '}'; } + public void setTenantId(int tenantId) { + this.tenantId = tenantId; + } + + public int getTenantId() { + return this.tenantId ; + } } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java index 3ecafde57a..e741966f06 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/runner/FetchTaskThread.java @@ -23,10 +23,7 @@ import cn.escheduler.common.thread.ThreadUtils; import cn.escheduler.common.utils.FileUtils; import cn.escheduler.common.utils.OSUtils; import cn.escheduler.dao.ProcessDao; -import cn.escheduler.dao.model.ProcessDefinition; -import cn.escheduler.dao.model.ProcessInstance; -import cn.escheduler.dao.model.TaskInstance; -import cn.escheduler.dao.model.WorkerGroup; +import cn.escheduler.dao.model.*; import cn.escheduler.server.zk.ZKWorkerClient; import com.cronutils.utils.StringUtils; import org.apache.commons.configuration.Configuration; @@ -194,9 +191,16 @@ public class FetchTaskThread implements Runnable{ // get process instance ProcessInstance processInstance = processDao.findProcessInstanceDetailById(taskInstance.getProcessInstanceId()); + // get process define ProcessDefinition processDefine = processDao.findProcessDefineById(taskInstance.getProcessDefinitionId()); + Tenant tenant = processDao.getTenantForProcess(processInstance.getTenantId(), + processDefine.getUserId()); + + if(tenant != null){ + processInstance.setTenantCode(tenant.getTenantCode()); + } taskInstance.setProcessInstance(processInstance); taskInstance.setProcessDefine(processDefine); From 8feb245e7259cc46bc9d93c33ca2a575d607898f Mon Sep 17 00:00:00 2001 From: huyuanming Date: Mon, 1 Jul 2019 18:08:08 +0800 Subject: [PATCH 64/68] the email input --- .../pages/definition/pages/list/_source/email.vue | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/email.vue b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/email.vue index cc6cb57646..f5c38f9a12 100644 --- a/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/email.vue +++ b/escheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/email.vue @@ -32,9 +32,12 @@ v-model="email" :disabled="disabled" :placeholder="$t('Please enter email')" + @blur="_emailEnter" @keydown.tab="_emailTab" @keyup.delete="_emailDelete" @keyup.enter="_emailEnter" + @keyup.space="_emailEnter" + @keyup.186="_emailEnter" @keyup.up="_emailKeyup('up')" @keyup.down="_emailKeyup('down')"> @@ -78,6 +81,11 @@ * Manually add a mailbox */ _manualEmail () { + if (this.email === '') { + return + } + this.email = _.trim(this.email).replace(/(;$)|(；$)/g, "") + let email = this.email let is = (n) => { From 867048892a61a84bccbe68f2f10970bfac7aa404 Mon Sep 17 00:00:00 2001 From: lenboo Date: Mon, 1 Jul 2019 19:00:07 +0800 Subject: [PATCH 65/68] remove the auth for query list. --- .../main/java/cn/escheduler/api/service/TenantService.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java index 35bb35a2b4..694266d4ad 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/TenantService.java @@ -125,9 +125,9 @@ public class TenantService extends BaseService{ public Map queryTenantList(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { Map result = new HashMap<>(5); - if (checkAdmin(loginUser, result)) { - return result; - } +// if (checkAdmin(loginUser, result)) { +// return result; +// } Integer count = tenantMapper.countTenantPaging(searchVal); From d07916e1b6f1bd03a461be00a67684b8865a0077 Mon Sep 17 00:00:00 2001 From: huyuanming Date: Tue, 2 Jul 2019 09:54:48 +0800 Subject: [PATCH 66/68] enter key event of search input, the width of the mr project type input --- .../src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue | 2 +- .../pages/projects/pages/_source/instanceConditions/index.vue | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue index 14b50a5fe4..3787d13541 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue @@ -3,7 +3,7 @@
{{$t('Program Type')}}
- +
- +
- +
From 2deb22223406f0d8b7f6dc3f2a31d7e24a53457a Mon Sep 17 00:00:00 2001 From: lenboo Date: Tue, 2 Jul 2019 10:57:33 +0800 Subject: [PATCH 67/68] add tenant_id field in mysql --- .../1.1.0_schema/mysql/escheduler_ddl.sql | 41 +++++++++++++++++++ .../1.1.0_schema/mysql/escheduler_dml.sql | 1 + 2 files changed, 42 insertions(+) create mode 100644 sql/upgrade/1.1.0_schema/mysql/escheduler_ddl.sql create mode 100644 sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql diff --git a/sql/upgrade/1.1.0_schema/mysql/escheduler_ddl.sql b/sql/upgrade/1.1.0_schema/mysql/escheduler_ddl.sql new file mode 100644 index 0000000000..c43b3d86b0 --- /dev/null +++ b/sql/upgrade/1.1.0_schema/mysql/escheduler_ddl.sql @@ -0,0 +1,41 @@ +SET sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY','')); + +-- ac_escheduler_T_t_escheduler_process_definition_C_tenant_id +drop PROCEDURE if EXISTS ac_escheduler_T_t_escheduler_process_definition_C_tenant_id; +delimiter d// +CREATE PROCEDURE ac_escheduler_T_t_escheduler_process_definition_C_tenant_id() + BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_escheduler_process_definition' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME='tenant_id') + THEN + ALTER TABLE `t_escheduler_process_definition` ADD COLUMN `tenant_id` int(11) NOT NULL DEFAULT -1 COMMENT '租户id' AFTER `timeout`; + END IF; + END; + +d// + +delimiter ; +CALL ac_escheduler_T_t_escheduler_process_definition_C_tenant_id; +DROP PROCEDURE ac_escheduler_T_t_escheduler_process_definition_C_tenant_id; + +-- ac_escheduler_T_t_escheduler_process_instance_C_tenant_id +drop PROCEDURE if EXISTS ac_escheduler_T_t_escheduler_process_instance_C_tenant_id; +delimiter d// +CREATE PROCEDURE ac_escheduler_T_t_escheduler_process_instance_C_tenant_id() + BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_escheduler_process_instance' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME='tenant_id') + THEN + ALTER TABLE `t_escheduler_process_instance` ADD COLUMN `tenant_id` int(11) NOT NULL DEFAULT -1 COMMENT '租户id' AFTER `timeout`; + END IF; + END; + +d// + +delimiter ; +CALL ac_escheduler_T_t_escheduler_process_instance_C_tenant_id; +DROP PROCEDURE ac_escheduler_T_t_escheduler_process_instance_C_tenant_id; diff --git a/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql b/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql new file mode 100644 index 0000000000..97da34884c --- /dev/null +++ b/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql @@ -0,0 +1 @@ +INSERT INTO `t_escheduler_version` (`version`) VALUES ('1.0.2'); \ No newline at end of file From 50e36b5ecec8982cb0a081cef2d808390355b3f3 Mon Sep 17 00:00:00 2001 From: lenboo Date: Tue, 2 Jul 2019 10:59:35 +0800 Subject: [PATCH 68/68] remove the auth for query list. --- sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql b/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql index 97da34884c..4ddd9ddcdc 100644 --- a/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql +++ b/sql/upgrade/1.1.0_schema/mysql/escheduler_dml.sql @@ -1 +1 @@ -INSERT INTO `t_escheduler_version` (`version`) VALUES ('1.0.2'); \ No newline at end of file +INSERT INTO `t_escheduler_version` (`version`) VALUES ('1.1.0'); \ No newline at end of file