From 3a7731a4551f97db71ab57c0e476da010f19fdb0 Mon Sep 17 00:00:00 2001 From: chendapao Date: Mon, 27 May 2019 11:34:24 +0800 Subject: [PATCH 1/8] this W/M --- .../common/utils/DependentUtils.java | 6 ++++++ .../utils/dependent/DependentDateUtils.java | 20 +++++++++++++++++++ .../common/utils/DependentUtilsTest.java | 20 +++++++++++++++++++ .../formModel/tasks/_source/commcon.js | 8 ++++++++ .../src/js/module/i18n/locale/en_US.js | 2 ++ .../src/js/module/i18n/locale/zh_CN.js | 2 ++ 6 files changed, 58 insertions(+) diff --git a/escheduler-common/src/main/java/cn/escheduler/common/utils/DependentUtils.java b/escheduler-common/src/main/java/cn/escheduler/common/utils/DependentUtils.java index 4e589c37fa..0b4f566ae5 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/utils/DependentUtils.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/utils/DependentUtils.java @@ -95,6 +95,9 @@ public class DependentUtils { case "last7Days": result = DependentDateUtils.getLastDayInterval(businessDate, 7); break; + case "thisWeek": + result = DependentDateUtils.getThisWeekInterval(businessDate); + break; case "lastWeek": result = DependentDateUtils.getLastWeekInterval(businessDate); break; @@ -119,6 +122,9 @@ public class DependentUtils { case "lastSunday": result = DependentDateUtils.getLastWeekOneDayInterval(businessDate, 7); break; + case "thisMonth": + result = DependentDateUtils.getThisMonthInterval(businessDate); + break; case "lastMonth": result = DependentDateUtils.getLastMonthInterval(businessDate); break; diff --git a/escheduler-common/src/main/java/cn/escheduler/common/utils/dependent/DependentDateUtils.java b/escheduler-common/src/main/java/cn/escheduler/common/utils/dependent/DependentDateUtils.java index df79dd163a..0127fe2ca9 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/utils/dependent/DependentDateUtils.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/utils/dependent/DependentDateUtils.java @@ -76,6 +76,16 @@ public class DependentDateUtils { return dateIntervals; } + /** + * get interval between this month first day and businessDate + * @param businessDate + * @return + */ + public static List getThisMonthInterval(Date businessDate) { + Date firstDay = DateUtils.getFirstDayOfMonth(businessDate); + return getDateIntervalListBetweenTwoDates(firstDay, businessDate); + } + /** * get interval between last month first day and last day * @param businessDate @@ -108,6 +118,16 @@ public class DependentDateUtils { } } + /** + * get interval between monday to businessDate of this week + * @param businessDate + * @return + */ + public static List getThisWeekInterval(Date businessDate) { + Date mondayThisWeek = DateUtils.getMonday(businessDate); + return getDateIntervalListBetweenTwoDates(mondayThisWeek, businessDate); + } + /** * get interval between monday to sunday of last week * default set monday the first day of week diff --git a/escheduler-common/src/test/java/cn/escheduler/common/utils/DependentUtilsTest.java b/escheduler-common/src/test/java/cn/escheduler/common/utils/DependentUtilsTest.java index a0569b02ee..8680ab0cb4 100644 --- a/escheduler-common/src/test/java/cn/escheduler/common/utils/DependentUtilsTest.java +++ b/escheduler-common/src/test/java/cn/escheduler/common/utils/DependentUtilsTest.java @@ -80,6 +80,26 @@ public class DependentUtilsTest { Assert.assertEquals(dateIntervals.get(0), diCur); + dateValue = "thisWeek"; + Date firstWeekDay = DateUtils.getMonday(curDay); + dateIntervals = DependentUtils.getDateIntervalList(curDay, dateValue); + + DateInterval weekHead = new DateInterval(DateUtils.getStartOfDay(firstWeekDay), DateUtils.getEndOfDay(firstWeekDay)); + DateInterval weekThis = new DateInterval(DateUtils.getStartOfDay(curDay), DateUtils.getEndOfDay(curDay)); + + Assert.assertEquals(dateIntervals.get(0), weekHead); + Assert.assertEquals(dateIntervals.get(dateIntervals.size() - 1), weekThis); + + + dateValue = "thisMonth"; + Date firstMonthDay = DateUtils.getFirstDayOfMonth(curDay); + dateIntervals = DependentUtils.getDateIntervalList(curDay, dateValue); + + DateInterval monthHead = new DateInterval(DateUtils.getStartOfDay(firstMonthDay), DateUtils.getEndOfDay(firstMonthDay)); + DateInterval monthThis = new DateInterval(DateUtils.getStartOfDay(curDay), DateUtils.getEndOfDay(curDay)); + + Assert.assertEquals(dateIntervals.get(0), monthHead); + Assert.assertEquals(dateIntervals.get(dateIntervals.size() - 1), monthThis); } diff --git a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/commcon.js b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/commcon.js index 6274533c1a..1897940064 100644 --- a/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/commcon.js +++ b/escheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/commcon.js @@ -64,6 +64,10 @@ const dateValueList = { } ], 'week': [ + { + value: 'thisWeek', + label: `${i18n.$t('ThisWeek')}` + }, { value: 'lastWeek', label: `${i18n.$t('LastWeek')}` @@ -98,6 +102,10 @@ const dateValueList = { } ], 'month': [ + { + value: 'thisMonth', + label: `${i18n.$t('ThisMonth')}` + }, { value: 'lastMonth', label: `${i18n.$t('LastMonth')}` diff --git a/escheduler-ui/src/js/module/i18n/locale/en_US.js b/escheduler-ui/src/js/module/i18n/locale/en_US.js index 50c33c061b..fbc621e18b 100644 --- a/escheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/escheduler-ui/src/js/module/i18n/locale/en_US.js @@ -395,6 +395,7 @@ export default { 'Last2Days': 'Last2Days', 'Last3Days': 'Last3Days', 'Last7Days': 'Last7Days', + 'ThisWeek': 'ThisWeek', 'LastWeek': 'LastWeek', 'LastMonday': 'LastMonday', 'LastTuesday': 'LastTuesday', @@ -403,6 +404,7 @@ export default { 'LastFriday': 'LastFriday', 'LastSaturday': 'LastSaturday', 'LastSunday': 'LastSunday', + 'ThisMonth': 'ThisMonth', 'LastMonth': 'LastMonth', 'LastMonthBegin': 'LastMonthBegin', 'LastMonthEnd': 'LastMonthEnd', diff --git a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js index d8ea823048..d77c55715a 100644 --- a/escheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/escheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -395,6 +395,7 @@ export default { 'Last2Days': '前两天', 'Last3Days': '前三天', 'Last7Days': '前七天', + 'ThisWeek': '本周', 'LastWeek': '上周', 'LastMonday': '上周一', 'LastTuesday': '上周二', @@ -403,6 +404,7 @@ export default { 'LastFriday': '上周五', 'LastSaturday': '上周六', 'LastSunday': '上周日', + 'ThisMonth': '本月', 'LastMonth': '上月', 'LastMonthBegin': '上月初', 'LastMonthEnd': '上月末', From ce6c53eb126561ae2ed41f9f9f63ae8a9b222702 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Thu, 30 May 2019 11:53:52 +0800 Subject: [PATCH 2/8] update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 96cef561d7..23fbd6ee7d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Easy Scheduler +Easy Scheduler ============ [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) From 0714f04d2a65623129c5d07f311197dc2a2f1ece Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Fri, 31 May 2019 11:58:33 +0800 Subject: [PATCH 3/8] update deleteJob --- .../src/main/java/cn/escheduler/api/quartz/QuartzExecutors.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/quartz/QuartzExecutors.java b/escheduler-api/src/main/java/cn/escheduler/api/quartz/QuartzExecutors.java index 5cb3b33853..916e4f3792 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/quartz/QuartzExecutors.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/quartz/QuartzExecutors.java @@ -230,6 +230,8 @@ public class QuartzExecutors { if(scheduler.checkExists(jobKey)){ logger.info("try to delete job, job name: {}, job group name: {},", jobName, jobGroupName); return scheduler.deleteJob(jobKey); + }else { + return true; } } catch (SchedulerException e) { From 5e89fc6844aade8393058a52d73ff85f998a6daf Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Mon, 3 Jun 2019 16:58:57 +0800 Subject: [PATCH 4/8] if hdfs path is not exists return result --- .../main/java/cn/escheduler/api/service/ResourcesService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java index 291c59ea22..64b81d5ebc 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java @@ -643,6 +643,7 @@ public class ResourcesService extends BaseService { logger.error("{} is not exist", resourcePath); result.setCode(Status.HDFS_OPERATION_ERROR.getCode()); result.setMsg(String.format("%s is not exist", resourcePath)); + return result; } } catch (Exception e) { logger.error(e.getMessage(), e); From b1a0cf3e7fc366503ad555d41c96b5268d9458f1 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Tue, 4 Jun 2019 11:37:43 +0800 Subject: [PATCH 5/8] email address trim and shell error does not determine yarn status --- .../server/worker/task/AbstractCommandExecutor.java | 8 +------- .../cn/escheduler/server/worker/task/sql/SqlTask.java | 6 +++--- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/AbstractCommandExecutor.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/AbstractCommandExecutor.java index 9e617e65c0..10ef9f72ce 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/AbstractCommandExecutor.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/AbstractCommandExecutor.java @@ -213,7 +213,7 @@ public abstract class AbstractCommandExecutor { */ private int updateState(ProcessDao processDao, int exitStatusCode, int pid, int taskInstId) { //get yarn state by log - if (exitStatusCode != -1) { + if (exitStatusCode != 0) { TaskInstance taskInstance = processDao.findTaskInstanceById(taskInstId); logger.info("process id is {}", pid); @@ -548,10 +548,4 @@ public abstract class AbstractCommandExecutor { protected abstract boolean checkShowLog(String line); protected abstract boolean checkFindApp(String line); protected abstract void createCommandFileIfNotExists(String execCommand, String commandFile) throws IOException; - - - -// if(line.contains(taskAppId) || !line.contains("cn.escheduler.server.worker.log.TaskLogger")){ -// logs.add(line); -// } } diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java index 3e85b55b91..bb57f60b6a 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java @@ -347,14 +347,14 @@ public class SqlTask extends AbstractTask { // receiving group list List receviersList = new ArrayList(); for(User user:users){ - receviersList.add(user.getEmail()); + receviersList.add(user.getEmail().trim()); } // custom receiver String receivers = sqlParameters.getReceivers(); if (StringUtils.isNotEmpty(receivers)){ String[] splits = receivers.split(Constants.COMMA); for (String receiver : splits){ - receviersList.add(receiver); + receviersList.add(receiver.trim()); } } @@ -365,7 +365,7 @@ public class SqlTask extends AbstractTask { if (StringUtils.isNotEmpty(receiversCc)){ String[] splits = receiversCc.split(Constants.COMMA); for (String receiverCc : splits){ - receviersCcList.add(receiverCc); + receviersCcList.add(receiverCc.trim()); } } From 341333655386b5e21621a5176a6b1eac6be482a1 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Wed, 5 Jun 2019 16:32:36 +0800 Subject: [PATCH 6/8] kill task need kill sub pids --- .../cn/escheduler/common/utils/OSUtils.java | 2 +- .../escheduler/server/utils/ProcessUtils.java | 23 ++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/escheduler-common/src/main/java/cn/escheduler/common/utils/OSUtils.java b/escheduler-common/src/main/java/cn/escheduler/common/utils/OSUtils.java index d8dcc621a8..0cf06d3b02 100644 --- a/escheduler-common/src/main/java/cn/escheduler/common/utils/OSUtils.java +++ b/escheduler-common/src/main/java/cn/escheduler/common/utils/OSUtils.java @@ -220,7 +220,7 @@ public class OSUtils { * @throws IOException */ public static String exeShell(String command) throws IOException { - return ShellExecutor.execCommand("groups"); + return ShellExecutor.execCommand(command); } /** diff --git a/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java b/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java index baf82de0df..5530e87e14 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java @@ -18,6 +18,7 @@ package cn.escheduler.server.utils; import cn.escheduler.common.Constants; import cn.escheduler.common.utils.CommonUtils; +import cn.escheduler.common.utils.OSUtils; import cn.escheduler.dao.model.TaskInstance; import cn.escheduler.server.rpc.LogClient; import org.apache.commons.io.FileUtils; @@ -33,6 +34,7 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; + /** * mainly used to get the start command line of a process */ @@ -139,6 +141,8 @@ public class ProcessUtils { {' ', '\t', '<', '>'}, {' ', '\t'}}; + private static Matcher matcher; + private static String createCommandLine(int verificationType, final String executablePath, final String[] cmd) { StringBuilder cmdbuf = new StringBuilder(80); @@ -256,7 +260,7 @@ public class ProcessUtils { return ; } - String cmd = String.format("sudo kill -9 %d", processId); + String cmd = String.format("sudo kill -9 %s", getPidsStr(processId)); logger.info("process id:{}, cmd:{}", processId, cmd); @@ -270,6 +274,23 @@ public class ProcessUtils { } } + /** + * get pids str + * @param processId + * @return + * @throws Exception + */ + private static String getPidsStr(int processId)throws Exception{ + StringBuilder sb = new StringBuilder(); + // pstree -p pid get sub pids + String pids = OSUtils.exeCmd("pstree -p " +processId+ ""); + Matcher mat = Pattern.compile("(\\d+)").matcher(pids); + while (mat.find()){ + sb.append(mat.group()+" "); + } + return sb.toString().trim(); + } + /** * find logs and kill yarn tasks * @param taskInstance From b997746e6257be4236e8451d17d8bf9e4737d990 Mon Sep 17 00:00:00 2001 From: qiaozhanwei <825193156@qq.com> Date: Wed, 5 Jun 2019 17:26:43 +0800 Subject: [PATCH 7/8] kill pid need kill sub process pids update --- .../src/main/java/cn/escheduler/server/utils/ProcessUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java b/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java index 5530e87e14..0c3aec602b 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/utils/ProcessUtils.java @@ -264,7 +264,7 @@ public class ProcessUtils { logger.info("process id:{}, cmd:{}", processId, cmd); - Runtime.getRuntime().exec(cmd); + OSUtils.exeCmd(cmd); // find log and kill yarn job killYarnJob(taskInstance); From 76662d31daa35c4c1111ac0faa4d622a0d0f2bfb Mon Sep 17 00:00:00 2001 From: xianhu Date: Thu, 6 Jun 2019 17:28:20 +0800 Subject: [PATCH 8/8] Solved the bugs that when editted file in resource center online, don't update the time and size of the file --- .../java/cn/escheduler/api/service/ResourcesService.java | 5 +++++ .../cn/escheduler/dao/mapper/ResourceMapperProvider.java | 1 + 2 files changed, 6 insertions(+) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java index 64b81d5ebc..f1c850a87a 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java @@ -569,6 +569,7 @@ public class ResourcesService extends BaseService { * @param resourceId * @return */ + @Transactional(value = "TransactionManager",rollbackFor = Exception.class) public Result updateResourceContent(int resourceId, String content) { Result result = new Result(); @@ -597,6 +598,10 @@ public class ResourcesService extends BaseService { } } + resource.setSize(content.getBytes().length); + resource.setUpdateTime(new Date()); + resourcesMapper.update(resource); + User user = userMapper.queryDetailsById(resource.getUserId()); String tenantCode = tenantMapper.queryById(user.getTenantId()).getTenantCode(); diff --git a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ResourceMapperProvider.java b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ResourceMapperProvider.java index 4122c7722c..4314b8f584 100644 --- a/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ResourceMapperProvider.java +++ b/escheduler-dao/src/main/java/cn/escheduler/dao/mapper/ResourceMapperProvider.java @@ -118,6 +118,7 @@ public class ResourceMapperProvider { SET("`alias` = #{resource.alias}"); SET("`desc` = #{resource.desc}"); SET("`update_time` = #{resource.updateTime}"); + SET("`size` = #{resource.size}"); WHERE("`id` = #{resource.id}"); }}.toString(); }