From 597f518ed5d49219a80ffd5e859208f94c9865a7 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Fri, 14 Jun 2019 14:30:42 +0800 Subject: [PATCH 1/9] The SQL task fails if the mail delivery fails --- .../main/java/cn/escheduler/alert/utils/MailUtils.java | 1 + .../cn/escheduler/server/worker/task/sql/SqlTask.java | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/escheduler-alert/src/main/java/cn/escheduler/alert/utils/MailUtils.java b/escheduler-alert/src/main/java/cn/escheduler/alert/utils/MailUtils.java index d97d9d9d57..bd918fc645 100644 --- a/escheduler-alert/src/main/java/cn/escheduler/alert/utils/MailUtils.java +++ b/escheduler-alert/src/main/java/cn/escheduler/alert/utils/MailUtils.java @@ -160,6 +160,7 @@ public class MailUtils { return retMap; }catch (Exception e){ handleException(receivers, retMap, e); + return retMap; } } return retMap; diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java index bb57f60b6a..b5a9dd6f50 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java @@ -17,6 +17,7 @@ package cn.escheduler.server.worker.task.sql; import cn.escheduler.alert.utils.MailUtils; +import cn.escheduler.api.enums.Status; import cn.escheduler.common.Constants; import cn.escheduler.common.enums.DbType; import cn.escheduler.common.enums.ShowType; @@ -310,6 +311,7 @@ public class SqlTask extends AbstractTask { } } catch (Exception e) { logger.error(e.getMessage(),e); + throw new RuntimeException(e.getMessage()); } return connection; } @@ -371,9 +373,14 @@ public class SqlTask extends AbstractTask { String showTypeName = sqlParameters.getShowType().replace(Constants.COMMA,"").trim(); if(EnumUtils.isValidEnum(ShowType.class,showTypeName)){ - MailUtils.sendMails(receviersList,receviersCcList,title, content, ShowType.valueOf(showTypeName)); + Map mailResult = MailUtils.sendMails(receviersList, receviersCcList, title, content, ShowType.valueOf(showTypeName)); + Status status = (Status) mailResult.get(cn.escheduler.api.utils.Constants.STATUS); + if(status != Status.SUCCESS){ + throw new RuntimeException("send mail failed!"); + } }else{ logger.error("showType: {} is not valid " ,showTypeName); + throw new RuntimeException(String.format("showType: %s is not valid ",showTypeName)); } } From c768445322b48c7dbb63a9fbe6e1ccd199e372f7 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Fri, 14 Jun 2019 17:27:32 +0800 Subject: [PATCH 2/9] The SQL task fails if the mail delivery fails --- .../java/cn/escheduler/server/worker/task/sql/SqlTask.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java index b5a9dd6f50..8b2f84c4a8 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java @@ -374,8 +374,7 @@ public class SqlTask extends AbstractTask { String showTypeName = sqlParameters.getShowType().replace(Constants.COMMA,"").trim(); if(EnumUtils.isValidEnum(ShowType.class,showTypeName)){ Map mailResult = MailUtils.sendMails(receviersList, receviersCcList, title, content, ShowType.valueOf(showTypeName)); - Status status = (Status) mailResult.get(cn.escheduler.api.utils.Constants.STATUS); - if(status != Status.SUCCESS){ + if(!(Boolean) mailResult.get(cn.escheduler.api.utils.Constants.STATUS)){ throw new RuntimeException("send mail failed!"); } }else{ From 25ff540ff2bf2e1f3ea43f988ccff5613879759f Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Wed, 19 Jun 2019 16:47:32 +0800 Subject: [PATCH 3/9] Modified SQL task custom variable replacement rules --- .../server/worker/task/sql/SqlTask.java | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java index 8b2f84c4a8..4eb567d8c8 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java @@ -17,7 +17,6 @@ package cn.escheduler.server.worker.task.sql; import cn.escheduler.alert.utils.MailUtils; -import cn.escheduler.api.enums.Status; import cn.escheduler.common.Constants; import cn.escheduler.common.enums.DbType; import cn.escheduler.common.enums.ShowType; @@ -197,7 +196,7 @@ public class SqlTask extends AbstractTask { } // special characters need to be escaped, ${} needs to be escaped - String rgex = "'?\\$\\{(.*?)\\}'?"; + String rgex = "['\"]*\\$\\{(.*?)\\}['\"]*"; setSqlParamsMap(sql,rgex,sqlParamsMap,paramsMap); // replace the ${} of the SQL statement with the Placeholder @@ -328,6 +327,7 @@ public class SqlTask extends AbstractTask { ParameterUtils.setInParameter(key,stmt,prop.getType(),prop.getValue()); } } + logger.info("prepare statement replace sql:{}",stmt.toString()); return stmt; } @@ -417,19 +417,5 @@ public class SqlTask extends AbstractTask { logPrint.append(sqlParamsMap.get(i).getValue()+"("+sqlParamsMap.get(i).getType()+")"); } logger.info(logPrint.toString()); - - //direct print style - Pattern pattern = Pattern.compile(rgex); - Matcher m = pattern.matcher(content); - int index = 1; - StringBuffer sb = new StringBuffer("replaced sql , direct:"); - while (m.find()) { - - m.appendReplacement(sb, sqlParamsMap.get(index).getValue()); - - index ++; - } - m.appendTail(sb); - logger.info(sb.toString()); } } From f72d647fb2dcf90a1e302fd390bd5e97802c3cff Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Thu, 20 Jun 2019 18:35:28 +0800 Subject: [PATCH 4/9] add RESOURCE_FILE_EXIST --- escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java | 1 + 1 file changed, 1 insertion(+) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java b/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java index 6e25d91825..d165fb6c16 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java @@ -174,6 +174,7 @@ public enum Status { RESOURCE_SUFFIX_FORBID_CHANGE(20008, "resource suffix not allowed to be modified"), UDF_RESOURCE_SUFFIX_NOT_JAR(20009, "UDF resource suffix name must be jar"), HDFS_COPY_FAIL(20009, "hdfs copy {0} -> {1} fail"), + RESOURCE_FILE_EXIST(20010, "resource file {0} already exists in hdfs,please delete it or change name!"), From 4a69289c41871cdbb827122a2ff4d725e89d430c Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Thu, 20 Jun 2019 18:42:29 +0800 Subject: [PATCH 5/9] add verify resource file is exists in hdfs --- .../api/service/ResourcesService.java | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java index f1c850a87a..7b56d927ad 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java @@ -420,6 +420,41 @@ public class ResourcesService extends BaseService { return result; } + /** + * verify resource by name and type + * @param name + * @param type + * @param loginUser + * @return + */ + public Result verifyResourceName(String name, ResourceType type,User loginUser) { + Result result = new Result(); + putMsg(result, Status.SUCCESS); + Resource resource = resourcesMapper.queryResourceByNameAndType(name, type.ordinal()); + if (resource != null) { + logger.error("resource type:{} name:{} has exist, can't create again.", type, name); + putMsg(result, Status.RESOURCE_EXIST); + } else { + // query tenant + String tenantCode = tenantMapper.queryById(loginUser.getTenantId()).getTenantCode(); + + try { + String hdfsFilename = getHdfsFileName(type,tenantCode,name); + if(HadoopUtils.getInstance().exists(hdfsFilename)){ + logger.error("resource type:{} name:{} has exist in hdfs {}, can't create again.", type, name,hdfsFilename); + putMsg(result, Status.RESOURCE_FILE_EXIST,hdfsFilename); + } + + } catch (Exception e) { + logger.error(e.getMessage(),e); + putMsg(result,Status.HDFS_OPERATION_ERROR); + } + } + + + return result; + } + /** * verify resource by name and type * @@ -815,6 +850,23 @@ public class ResourcesService extends BaseService { return hdfsFileName; } + /** + * get hdfs file name + * + * @param resourceType + * @param tenantCode + * @param hdfsFileName + * @return + */ + private String getHdfsFileName(ResourceType resourceType, String tenantCode, String hdfsFileName) { + if (resourceType.equals(ResourceType.FILE)) { + hdfsFileName = HadoopUtils.getHdfsFilename(tenantCode, hdfsFileName); + } else if (resourceType.equals(ResourceType.UDF)) { + hdfsFileName = HadoopUtils.getHdfsUdfFilename(tenantCode, hdfsFileName); + } + return hdfsFileName; + } + /** * get authorized resource list * From 33be5ff9dd19493fbfa99dc50f65df8b4f418caf Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Thu, 20 Jun 2019 18:44:58 +0800 Subject: [PATCH 6/9] update verifyResourceName --- .../cn/escheduler/api/controller/ResourcesController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/escheduler-api/src/main/java/cn/escheduler/api/controller/ResourcesController.java b/escheduler-api/src/main/java/cn/escheduler/api/controller/ResourcesController.java index 1574012d19..947eace7d9 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/controller/ResourcesController.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/controller/ResourcesController.java @@ -195,9 +195,9 @@ public class ResourcesController extends BaseController{ ) { try { logger.info("login user {}, verfiy resource alias: {},resource type: {}", - loginUser.getUserName(), alias); + loginUser.getUserName(), alias,type); - return resourceService.verifyResourceName(alias, type); + return resourceService.verifyResourceName(alias,type,loginUser); } catch (Exception e) { logger.error(VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR.getMsg(), e); return error(Status.VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR.getCode(), Status.VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR.getMsg()); From 48389f1822d38ddb65e3a2bef51aa1676e745a01 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Mon, 24 Jun 2019 11:49:35 +0800 Subject: [PATCH 7/9] add verifyResourceName test --- .../controller/ResourcesControllerTest.java | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/escheduler-api/src/test/java/cn/escheduler/api/controller/ResourcesControllerTest.java b/escheduler-api/src/test/java/cn/escheduler/api/controller/ResourcesControllerTest.java index 0d4ac945dd..1dc7855186 100644 --- a/escheduler-api/src/test/java/cn/escheduler/api/controller/ResourcesControllerTest.java +++ b/escheduler-api/src/test/java/cn/escheduler/api/controller/ResourcesControllerTest.java @@ -34,6 +34,8 @@ import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.MvcResult; import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; import org.springframework.web.context.WebApplicationContext; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; @@ -43,7 +45,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @RunWith(SpringRunner.class) @SpringBootTest public class ResourcesControllerTest { - private static Logger logger = LoggerFactory.getLogger(QueueControllerTest.class); + private static Logger logger = LoggerFactory.getLogger(ResourcesControllerTest.class); private MockMvc mockMvc; @@ -71,4 +73,24 @@ public class ResourcesControllerTest { Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); } + + @Test + public void verifyResourceName() throws Exception { + + MultiValueMap paramsMap = new LinkedMultiValueMap<>(); + paramsMap.add("name","list_resources_1.sh"); + paramsMap.add("type","FILE"); + + MvcResult mvcResult = mockMvc.perform(get("/resources/verify-name") + .header("sessionId", "c24ed9d9-1c20-48a0-bd9c-5cfca14a4dcb") + .params(paramsMap)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) + .andReturn(); + + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + + Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); + logger.info(mvcResult.getResponse().getContentAsString()); + } } \ No newline at end of file From cc3c7e911e6d59381d06fbd502fa390b9d7d3688 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Mon, 24 Jun 2019 13:51:31 +0800 Subject: [PATCH 8/9] change soft_version to 1.0.4 --- sql/soft_version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/soft_version b/sql/soft_version index e6d5cb833c..a6a3a43c3a 100644 --- a/sql/soft_version +++ b/sql/soft_version @@ -1 +1 @@ -1.0.2 \ No newline at end of file +1.0.4 \ No newline at end of file From 3f59730282943bb8080c54c8f614357df752b46b Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Mon, 24 Jun 2019 14:05:05 +0800 Subject: [PATCH 9/9] Update the latest version number --- docs/zh_CN/前端部署文档.md | 4 ++-- docs/zh_CN/后端部署文档.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/zh_CN/前端部署文档.md b/docs/zh_CN/前端部署文档.md index 96d1d48cf3..c5bfa2f833 100644 --- a/docs/zh_CN/前端部署文档.md +++ b/docs/zh_CN/前端部署文档.md @@ -5,9 +5,9 @@ ## 1、准备工作 #### 下载安装包 -目前最新安装包版本是1.0.2，下载地址： [码云下载](https://gitee.com/easyscheduler/EasyScheduler/attach_files/) +目前最新安装包版本是1.0.4，下载地址： [码云下载](https://gitee.com/easyscheduler/EasyScheduler/attach_files/) -下载 escheduler-ui-1.0.2.tar.gz 后，解压`tar -zxvf escheduler-ui-1.0.2.tar.gz ./`后，进入`escheduler-ui`目录 +下载 escheduler-ui-1.0.4.tar.gz 后，解压`tar -zxvf escheduler-ui-1.0.4.tar.gz ./`后，进入`escheduler-ui`目录 diff --git a/docs/zh_CN/后端部署文档.md b/docs/zh_CN/后端部署文档.md index f2df8a6989..273f34dd22 100644 --- a/docs/zh_CN/后端部署文档.md +++ b/docs/zh_CN/后端部署文档.md @@ -4,7 +4,7 @@ ## 1、准备工作 -目前最新安装包版本是1.0.3，下载地址： [码云下载](https://gitee.com/easyscheduler/EasyScheduler/attach_files/) ，下载escheduler-backend-1.0.3.tar.gz(后端简称escheduler-backend),escheduler-ui-1.0.3.tar.gz(前端简称escheduler-ui) +目前最新安装包版本是1.0.4，下载地址： [码云下载](https://gitee.com/easyscheduler/EasyScheduler/attach_files/) ，下载escheduler-backend-1.0.4.tar.gz(后端简称escheduler-backend),escheduler-ui-1.0.4.tar.gz(前端简称escheduler-ui) #### 准备一: 基础软件安装(必装项请自行安装) @@ -149,7 +149,7 @@ install.sh : 一键部署脚本 ### 2.2 编译源码来部署 -将源码包release版本1.0.3下载后，解压进入根目录 +将源码包release版本1.0.4下载后，解压进入根目录 * 执行编译命令：