diff --git a/docs/zh_CN/后端部署文档.md b/docs/zh_CN/后端部署文档.md index f2df8a6989..273f34dd22 100644 --- a/docs/zh_CN/后端部署文档.md +++ b/docs/zh_CN/后端部署文档.md @@ -4,7 +4,7 @@ ## 1、准备工作 -目前最新安装包版本是1.0.3，下载地址： [码云下载](https://gitee.com/easyscheduler/EasyScheduler/attach_files/) ，下载escheduler-backend-1.0.3.tar.gz(后端简称escheduler-backend),escheduler-ui-1.0.3.tar.gz(前端简称escheduler-ui) +目前最新安装包版本是1.0.4，下载地址： [码云下载](https://gitee.com/easyscheduler/EasyScheduler/attach_files/) ，下载escheduler-backend-1.0.4.tar.gz(后端简称escheduler-backend),escheduler-ui-1.0.4.tar.gz(前端简称escheduler-ui) #### 准备一: 基础软件安装(必装项请自行安装) @@ -149,7 +149,7 @@ install.sh : 一键部署脚本 ### 2.2 编译源码来部署 -将源码包release版本1.0.3下载后，解压进入根目录 +将源码包release版本1.0.4下载后，解压进入根目录 * 执行编译命令： diff --git a/escheduler-alert/src/main/java/cn/escheduler/alert/utils/MailUtils.java b/escheduler-alert/src/main/java/cn/escheduler/alert/utils/MailUtils.java index d97d9d9d57..bd918fc645 100644 --- a/escheduler-alert/src/main/java/cn/escheduler/alert/utils/MailUtils.java +++ b/escheduler-alert/src/main/java/cn/escheduler/alert/utils/MailUtils.java @@ -160,6 +160,7 @@ public class MailUtils { return retMap; }catch (Exception e){ handleException(receivers, retMap, e); + return retMap; } } return retMap; diff --git a/escheduler-api/src/main/java/cn/escheduler/api/controller/ResourcesController.java b/escheduler-api/src/main/java/cn/escheduler/api/controller/ResourcesController.java index 1574012d19..947eace7d9 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/controller/ResourcesController.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/controller/ResourcesController.java @@ -195,9 +195,9 @@ public class ResourcesController extends BaseController{ ) { try { logger.info("login user {}, verfiy resource alias: {},resource type: {}", - loginUser.getUserName(), alias); + loginUser.getUserName(), alias,type); - return resourceService.verifyResourceName(alias, type); + return resourceService.verifyResourceName(alias,type,loginUser); } catch (Exception e) { logger.error(VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR.getMsg(), e); return error(Status.VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR.getCode(), Status.VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR.getMsg()); diff --git a/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java b/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java index 6e25d91825..d165fb6c16 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/enums/Status.java @@ -174,6 +174,7 @@ public enum Status { RESOURCE_SUFFIX_FORBID_CHANGE(20008, "resource suffix not allowed to be modified"), UDF_RESOURCE_SUFFIX_NOT_JAR(20009, "UDF resource suffix name must be jar"), HDFS_COPY_FAIL(20009, "hdfs copy {0} -> {1} fail"), + RESOURCE_FILE_EXIST(20010, "resource file {0} already exists in hdfs,please delete it or change name!"), diff --git a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java index f1c850a87a..7b56d927ad 100644 --- a/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java +++ b/escheduler-api/src/main/java/cn/escheduler/api/service/ResourcesService.java @@ -420,6 +420,41 @@ public class ResourcesService extends BaseService { return result; } + /** + * verify resource by name and type + * @param name + * @param type + * @param loginUser + * @return + */ + public Result verifyResourceName(String name, ResourceType type,User loginUser) { + Result result = new Result(); + putMsg(result, Status.SUCCESS); + Resource resource = resourcesMapper.queryResourceByNameAndType(name, type.ordinal()); + if (resource != null) { + logger.error("resource type:{} name:{} has exist, can't create again.", type, name); + putMsg(result, Status.RESOURCE_EXIST); + } else { + // query tenant + String tenantCode = tenantMapper.queryById(loginUser.getTenantId()).getTenantCode(); + + try { + String hdfsFilename = getHdfsFileName(type,tenantCode,name); + if(HadoopUtils.getInstance().exists(hdfsFilename)){ + logger.error("resource type:{} name:{} has exist in hdfs {}, can't create again.", type, name,hdfsFilename); + putMsg(result, Status.RESOURCE_FILE_EXIST,hdfsFilename); + } + + } catch (Exception e) { + logger.error(e.getMessage(),e); + putMsg(result,Status.HDFS_OPERATION_ERROR); + } + } + + + return result; + } + /** * verify resource by name and type * @@ -815,6 +850,23 @@ public class ResourcesService extends BaseService { return hdfsFileName; } + /** + * get hdfs file name + * + * @param resourceType + * @param tenantCode + * @param hdfsFileName + * @return + */ + private String getHdfsFileName(ResourceType resourceType, String tenantCode, String hdfsFileName) { + if (resourceType.equals(ResourceType.FILE)) { + hdfsFileName = HadoopUtils.getHdfsFilename(tenantCode, hdfsFileName); + } else if (resourceType.equals(ResourceType.UDF)) { + hdfsFileName = HadoopUtils.getHdfsUdfFilename(tenantCode, hdfsFileName); + } + return hdfsFileName; + } + /** * get authorized resource list * diff --git a/escheduler-api/src/test/java/cn/escheduler/api/controller/ResourcesControllerTest.java b/escheduler-api/src/test/java/cn/escheduler/api/controller/ResourcesControllerTest.java index 0d4ac945dd..1dc7855186 100644 --- a/escheduler-api/src/test/java/cn/escheduler/api/controller/ResourcesControllerTest.java +++ b/escheduler-api/src/test/java/cn/escheduler/api/controller/ResourcesControllerTest.java @@ -34,6 +34,8 @@ import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.MvcResult; import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; import org.springframework.web.context.WebApplicationContext; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; @@ -43,7 +45,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @RunWith(SpringRunner.class) @SpringBootTest public class ResourcesControllerTest { - private static Logger logger = LoggerFactory.getLogger(QueueControllerTest.class); + private static Logger logger = LoggerFactory.getLogger(ResourcesControllerTest.class); private MockMvc mockMvc; @@ -71,4 +73,24 @@ public class ResourcesControllerTest { Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); } + + @Test + public void verifyResourceName() throws Exception { + + MultiValueMap paramsMap = new LinkedMultiValueMap<>(); + paramsMap.add("name","list_resources_1.sh"); + paramsMap.add("type","FILE"); + + MvcResult mvcResult = mockMvc.perform(get("/resources/verify-name") + .header("sessionId", "c24ed9d9-1c20-48a0-bd9c-5cfca14a4dcb") + .params(paramsMap)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) + .andReturn(); + + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + + Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); + logger.info(mvcResult.getResponse().getContentAsString()); + } } \ No newline at end of file diff --git a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java index bb57f60b6a..4eb567d8c8 100644 --- a/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java +++ b/escheduler-server/src/main/java/cn/escheduler/server/worker/task/sql/SqlTask.java @@ -196,7 +196,7 @@ public class SqlTask extends AbstractTask { } // special characters need to be escaped, ${} needs to be escaped - String rgex = "'?\\$\\{(.*?)\\}'?"; + String rgex = "['\"]*\\$\\{(.*?)\\}['\"]*"; setSqlParamsMap(sql,rgex,sqlParamsMap,paramsMap); // replace the ${} of the SQL statement with the Placeholder @@ -310,6 +310,7 @@ public class SqlTask extends AbstractTask { } } catch (Exception e) { logger.error(e.getMessage(),e); + throw new RuntimeException(e.getMessage()); } return connection; } @@ -326,6 +327,7 @@ public class SqlTask extends AbstractTask { ParameterUtils.setInParameter(key,stmt,prop.getType(),prop.getValue()); } } + logger.info("prepare statement replace sql:{}",stmt.toString()); return stmt; } @@ -371,9 +373,13 @@ public class SqlTask extends AbstractTask { String showTypeName = sqlParameters.getShowType().replace(Constants.COMMA,"").trim(); if(EnumUtils.isValidEnum(ShowType.class,showTypeName)){ - MailUtils.sendMails(receviersList,receviersCcList,title, content, ShowType.valueOf(showTypeName)); + Map mailResult = MailUtils.sendMails(receviersList, receviersCcList, title, content, ShowType.valueOf(showTypeName)); + if(!(Boolean) mailResult.get(cn.escheduler.api.utils.Constants.STATUS)){ + throw new RuntimeException("send mail failed!"); + } }else{ logger.error("showType: {} is not valid " ,showTypeName); + throw new RuntimeException(String.format("showType: %s is not valid ",showTypeName)); } } @@ -411,19 +417,5 @@ public class SqlTask extends AbstractTask { logPrint.append(sqlParamsMap.get(i).getValue()+"("+sqlParamsMap.get(i).getType()+")"); } logger.info(logPrint.toString()); - - //direct print style - Pattern pattern = Pattern.compile(rgex); - Matcher m = pattern.matcher(content); - int index = 1; - StringBuffer sb = new StringBuffer("replaced sql , direct:"); - while (m.find()) { - - m.appendReplacement(sb, sqlParamsMap.get(index).getValue()); - - index ++; - } - m.appendTail(sb); - logger.info(sb.toString()); } } diff --git a/sql/soft_version b/sql/soft_version index e6d5cb833c..a6a3a43c3a 100644 --- a/sql/soft_version +++ b/sql/soft_version @@ -1 +1 @@ -1.0.2 \ No newline at end of file +1.0.4 \ No newline at end of file