diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannelFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannelFactory.java index 01b9060204..2f7afee07e 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannelFactory.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannelFactory.java @@ -71,6 +71,39 @@ public final class DingTalkAlertChannelFactory implements AlertChannelFactory { .setRequired(false) .build()) .build(); + + RadioParam msgTypeParam = RadioParam + .newBuilder(DingTalkParamsConstants.NAME_DING_TALK_MSG_TYPE, DingTalkParamsConstants.DING_TALK_MSG_TYPE) + .addParamsOptions(new ParamsOptions(DingTalkParamsConstants.DING_TALK_MSG_TYPE_TEXT, DingTalkParamsConstants.DING_TALK_MSG_TYPE_TEXT, false)) + .addParamsOptions(new ParamsOptions(DingTalkParamsConstants.DING_TALK_MSG_TYPE_MARKDOWN, DingTalkParamsConstants.DING_TALK_MSG_TYPE_MARKDOWN, false)) + .setValue(DingTalkParamsConstants.DING_TALK_MSG_TYPE_TEXT) + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .build(); + + InputParam atMobilesParam = InputParam + .newBuilder(DingTalkParamsConstants.NAME_DING_TALK_AT_MOBILES, DingTalkParamsConstants.DING_TALK_AT_MOBILES) + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .build(); + InputParam atUserIdsParam = InputParam + .newBuilder(DingTalkParamsConstants.NAME_DING_TALK_AT_USERIDS, DingTalkParamsConstants.DING_TALK_AT_USERIDS) + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .build(); + RadioParam isAtAll = RadioParam + .newBuilder(DingTalkParamsConstants.NAME_DING_TALK_AT_ALL, DingTalkParamsConstants.DING_TALK_AT_ALL) + .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false)) + .addParamsOptions(new ParamsOptions(STRING_NO, STRING_FALSE, false)) + .setValue(STRING_FALSE) + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .build(); + RadioParam isEnableProxy = RadioParam .newBuilder(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE, DingTalkParamsConstants.DING_TALK_PROXY_ENABLE) .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false)) @@ -105,7 +138,7 @@ public final class DingTalkAlertChannelFactory implements AlertChannelFactory { .setPlaceholder("if enable use authentication, you need input password") .build(); - return Arrays.asList(webHookParam, keywordParam, secretParam, isEnableProxy, proxyParam, portParam, userParam, passwordParam); + return Arrays.asList(webHookParam, keywordParam, secretParam, msgTypeParam, atMobilesParam, atUserIdsParam, isAtAll, isEnableProxy, proxyParam, portParam, userParam, passwordParam); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkParamsConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkParamsConstants.java index fa47b0c87b..e5b667ae30 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkParamsConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkParamsConstants.java @@ -18,25 +18,43 @@ package org.apache.dolphinscheduler.plugin.alert.dingtalk; public final class DingTalkParamsConstants { - - static final String DING_TALK_PROXY_ENABLE = "isEnableProxy"; + static final String DING_TALK_PROXY_ENABLE = "$t('isEnableProxy')"; static final String NAME_DING_TALK_PROXY_ENABLE = "IsEnableProxy"; - static final String DING_TALK_WEB_HOOK = "webhook"; + static final String DING_TALK_WEB_HOOK = "$t('webhook')"; static final String NAME_DING_TALK_WEB_HOOK = "WebHook"; - static final String DING_TALK_KEYWORD = "keyword"; + + static final String DING_TALK_KEYWORD = "$t('keyword')"; static final String NAME_DING_TALK_KEYWORD = "Keyword"; - static final String DING_TALK_SECRET = "secret"; + static final String DING_TALK_SECRET = "$t('secret')"; static final String NAME_DING_TALK_SECRET = "Secret"; - static final String DING_TALK_PROXY = "proxy"; + static final String DING_TALK_MSG_TYPE = "$t('msgType')"; + static final String NAME_DING_TALK_MSG_TYPE = "MsgType"; + + static final String DING_TALK_MSG_TYPE_TEXT = "text"; + static final String DING_TALK_MSG_TYPE_MARKDOWN = "markdown"; + + static final String DING_TALK_AT_MOBILES = "$t('atMobiles')"; + static final String NAME_DING_TALK_AT_MOBILES = "AtMobiles"; + + static final String DING_TALK_AT_USERIDS = "$t('atUserIds')"; + static final String NAME_DING_TALK_AT_USERIDS = "AtUserIds"; + + static final String DING_TALK_AT_ALL = "$t('isAtAll')"; + static final String NAME_DING_TALK_AT_ALL = "IsAtAll"; + + static final String DING_TALK_PROXY = "$t('proxy')"; static final String NAME_DING_TALK_PROXY = "Proxy"; - static final String DING_TALK_PORT = "port"; + + static final String DING_TALK_PORT = "$t('port')"; static final String NAME_DING_TALK_PORT = "Port"; - static final String DING_TALK_USER = "user"; + + static final String DING_TALK_USER = "$t('user')"; static final String NAME_DING_TALK_USER = "User"; - static final String DING_TALK_PASSWORD = "password"; + + static final String DING_TALK_PASSWORD = "$t('password')"; static final String NAME_DING_TALK_PASSWORD = "Password"; private DingTalkParamsConstants() { diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java index eeca8c03c3..8519e5468c 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java @@ -39,8 +39,10 @@ import org.apache.http.util.EntityUtils; import java.io.IOException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; @@ -61,6 +63,12 @@ public final class DingTalkSender { private final String url; private final String keyword; private final String secret; + private String msgType; + + private final String atMobiles; + private final String atUserIds; + private final Boolean atAll; + private final Boolean enableProxy; private String proxy; @@ -75,11 +83,17 @@ public final class DingTalkSender { url = config.get(DingTalkParamsConstants.NAME_DING_TALK_WEB_HOOK); keyword = config.get(DingTalkParamsConstants.NAME_DING_TALK_KEYWORD); secret = config.get(DingTalkParamsConstants.NAME_DING_TALK_SECRET); + msgType = config.get(DingTalkParamsConstants.NAME_DING_TALK_MSG_TYPE); + + atMobiles = config.get(DingTalkParamsConstants.NAME_DING_TALK_AT_MOBILES); + atUserIds = config.get(DingTalkParamsConstants.NAME_DING_TALK_AT_USERIDS); + atAll = Boolean.valueOf(config.get(DingTalkParamsConstants.NAME_DING_TALK_AT_ALL)); + enableProxy = Boolean.valueOf(config.get(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE)); if (Boolean.TRUE.equals(enableProxy)) { port = Integer.parseInt(config.get(DingTalkParamsConstants.NAME_DING_TALK_PORT)); proxy = config.get(DingTalkParamsConstants.NAME_DING_TALK_PROXY); - user = config.get(DingTalkParamsConstants.DING_TALK_USER); + user = config.get(DingTalkParamsConstants.NAME_DING_TALK_USER); password = config.get(DingTalkParamsConstants.NAME_DING_TALK_PASSWORD); } } @@ -108,18 +122,7 @@ public final class DingTalkSender { return RequestConfig.custom().setProxy(httpProxy).build(); } - private static String textToJsonString(String text) { - Map items = new HashMap<>(); - items.put("msgtype", "text"); - Map textContent = new HashMap<>(); - byte[] byt = StringUtils.getBytesUtf8(text); - String txt = StringUtils.newStringUtf8(byt); - textContent.put("content", txt); - items.put("text", textContent); - return JSONUtils.toJsonString(items); - } - - private static AlertResult checkSendDingTalkSendMsgResult(String result) { + private AlertResult checkSendDingTalkSendMsgResult(String result) { AlertResult alertResult = new AlertResult(); alertResult.setStatus("false"); @@ -144,6 +147,13 @@ public final class DingTalkSender { return alertResult; } + /** + * send dingtalk msg handler + * + * @param title title + * @param content content + * @return + */ public AlertResult sendDingTalkMsg(String title, String content) { AlertResult alertResult; try { @@ -160,18 +170,9 @@ public final class DingTalkSender { private String sendMsg(String title, String content) throws IOException { - StringBuilder text = new StringBuilder(); - if (org.apache.dolphinscheduler.spi.utils.StringUtils.isNotBlank(keyword)) { - text.append(keyword); - text.append(":"); - } - text.append(title); - text.append("\n"); - text.append(content); - - String msgToJson = textToJsonString(text.toString()); + String msg = generateMsgJson(title, content); - HttpPost httpPost = constructHttpPost(org.apache.dolphinscheduler.spi.utils.StringUtils.isBlank(secret) ? url : generateSignedUrl(), msgToJson); + HttpPost httpPost = constructHttpPost(org.apache.dolphinscheduler.spi.utils.StringUtils.isBlank(secret) ? url : generateSignedUrl(), msg); CloseableHttpClient httpClient; if (Boolean.TRUE.equals(enableProxy)) { @@ -192,13 +193,119 @@ public final class DingTalkSender { } finally { response.close(); } - logger.info("Ding Talk send title :{},content : {}, resp: {}", title, content, resp); + logger.info("Ding Talk send msg :{}, resp: {}", msg, resp); return resp; } finally { httpClient.close(); } } + /** + * generate msg json + * + * @param title title + * @param content content + * @return msg + */ + private String generateMsgJson(String title, String content) { + if (org.apache.dolphinscheduler.spi.utils.StringUtils.isBlank(msgType)) { + msgType = DingTalkParamsConstants.DING_TALK_MSG_TYPE_TEXT; + } + Map items = new HashMap<>(); + items.put("msgtype", msgType); + Map text = new HashMap<>(); + items.put(msgType, text); + + if (DingTalkParamsConstants.DING_TALK_MSG_TYPE_MARKDOWN.equals(msgType)) { + generateMarkdownMsg(title, content, text); + } else { + generateTextMsg(title, content, text); + } + + setMsgAt(items); + return JSONUtils.toJsonString(items); + + } + + /** + * generate text msg + * + * @param title title + * @param content content + * @param text text + */ + private void generateTextMsg(String title, String content, Map text) { + StringBuilder builder = new StringBuilder(title); + builder.append("\n"); + builder.append(content); + if (org.apache.dolphinscheduler.spi.utils.StringUtils.isNotBlank(keyword)) { + builder.append(" "); + builder.append(keyword); + } + byte[] byt = StringUtils.getBytesUtf8(builder.toString()); + String txt = StringUtils.newStringUtf8(byt); + text.put("content", txt); + } + + /** + * generate markdown msg + * + * @param title title + * @param content content + * @param text text + */ + private void generateMarkdownMsg(String title, String content, Map text) { + StringBuilder builder = new StringBuilder(content); + if (org.apache.dolphinscheduler.spi.utils.StringUtils.isNotBlank(keyword)) { + builder.append(" "); + builder.append(keyword); + } + builder.append("\n\n"); + if (org.apache.dolphinscheduler.spi.utils.StringUtils.isNotBlank(atMobiles)) { + Arrays.stream(atMobiles.split(",")).forEach(value -> { + builder.append("@"); + builder.append(value); + builder.append(" "); + }); + } + if (org.apache.dolphinscheduler.spi.utils.StringUtils.isNotBlank(atUserIds)) { + Arrays.stream(atUserIds.split(",")).forEach(value -> { + builder.append("@"); + builder.append(value); + builder.append(" "); + }); + } + + byte[] byt = StringUtils.getBytesUtf8(builder.toString()); + String txt = StringUtils.newStringUtf8(byt); + text.put("title", title); + text.put("text", txt); + } + + /** + * configure msg @person + * + * @param items items + */ + private void setMsgAt(Map items) { + Map at = new HashMap<>(); + + String[] atMobileArray = org.apache.dolphinscheduler.spi.utils.StringUtils.isNotBlank(atMobiles) ? atMobiles.split(",") : new String[0]; + String[] atUserArray = org.apache.dolphinscheduler.spi.utils.StringUtils.isNotBlank(atUserIds) ? atUserIds.split(",") : new String[0]; + boolean isAtAll = Objects.isNull(atAll) ? false : atAll; + + at.put("atMobiles", atMobileArray); + at.put("atUserIds", atUserArray); + at.put("isAtAll", isAtAll); + + items.put("at", at); + } + + /** + * generate sign url + * + * @return sign url + */ private String generateSignedUrl() { Long timestamp = System.currentTimeMillis(); String stringToSign = timestamp + "\n" + secret; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannelFactoryTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannelFactoryTest.java index 99051952e5..ea1cd1a0b9 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannelFactoryTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannelFactoryTest.java @@ -32,7 +32,7 @@ public class DingTalkAlertChannelFactoryTest { DingTalkAlertChannelFactory dingTalkAlertChannelFactory = new DingTalkAlertChannelFactory(); List params = dingTalkAlertChannelFactory.params(); JSONUtils.toJsonString(params); - Assert.assertEquals(8, params.size()); + Assert.assertEquals(12, params.size()); } @Test diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java index d2267a4919..791a96f2f5 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java @@ -33,8 +33,10 @@ public class DingTalkSenderTest { @Before public void initDingTalkConfig() { - dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_KEYWORD, "keyWord"); + dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_KEYWORD, "keyword"); dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_WEB_HOOK, "url"); + dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_MSG_TYPE, DingTalkParamsConstants.DING_TALK_MSG_TYPE_MARKDOWN); + dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE, "false"); dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_PASSWORD, "password"); dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_PORT, "9988"); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index 755d697026..4e7abf36b0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -19,8 +19,10 @@ package org.apache.dolphinscheduler.api.service; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.ReleaseState; +import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog; import org.apache.dolphinscheduler.dao.entity.User; +import java.util.List; import java.util.Map; import javax.servlet.http.HttpServletResponse; @@ -245,7 +247,7 @@ public interface ProcessDefinitionService { * @param processTaskRelationJson process task relation json * @return check result code */ - Map checkProcessNodeList(String processTaskRelationJson); + Map checkProcessNodeList(String processTaskRelationJson, List taskDefinitionLogs); /** * get task node details based on process definition diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index 58c8ab428f..ad38475a44 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -76,6 +76,7 @@ import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.fs.Stat; import java.io.BufferedOutputStream; import java.io.IOException; @@ -1071,7 +1072,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro * @return check result code */ @Override - public Map checkProcessNodeList(String processTaskRelationJson) { + public Map checkProcessNodeList(String processTaskRelationJson, List taskDefinitionLogsList) { Map result = new HashMap<>(); try { if (processTaskRelationJson == null) { @@ -1082,7 +1083,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro List taskRelationList = JSONUtils.toList(processTaskRelationJson, ProcessTaskRelation.class); // Check whether the task node is normal - List taskNodes = processService.transformTask(taskRelationList, Lists.newArrayList()); + List taskNodes = processService.transformTask(taskRelationList, taskDefinitionLogsList); if (CollectionUtils.isEmpty(taskNodes)) { logger.error("process node info is empty"); @@ -1110,8 +1111,9 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro } putMsg(result, Status.SUCCESS); } catch (Exception e) { - result.put(Constants.STATUS, Status.REQUEST_PARAMS_NOT_VALID_ERROR); - result.put(Constants.MSG, e.getMessage()); + result.put(Constants.STATUS, Status.INTERNAL_SERVER_ERROR_ARGS); + putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS, e.getMessage()); + logger.error(Status.INTERNAL_SERVER_ERROR_ARGS.getMsg(), e); } return result; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java index 57171c7089..5189616602 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java @@ -506,7 +506,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce ProcessDefinition processDefinition = processDefineMapper.queryByCode(processInstance.getProcessDefinitionCode()); List taskRelationList = JSONUtils.toList(taskRelationJson, ProcessTaskRelationLog.class); //check workflow json is valid - result = processDefinitionService.checkProcessNodeList(taskRelationJson); + result = processDefinitionService.checkProcessNodeList(taskRelationJson, taskDefinitionLogs); if (result.get(Constants.STATUS) != Status.SUCCESS) { return result; } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java index 497c3d3382..5d7e87ac13 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java @@ -30,11 +30,13 @@ import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.graph.DAG; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.DagData; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.Schedule; +import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; @@ -82,6 +84,14 @@ public class ProcessDefinitionServiceTest { + "\"postTaskVersion\":1,\"conditionType\":0,\"conditionParams\":\"{}\"},{\"name\":\"\",\"preTaskCode\":123456789," + "\"preTaskVersion\":1,\"postTaskCode\":123451234,\"postTaskVersion\":1,\"conditionType\":0,\"conditionParams\":\"{}\"}]"; + private static final String taskDefinitionJson = "[{\"code\":123456789,\"name\":\"test1\",\"version\":1,\"description\":\"\",\"delayTime\":0,\"taskType\":\"SHELL\"," + + "\"taskParams\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo 1\",\"dependence\":{},\"conditionResult\":{\"successNode\":[],\"failedNode\":[]},\"waitStartTimeout\":{}," + + "\"switchResult\":{}},\"flag\":\"YES\",\"taskPriority\":\"MEDIUM\",\"workerGroup\":\"default\",\"failRetryTimes\":0,\"failRetryInterval\":1,\"timeoutFlag\":\"CLOSE\"," + + "\"timeoutNotifyStrategy\":null,\"timeout\":0,\"environmentCode\":-1},{\"code\":123451234,\"name\":\"test2\",\"version\":1,\"description\":\"\",\"delayTime\":0,\"taskType\":\"SHELL\"," + + "\"taskParams\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo 2\",\"dependence\":{},\"conditionResult\":{\"successNode\":[],\"failedNode\":[]},\"waitStartTimeout\":{}," + + "\"switchResult\":{}},\"flag\":\"YES\",\"taskPriority\":\"MEDIUM\",\"workerGroup\":\"default\",\"failRetryTimes\":0,\"failRetryInterval\":1,\"timeoutFlag\":\"CLOSE\"," + + "\"timeoutNotifyStrategy\":\"WARN\",\"timeout\":0,\"environmentCode\":-1}]"; + @InjectMocks private ProcessDefinitionServiceImpl processDefinitionService; @@ -483,10 +493,12 @@ public class ProcessDefinitionServiceTest { @Test public void testCheckProcessNodeList() { - Map dataNotValidRes = processDefinitionService.checkProcessNodeList(null); + Map dataNotValidRes = processDefinitionService.checkProcessNodeList(null, null); Assert.assertEquals(Status.DATA_IS_NOT_VALID, dataNotValidRes.get(Constants.STATUS)); - Map taskEmptyRes = processDefinitionService.checkProcessNodeList(taskRelationJson); + List taskDefinitionLogs = JSONUtils.toList(taskDefinitionJson, TaskDefinitionLog.class); + + Map taskEmptyRes = processDefinitionService.checkProcessNodeList(taskRelationJson, taskDefinitionLogs); Assert.assertEquals(Status.PROCESS_DAG_IS_EMPTY, taskEmptyRes.get(Constants.STATUS)); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java index 207462172c..248ca68608 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java @@ -36,11 +36,13 @@ import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.TaskDefinition; +import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.User; @@ -128,6 +130,17 @@ public class ProcessInstanceServiceTest { + ":[\"\"],\"failedNode\":[\"\"]},\"dependence\":{}},\"flag\":\"NORMAL\",\"taskPriority\":\"MEDIUM\",\"workerGroup\":\"default\"," + "\"failRetryTimes\":\"0\",\"failRetryInterval\":\"1\",\"timeoutFlag\":\"CLOSE\",\"timeoutNotifyStrategy\":\"\",\"timeout\":null,\"delayTime\":\"0\"}]"; + private String taskRelationJson = "[{\"name\":\"\",\"preTaskCode\":4254865123776,\"preTaskVersion\":1,\"postTaskCode\":4254862762304,\"postTaskVersion\":1,\"conditionType\":0," + + "\"conditionParams\":{}},{\"name\":\"\",\"preTaskCode\":0,\"preTaskVersion\":0,\"postTaskCode\":4254865123776,\"postTaskVersion\":1,\"conditionType\":0,\"conditionParams\":{}}]"; + + private String taskDefinitionJson = "[{\"code\":4254862762304,\"name\":\"test1\",\"version\":1,\"description\":\"\",\"delayTime\":0,\"taskType\":\"SHELL\",\"taskParams\":{\"resourceList\":[]," + + "\"localParams\":[],\"rawScript\":\"echo 1\",\"dependence\":{},\"conditionResult\":{\"successNode\":[],\"failedNode\":[]},\"waitStartTimeout\":{},\"switchResult\":{}},\"flag\":\"YES\"," + + "\"taskPriority\":\"MEDIUM\",\"workerGroup\":\"default\",\"failRetryTimes\":0,\"failRetryInterval\":1,\"timeoutFlag\":\"CLOSE\",\"timeoutNotifyStrategy\":null,\"timeout\":0," + + "\"environmentCode\":-1},{\"code\":4254865123776,\"name\":\"test2\",\"version\":1,\"description\":\"\",\"delayTime\":0,\"taskType\":\"SHELL\",\"taskParams\":{\"resourceList\":[]," + + "\"localParams\":[],\"rawScript\":\"echo 2\",\"dependence\":{},\"conditionResult\":{\"successNode\":[],\"failedNode\":[]},\"waitStartTimeout\":{},\"switchResult\":{}},\"flag\":\"YES\"," + + "\"taskPriority\":\"MEDIUM\",\"workerGroup\":\"default\",\"failRetryTimes\":0,\"failRetryInterval\":1,\"timeoutFlag\":\"CLOSE\",\"timeoutNotifyStrategy\":\"WARN\",\"timeout\":0," + + "\"environmentCode\":-1}]"; + @Test public void testQueryProcessInstanceList() { long projectCode = 1L; @@ -425,10 +438,12 @@ public class ProcessInstanceServiceTest { when(processService.getTenantForProcess(Mockito.anyInt(), Mockito.anyInt())).thenReturn(tenant); when(processService.updateProcessInstance(processInstance)).thenReturn(1); when(processService.saveProcessDefine(loginUser, processDefinition, Boolean.TRUE, Boolean.FALSE)).thenReturn(1); - when(processDefinitionService.checkProcessNodeList(shellJson)).thenReturn(result); + + List taskDefinitionLogs = JSONUtils.toList(taskDefinitionJson, TaskDefinitionLog.class); + when(processDefinitionService.checkProcessNodeList(taskRelationJson, taskDefinitionLogs)).thenReturn(result); putMsg(result, Status.SUCCESS, projectCode); Map processInstanceFinishRes = processInstanceService.updateProcessInstance(loginUser, projectCode, 1, - shellJson, taskJson,"2020-02-21 00:00:00", true, "", "", 0, "root"); + taskRelationJson, taskDefinitionJson,"2020-02-21 00:00:00", true, "", "", 0, "root"); Assert.assertEquals(Status.SUCCESS, processInstanceFinishRes.get(Constants.STATUS)); //success @@ -437,7 +452,7 @@ public class ProcessInstanceServiceTest { when(processService.saveProcessDefine(loginUser, processDefinition, Boolean.FALSE, Boolean.FALSE)).thenReturn(1); Map successRes = processInstanceService.updateProcessInstance(loginUser, projectCode, 1, - shellJson, taskJson,"2020-02-21 00:00:00", Boolean.FALSE, "", "", 0, "root"); + taskRelationJson, taskDefinitionJson,"2020-02-21 00:00:00", Boolean.FALSE, "", "", 0, "root"); Assert.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS)); } @@ -638,5 +653,4 @@ public class ProcessInstanceServiceTest { result.put(Constants.MSG, status.getMsg()); } } - -} \ No newline at end of file +} diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/AbstractCycle.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/AbstractCycle.java index 60c862340b..b00f1476ab 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/AbstractCycle.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/AbstractCycle.java @@ -173,6 +173,16 @@ public abstract class AbstractCycle { FieldExpression dayOfWeekFieldExpression = dayOfWeekField.getExpression(); return (dayOfWeekFieldExpression instanceof Every || dayOfWeekFieldExpression instanceof Always); } + + /** + * whether the year field has a value of every or always + * + * @return if year field has a value of every or always return true，else return false + */ + protected boolean yearFieldIsEvery() { + FieldExpression yearFieldExpression = yearField.getExpression(); + return (yearFieldExpression instanceof Every || yearFieldExpression instanceof Always); + } /** * get cycle enum diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtils.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtils.java index d7847229ed..49810cd1f4 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtils.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtils.java @@ -22,6 +22,7 @@ import static org.apache.dolphinscheduler.service.quartz.cron.CycleFactory.hour; import static org.apache.dolphinscheduler.service.quartz.cron.CycleFactory.min; import static org.apache.dolphinscheduler.service.quartz.cron.CycleFactory.month; import static org.apache.dolphinscheduler.service.quartz.cron.CycleFactory.week; +import static org.apache.dolphinscheduler.service.quartz.cron.CycleFactory.year; import static com.cronutils.model.CronType.QUARTZ; @@ -90,7 +91,7 @@ public class CronUtils { * @return CycleEnum */ public static CycleEnum getMaxCycle(Cron cron) { - return min(cron).addCycle(hour(cron)).addCycle(day(cron)).addCycle(week(cron)).addCycle(month(cron)).getCycle(); + return min(cron).addCycle(hour(cron)).addCycle(day(cron)).addCycle(week(cron)).addCycle(month(cron)).addCycle(year(cron)).getCycle(); } /** @@ -100,7 +101,7 @@ public class CronUtils { * @return CycleEnum */ public static CycleEnum getMiniCycle(Cron cron) { - return min(cron).addCycle(hour(cron)).addCycle(day(cron)).addCycle(week(cron)).addCycle(month(cron)).getMiniCycle(); + return min(cron).addCycle(hour(cron)).addCycle(day(cron)).addCycle(week(cron)).addCycle(month(cron)).addCycle(year(cron)).getMiniCycle(); } /** @@ -186,7 +187,7 @@ public class CronUtils { */ public static List getSelfFireDateList(final Date startTime, final Date endTime, final List schedules) { List result = new ArrayList<>(); - if(startTime.equals(endTime)){ + if (startTime.equals(endTime)) { result.add(startTime); return result; } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CycleFactory.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CycleFactory.java index 1f807dce7f..9f931d20bb 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CycleFactory.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CycleFactory.java @@ -72,6 +72,15 @@ public class CycleFactory { public static AbstractCycle month(Cron cron) { return new MonthCycle(cron); } + + /** + * year + * @param cron cron + * @return AbstractCycle + */ + public static AbstractCycle year(Cron cron) { + return new YearCycle(cron); + } /** * day cycle @@ -275,4 +284,52 @@ public class CycleFactory { return null; } } + + /** + * year cycle + */ + public static class YearCycle extends AbstractCycle { + public YearCycle(Cron cron) { + super(cron); + } + + /** + * get cycle + * @return CycleEnum + */ + @Override + protected CycleEnum getCycle() { + boolean flag = (minFiledIsSetAll() + && hourFiledIsSetAll() + && dayOfMonthFieldIsSetAll() + && dayOfWeekField.getExpression() instanceof QuestionMark + && monthFieldIsSetAll()) + && yearFieldIsEvery() || + (minFiledIsSetAll() + && hourFiledIsSetAll() + && dayOfMonthField.getExpression() instanceof QuestionMark + && dayofWeekFieldIsSetAll() + && monthFieldIsSetAll() + && yearFieldIsEvery()); + + if (flag) { + return CycleEnum.YEAR; + } + + return null; + } + + /** + * get mini cycle + * @return CycleEnum + */ + @Override + protected CycleEnum getMiniCycle() { + if (yearFieldIsEvery()) { + return CycleEnum.YEAR; + } + + return null; + } + } } diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtilsTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtilsTest.java index 55cc19d338..4fbcd8f9c0 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtilsTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtilsTest.java @@ -95,6 +95,20 @@ public class CronUtilsTest { CycleEnum cycleEnum3 = CronUtils.getMiniCycle(CronUtils.parse2Cron("0 * * * * ? *")); Assert.assertEquals("MINUTE", cycleEnum3.name()); + + CycleEnum cycleEnum4 = CronUtils.getMaxCycle(CronUtils.parse2Cron("0 0 7 * 1 ? *")); + Assert.assertEquals("YEAR", cycleEnum4.name()); + cycleEnum4 = CronUtils.getMiniCycle(CronUtils.parse2Cron("0 0 7 * 1 ? *")); + Assert.assertEquals("DAY", cycleEnum4.name()); + + CycleEnum cycleEnum5 = CronUtils.getMaxCycle(CronUtils.parse2Cron("0 0 7 * 1/1 ? *")); + Assert.assertEquals("MONTH", cycleEnum5.name()); + + CycleEnum cycleEnum6 = CronUtils.getMaxCycle(CronUtils.parse2Cron("0 0 7 * 1-2 ? *")); + Assert.assertEquals("YEAR", cycleEnum6.name()); + + CycleEnum cycleEnum7 = CronUtils.getMaxCycle(CronUtils.parse2Cron("0 0 7 * 1,2 ? *")); + Assert.assertEquals("YEAR", cycleEnum7.name()); } /** @@ -113,7 +127,7 @@ public class CronUtilsTest { .instance(); // minute cycle String[] cronArayy = new String[]{"* * * * * ? *","* 0 * * * ? *", - "* 5 * * 3/5 ? *","0 0 * * * ? *"}; + "* 5 * * 3/5 ? *","0 0 * * * ? *", "0 0 7 * 1 ? *", "0 0 7 * 1/1 ? *", "0 0 7 * 1-2 ? *" , "0 0 7 * 1,2 ? *"}; for(String minCrontab:cronArayy){ if (!org.quartz.CronExpression.isValidExpression(minCrontab)) { throw new RuntimeException(minCrontab+" verify failure, cron expression not valid"); @@ -155,6 +169,14 @@ public class CronUtilsTest { logger.info("dayOfWeekField instanceof On:"+(dayOfWeekField.getExpression() instanceof On)); logger.info("dayOfWeekField instanceof And:"+(dayOfWeekField.getExpression() instanceof And)); logger.info("dayOfWeekField instanceof QuestionMark:"+(dayOfWeekField.getExpression() instanceof QuestionMark)); + + CronField yearField = cron.retrieve(CronFieldName.YEAR); + logger.info("yearField instanceof Between:"+(yearField.getExpression() instanceof Between)); + logger.info("yearField instanceof Always:"+(yearField.getExpression() instanceof Always)); + logger.info("yearField instanceof Every:"+(yearField.getExpression() instanceof Every)); + logger.info("yearField instanceof On:"+(yearField.getExpression() instanceof On)); + logger.info("yearField instanceof And:"+(yearField.getExpression() instanceof And)); + logger.info("yearField instanceof QuestionMark:"+(yearField.getExpression() instanceof QuestionMark)); CycleEnum cycleEnum = CronUtils.getMaxCycle(minCrontab); if(cycleEnum !=null){ @@ -204,4 +226,4 @@ public class CronUtilsTest { expirationTime = CronUtils.getExpirationTime(startTime, CycleEnum.YEAR); Assert.assertEquals("2020-02-07 18:30:00", DateUtils.dateToString(expirationTime)); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java index 805d03529d..4112e2ca7b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java @@ -122,10 +122,12 @@ public abstract class AbstractCommandExecutor { // merge error information to standard output stream processBuilder.redirectErrorStream(true); - // setting up user to run commands - command.add("sudo"); - command.add("-u"); - command.add(taskRequest.getTenantCode()); + // if sudo.enable=true,setting up user to run commands + if (OSUtils.isSudoEnable()) { + command.add("sudo"); + command.add("-u"); + command.add(taskRequest.getTenantCode()); + } command.add(commandInterpreter()); command.addAll(Collections.emptyList()); command.add(commandFile); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/util/OSUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/util/OSUtils.java index e8c66a7cf1..f34268d179 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/util/OSUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/util/OSUtils.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.task.util; import org.apache.dolphinscheduler.plugin.task.api.ShellExecutor; +import org.apache.dolphinscheduler.spi.utils.PropertyUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; import java.io.IOException; @@ -40,6 +41,15 @@ public class OSUtils { return StringUtils.isEmpty(tenantCode) ? command : "sudo -u " + tenantCode + " " + command; } + /** + * use sudo or not + * + * @return true is use sudo + */ + public static boolean isSudoEnable() { + return PropertyUtils.getBoolean("sudo.enable", Boolean.TRUE); + } + /** * whether is macOS * diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java index c4edc4eab0..42ba975272 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.task.datax; import org.apache.dolphinscheduler.spi.enums.DbType; +import com.alibaba.druid.sql.dialect.clickhouse.parser.ClickhouseStatementParser; import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser; import com.alibaba.druid.sql.dialect.postgresql.parser.PGSQLStatementParser; @@ -91,6 +92,8 @@ public class DataxUtils { return new OracleStatementParser(sql); case SQLSERVER: return new SQLServerStatementParser(sql); + case CLICKHOUSE: + return new ClickhouseStatementParser(sql); default: return null; } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue index c79a59fb31..bac2aa1ce1 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue @@ -330,11 +330,11 @@ }) if (this.type === 'instance') { this.$router.push({ - path: `/projects/${this.projectCode}/instance/list` + path: `/projects/${this.projectCode}/instance/list/${methodParam}` }) } else { this.$router.push({ - path: `/projects/${this.projectCode}/definition/list` + path: `/projects/${this.projectCode}/definition/list/${methodParam}` }) } }) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/_source/list.vue index ab68e5a2c2..98ab71355a 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/_source/list.vue @@ -51,8 +51,11 @@ {{scope.row.updateTime | formatDate}} - + - +