Browse Source

[Cherry-pick-to-204] pick to 204-prepare (#8429)

* fix relation unbinding bug

* [Bug-8053] Fix CronUtils.getMaxCycle return null (#8086)

* [bug] Fix datax task type error parse clickhouse reader sql (#8091)

* [Fix-8119][API] Update ProcessInstance error and without reason output (#8122)

* fix bug_8119

* fix bug_8119

* fix bug_8119

* fix bug_8119

* fix bug_8119

* pick-8086/8091/8122/8139

* [Bug-8169] [dolphinscheduler-ui] Fix `After you save the workflow, it… (#8198)

* [Fix-8187][UI] Add the function to the module of resource that you can re-upload the file on the page of the file management. (#8359)

* add reupload file

* develop reupload file

* fix this issue

* [Improvement-8284][Alert] Dingtalk alert plugin supports markdown message type (#8285)

* add msgtype in the dingtalk alert plugin

* update markdown msgtype 'at persion'

* fix sudo.enable=false Is invalid (#8388)

* [Fix][UI] Rectify this issue with missing the re-uploading file button when the file is under the root directory.

Co-authored-by: springmonster <charleskuanghc@163.com>
Co-authored-by: seagle <seagle-yuan@users.noreply.github.com>
Co-authored-by: xiangzihao <460888207@qq.com>
Co-authored-by: Kerwin <37063904+zhuangchong@users.noreply.github.com>
Co-authored-by: calvin <jianghuachinacom@163.com>
Co-authored-by: ShuiMuNianHuaLP <46993277+ShuiMuNianHuaLP@users.noreply.github.com>
2.0.7-release
JinYong Li 2 years ago committed by GitHub
parent
commit
2aa5b4c6cc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 35
      dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannelFactory.java
  2. 36
      dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkParamsConstants.java
  3. 157
      dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java
  4. 2
      dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannelFactoryTest.java
  5. 4
      dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java
  6. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java
  7. 10
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
  8. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java
  9. 16
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java
  10. 22
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java
  11. 10
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/AbstractCycle.java
  12. 7
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtils.java
  13. 57
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CycleFactory.java
  14. 24
      dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtilsTest.java
  15. 4
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java
  16. 10
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/util/OSUtils.java
  17. 3
      dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java
  18. 4
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue
  19. 10
      dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/_source/list.vue
  20. 9
      dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/_source/list.vue
  21. 82
      dolphinscheduler-ui/src/js/module/components/fileUpdate/fileReUpload.vue
  22. 46
      dolphinscheduler-ui/src/js/module/components/nav/nav.vue
  23. 4
      dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js
  24. 4
      dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js

35
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannelFactory.java

@ -71,6 +71,39 @@ public final class DingTalkAlertChannelFactory implements AlertChannelFactory {
.setRequired(false) .setRequired(false)
.build()) .build())
.build(); .build();
RadioParam msgTypeParam = RadioParam
.newBuilder(DingTalkParamsConstants.NAME_DING_TALK_MSG_TYPE, DingTalkParamsConstants.DING_TALK_MSG_TYPE)
.addParamsOptions(new ParamsOptions(DingTalkParamsConstants.DING_TALK_MSG_TYPE_TEXT, DingTalkParamsConstants.DING_TALK_MSG_TYPE_TEXT, false))
.addParamsOptions(new ParamsOptions(DingTalkParamsConstants.DING_TALK_MSG_TYPE_MARKDOWN, DingTalkParamsConstants.DING_TALK_MSG_TYPE_MARKDOWN, false))
.setValue(DingTalkParamsConstants.DING_TALK_MSG_TYPE_TEXT)
.addValidate(Validate.newBuilder()
.setRequired(false)
.build())
.build();
InputParam atMobilesParam = InputParam
.newBuilder(DingTalkParamsConstants.NAME_DING_TALK_AT_MOBILES, DingTalkParamsConstants.DING_TALK_AT_MOBILES)
.addValidate(Validate.newBuilder()
.setRequired(false)
.build())
.build();
InputParam atUserIdsParam = InputParam
.newBuilder(DingTalkParamsConstants.NAME_DING_TALK_AT_USERIDS, DingTalkParamsConstants.DING_TALK_AT_USERIDS)
.addValidate(Validate.newBuilder()
.setRequired(false)
.build())
.build();
RadioParam isAtAll = RadioParam
.newBuilder(DingTalkParamsConstants.NAME_DING_TALK_AT_ALL, DingTalkParamsConstants.DING_TALK_AT_ALL)
.addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false))
.addParamsOptions(new ParamsOptions(STRING_NO, STRING_FALSE, false))
.setValue(STRING_FALSE)
.addValidate(Validate.newBuilder()
.setRequired(false)
.build())
.build();
RadioParam isEnableProxy = RadioParam RadioParam isEnableProxy = RadioParam
.newBuilder(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE, DingTalkParamsConstants.DING_TALK_PROXY_ENABLE) .newBuilder(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE, DingTalkParamsConstants.DING_TALK_PROXY_ENABLE)
.addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false)) .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false))
@ -105,7 +138,7 @@ public final class DingTalkAlertChannelFactory implements AlertChannelFactory {
.setPlaceholder("if enable use authentication, you need input password") .setPlaceholder("if enable use authentication, you need input password")
.build(); .build();
return Arrays.asList(webHookParam, keywordParam, secretParam, isEnableProxy, proxyParam, portParam, userParam, passwordParam); return Arrays.asList(webHookParam, keywordParam, secretParam, msgTypeParam, atMobilesParam, atUserIdsParam, isAtAll, isEnableProxy, proxyParam, portParam, userParam, passwordParam);
} }
} }

36
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkParamsConstants.java

@ -18,25 +18,43 @@
package org.apache.dolphinscheduler.plugin.alert.dingtalk; package org.apache.dolphinscheduler.plugin.alert.dingtalk;
public final class DingTalkParamsConstants { public final class DingTalkParamsConstants {
static final String DING_TALK_PROXY_ENABLE = "$t('isEnableProxy')";
static final String DING_TALK_PROXY_ENABLE = "isEnableProxy";
static final String NAME_DING_TALK_PROXY_ENABLE = "IsEnableProxy"; static final String NAME_DING_TALK_PROXY_ENABLE = "IsEnableProxy";
static final String DING_TALK_WEB_HOOK = "webhook"; static final String DING_TALK_WEB_HOOK = "$t('webhook')";
static final String NAME_DING_TALK_WEB_HOOK = "WebHook"; static final String NAME_DING_TALK_WEB_HOOK = "WebHook";
static final String DING_TALK_KEYWORD = "keyword";
static final String DING_TALK_KEYWORD = "$t('keyword')";
static final String NAME_DING_TALK_KEYWORD = "Keyword"; static final String NAME_DING_TALK_KEYWORD = "Keyword";
static final String DING_TALK_SECRET = "secret"; static final String DING_TALK_SECRET = "$t('secret')";
static final String NAME_DING_TALK_SECRET = "Secret"; static final String NAME_DING_TALK_SECRET = "Secret";
static final String DING_TALK_PROXY = "proxy"; static final String DING_TALK_MSG_TYPE = "$t('msgType')";
static final String NAME_DING_TALK_MSG_TYPE = "MsgType";
static final String DING_TALK_MSG_TYPE_TEXT = "text";
static final String DING_TALK_MSG_TYPE_MARKDOWN = "markdown";
static final String DING_TALK_AT_MOBILES = "$t('atMobiles')";
static final String NAME_DING_TALK_AT_MOBILES = "AtMobiles";
static final String DING_TALK_AT_USERIDS = "$t('atUserIds')";
static final String NAME_DING_TALK_AT_USERIDS = "AtUserIds";
static final String DING_TALK_AT_ALL = "$t('isAtAll')";
static final String NAME_DING_TALK_AT_ALL = "IsAtAll";
static final String DING_TALK_PROXY = "$t('proxy')";
static final String NAME_DING_TALK_PROXY = "Proxy"; static final String NAME_DING_TALK_PROXY = "Proxy";
static final String DING_TALK_PORT = "port";
static final String DING_TALK_PORT = "$t('port')";
static final String NAME_DING_TALK_PORT = "Port"; static final String NAME_DING_TALK_PORT = "Port";
static final String DING_TALK_USER = "user";
static final String DING_TALK_USER = "$t('user')";
static final String NAME_DING_TALK_USER = "User"; static final String NAME_DING_TALK_USER = "User";
static final String DING_TALK_PASSWORD = "password";
static final String DING_TALK_PASSWORD = "$t('password')";
static final String NAME_DING_TALK_PASSWORD = "Password"; static final String NAME_DING_TALK_PASSWORD = "Password";
private DingTalkParamsConstants() { private DingTalkParamsConstants() {

157
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java

@ -39,8 +39,10 @@ import org.apache.http.util.EntityUtils;
import java.io.IOException; import java.io.IOException;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Objects;
import javax.crypto.Mac; import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec; import javax.crypto.spec.SecretKeySpec;
@ -61,6 +63,12 @@ public final class DingTalkSender {
private final String url; private final String url;
private final String keyword; private final String keyword;
private final String secret; private final String secret;
private String msgType;
private final String atMobiles;
private final String atUserIds;
private final Boolean atAll;
private final Boolean enableProxy; private final Boolean enableProxy;
private String proxy; private String proxy;
@ -75,11 +83,17 @@ public final class DingTalkSender {
url = config.get(DingTalkParamsConstants.NAME_DING_TALK_WEB_HOOK); url = config.get(DingTalkParamsConstants.NAME_DING_TALK_WEB_HOOK);
keyword = config.get(DingTalkParamsConstants.NAME_DING_TALK_KEYWORD); keyword = config.get(DingTalkParamsConstants.NAME_DING_TALK_KEYWORD);
secret = config.get(DingTalkParamsConstants.NAME_DING_TALK_SECRET); secret = config.get(DingTalkParamsConstants.NAME_DING_TALK_SECRET);
msgType = config.get(DingTalkParamsConstants.NAME_DING_TALK_MSG_TYPE);
atMobiles = config.get(DingTalkParamsConstants.NAME_DING_TALK_AT_MOBILES);
atUserIds = config.get(DingTalkParamsConstants.NAME_DING_TALK_AT_USERIDS);
atAll = Boolean.valueOf(config.get(DingTalkParamsConstants.NAME_DING_TALK_AT_ALL));
enableProxy = Boolean.valueOf(config.get(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE)); enableProxy = Boolean.valueOf(config.get(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE));
if (Boolean.TRUE.equals(enableProxy)) { if (Boolean.TRUE.equals(enableProxy)) {
port = Integer.parseInt(config.get(DingTalkParamsConstants.NAME_DING_TALK_PORT)); port = Integer.parseInt(config.get(DingTalkParamsConstants.NAME_DING_TALK_PORT));
proxy = config.get(DingTalkParamsConstants.NAME_DING_TALK_PROXY); proxy = config.get(DingTalkParamsConstants.NAME_DING_TALK_PROXY);
user = config.get(DingTalkParamsConstants.DING_TALK_USER); user = config.get(DingTalkParamsConstants.NAME_DING_TALK_USER);
password = config.get(DingTalkParamsConstants.NAME_DING_TALK_PASSWORD); password = config.get(DingTalkParamsConstants.NAME_DING_TALK_PASSWORD);
} }
} }
@ -108,18 +122,7 @@ public final class DingTalkSender {
return RequestConfig.custom().setProxy(httpProxy).build(); return RequestConfig.custom().setProxy(httpProxy).build();
} }
private static String textToJsonString(String text) { private AlertResult checkSendDingTalkSendMsgResult(String result) {
Map<String, Object> items = new HashMap<>();
items.put("msgtype", "text");
Map<String, String> textContent = new HashMap<>();
byte[] byt = StringUtils.getBytesUtf8(text);
String txt = StringUtils.newStringUtf8(byt);
textContent.put("content", txt);
items.put("text", textContent);
return JSONUtils.toJsonString(items);
}
private static AlertResult checkSendDingTalkSendMsgResult(String result) {
AlertResult alertResult = new AlertResult(); AlertResult alertResult = new AlertResult();
alertResult.setStatus("false"); alertResult.setStatus("false");
@ -144,6 +147,13 @@ public final class DingTalkSender {
return alertResult; return alertResult;
} }
/**
* send dingtalk msg handler
*
* @param title title
* @param content content
* @return
*/
public AlertResult sendDingTalkMsg(String title, String content) { public AlertResult sendDingTalkMsg(String title, String content) {
AlertResult alertResult; AlertResult alertResult;
try { try {
@ -160,18 +170,9 @@ public final class DingTalkSender {
private String sendMsg(String title, String content) throws IOException { private String sendMsg(String title, String content) throws IOException {
StringBuilder text = new StringBuilder(); String msg = generateMsgJson(title, content);
if (org.apache.dolphinscheduler.spi.utils.StringUtils.isNotBlank(keyword)) {
text.append(keyword);
text.append(":");
}
text.append(title);
text.append("\n");
text.append(content);
String msgToJson = textToJsonString(text.toString());
HttpPost httpPost = constructHttpPost(org.apache.dolphinscheduler.spi.utils.StringUtils.isBlank(secret) ? url : generateSignedUrl(), msgToJson); HttpPost httpPost = constructHttpPost(org.apache.dolphinscheduler.spi.utils.StringUtils.isBlank(secret) ? url : generateSignedUrl(), msg);
CloseableHttpClient httpClient; CloseableHttpClient httpClient;
if (Boolean.TRUE.equals(enableProxy)) { if (Boolean.TRUE.equals(enableProxy)) {
@ -192,13 +193,119 @@ public final class DingTalkSender {
} finally { } finally {
response.close(); response.close();
} }
logger.info("Ding Talk send title :{},content : {}, resp: {}", title, content, resp); logger.info("Ding Talk send msg :{}, resp: {}", msg, resp);
return resp; return resp;
} finally { } finally {
httpClient.close(); httpClient.close();
} }
} }
/**
* generate msg json
*
* @param title title
* @param content content
* @return msg
*/
private String generateMsgJson(String title, String content) {
if (org.apache.dolphinscheduler.spi.utils.StringUtils.isBlank(msgType)) {
msgType = DingTalkParamsConstants.DING_TALK_MSG_TYPE_TEXT;
}
Map<String, Object> items = new HashMap<>();
items.put("msgtype", msgType);
Map<String, Object> text = new HashMap<>();
items.put(msgType, text);
if (DingTalkParamsConstants.DING_TALK_MSG_TYPE_MARKDOWN.equals(msgType)) {
generateMarkdownMsg(title, content, text);
} else {
generateTextMsg(title, content, text);
}
setMsgAt(items);
return JSONUtils.toJsonString(items);
}
/**
* generate text msg
*
* @param title title
* @param content content
* @param text text
*/
private void generateTextMsg(String title, String content, Map<String, Object> text) {
StringBuilder builder = new StringBuilder(title);
builder.append("\n");
builder.append(content);
if (org.apache.dolphinscheduler.spi.utils.StringUtils.isNotBlank(keyword)) {
builder.append(" ");
builder.append(keyword);
}
byte[] byt = StringUtils.getBytesUtf8(builder.toString());
String txt = StringUtils.newStringUtf8(byt);
text.put("content", txt);
}
/**
* generate markdown msg
*
* @param title title
* @param content content
* @param text text
*/
private void generateMarkdownMsg(String title, String content, Map<String, Object> text) {
StringBuilder builder = new StringBuilder(content);
if (org.apache.dolphinscheduler.spi.utils.StringUtils.isNotBlank(keyword)) {
builder.append(" ");
builder.append(keyword);
}
builder.append("\n\n");
if (org.apache.dolphinscheduler.spi.utils.StringUtils.isNotBlank(atMobiles)) {
Arrays.stream(atMobiles.split(",")).forEach(value -> {
builder.append("@");
builder.append(value);
builder.append(" ");
});
}
if (org.apache.dolphinscheduler.spi.utils.StringUtils.isNotBlank(atUserIds)) {
Arrays.stream(atUserIds.split(",")).forEach(value -> {
builder.append("@");
builder.append(value);
builder.append(" ");
});
}
byte[] byt = StringUtils.getBytesUtf8(builder.toString());
String txt = StringUtils.newStringUtf8(byt);
text.put("title", title);
text.put("text", txt);
}
/**
* configure msg @person
*
* @param items items
*/
private void setMsgAt(Map<String, Object> items) {
Map<String, Object> at = new HashMap<>();
String[] atMobileArray = org.apache.dolphinscheduler.spi.utils.StringUtils.isNotBlank(atMobiles) ? atMobiles.split(",") : new String[0];
String[] atUserArray = org.apache.dolphinscheduler.spi.utils.StringUtils.isNotBlank(atUserIds) ? atUserIds.split(",") : new String[0];
boolean isAtAll = Objects.isNull(atAll) ? false : atAll;
at.put("atMobiles", atMobileArray);
at.put("atUserIds", atUserArray);
at.put("isAtAll", isAtAll);
items.put("at", at);
}
/**
* generate sign url
*
* @return sign url
*/
private String generateSignedUrl() { private String generateSignedUrl() {
Long timestamp = System.currentTimeMillis(); Long timestamp = System.currentTimeMillis();
String stringToSign = timestamp + "\n" + secret; String stringToSign = timestamp + "\n" + secret;

2
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannelFactoryTest.java

@ -32,7 +32,7 @@ public class DingTalkAlertChannelFactoryTest {
DingTalkAlertChannelFactory dingTalkAlertChannelFactory = new DingTalkAlertChannelFactory(); DingTalkAlertChannelFactory dingTalkAlertChannelFactory = new DingTalkAlertChannelFactory();
List<PluginParams> params = dingTalkAlertChannelFactory.params(); List<PluginParams> params = dingTalkAlertChannelFactory.params();
JSONUtils.toJsonString(params); JSONUtils.toJsonString(params);
Assert.assertEquals(8, params.size()); Assert.assertEquals(12, params.size());
} }
@Test @Test

4
dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java

@ -33,8 +33,10 @@ public class DingTalkSenderTest {
@Before @Before
public void initDingTalkConfig() { public void initDingTalkConfig() {
dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_KEYWORD, "keyWord"); dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_KEYWORD, "keyword");
dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_WEB_HOOK, "url"); dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_WEB_HOOK, "url");
dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_MSG_TYPE, DingTalkParamsConstants.DING_TALK_MSG_TYPE_MARKDOWN);
dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE, "false"); dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE, "false");
dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_PASSWORD, "password"); dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_PASSWORD, "password");
dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_PORT, "9988"); dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_PORT, "9988");

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java

@ -19,8 +19,10 @@ package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.ReleaseState;
import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import java.util.List;
import java.util.Map; import java.util.Map;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
@ -245,7 +247,7 @@ public interface ProcessDefinitionService {
* @param processTaskRelationJson process task relation json * @param processTaskRelationJson process task relation json
* @return check result code * @return check result code
*/ */
Map<String, Object> checkProcessNodeList(String processTaskRelationJson); Map<String, Object> checkProcessNodeList(String processTaskRelationJson, List<TaskDefinitionLog> taskDefinitionLogs);
/** /**
* get task node details based on process definition * get task node details based on process definition

10
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java

@ -76,6 +76,7 @@ import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.Stat;
import java.io.BufferedOutputStream; import java.io.BufferedOutputStream;
import java.io.IOException; import java.io.IOException;
@ -1071,7 +1072,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
* @return check result code * @return check result code
*/ */
@Override @Override
public Map<String, Object> checkProcessNodeList(String processTaskRelationJson) { public Map<String, Object> checkProcessNodeList(String processTaskRelationJson, List<TaskDefinitionLog> taskDefinitionLogsList) {
Map<String, Object> result = new HashMap<>(); Map<String, Object> result = new HashMap<>();
try { try {
if (processTaskRelationJson == null) { if (processTaskRelationJson == null) {
@ -1082,7 +1083,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
List<ProcessTaskRelation> taskRelationList = JSONUtils.toList(processTaskRelationJson, ProcessTaskRelation.class); List<ProcessTaskRelation> taskRelationList = JSONUtils.toList(processTaskRelationJson, ProcessTaskRelation.class);
// Check whether the task node is normal // Check whether the task node is normal
List<TaskNode> taskNodes = processService.transformTask(taskRelationList, Lists.newArrayList()); List<TaskNode> taskNodes = processService.transformTask(taskRelationList, taskDefinitionLogsList);
if (CollectionUtils.isEmpty(taskNodes)) { if (CollectionUtils.isEmpty(taskNodes)) {
logger.error("process node info is empty"); logger.error("process node info is empty");
@ -1110,8 +1111,9 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
} }
putMsg(result, Status.SUCCESS); putMsg(result, Status.SUCCESS);
} catch (Exception e) { } catch (Exception e) {
result.put(Constants.STATUS, Status.REQUEST_PARAMS_NOT_VALID_ERROR); result.put(Constants.STATUS, Status.INTERNAL_SERVER_ERROR_ARGS);
result.put(Constants.MSG, e.getMessage()); putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS, e.getMessage());
logger.error(Status.INTERNAL_SERVER_ERROR_ARGS.getMsg(), e);
} }
return result; return result;
} }

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java

@ -506,7 +506,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
ProcessDefinition processDefinition = processDefineMapper.queryByCode(processInstance.getProcessDefinitionCode()); ProcessDefinition processDefinition = processDefineMapper.queryByCode(processInstance.getProcessDefinitionCode());
List<ProcessTaskRelationLog> taskRelationList = JSONUtils.toList(taskRelationJson, ProcessTaskRelationLog.class); List<ProcessTaskRelationLog> taskRelationList = JSONUtils.toList(taskRelationJson, ProcessTaskRelationLog.class);
//check workflow json is valid //check workflow json is valid
result = processDefinitionService.checkProcessNodeList(taskRelationJson); result = processDefinitionService.checkProcessNodeList(taskRelationJson, taskDefinitionLogs);
if (result.get(Constants.STATUS) != Status.SUCCESS) { if (result.get(Constants.STATUS) != Status.SUCCESS) {
return result; return result;
} }

16
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java

@ -30,11 +30,13 @@ import org.apache.dolphinscheduler.common.enums.ReleaseState;
import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.enums.WarningType;
import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.graph.DAG;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.DagData; import org.apache.dolphinscheduler.dao.entity.DagData;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation; import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation;
import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.Schedule; import org.apache.dolphinscheduler.dao.entity.Schedule;
import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog;
import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
@ -82,6 +84,14 @@ public class ProcessDefinitionServiceTest {
+ "\"postTaskVersion\":1,\"conditionType\":0,\"conditionParams\":\"{}\"},{\"name\":\"\",\"preTaskCode\":123456789," + "\"postTaskVersion\":1,\"conditionType\":0,\"conditionParams\":\"{}\"},{\"name\":\"\",\"preTaskCode\":123456789,"
+ "\"preTaskVersion\":1,\"postTaskCode\":123451234,\"postTaskVersion\":1,\"conditionType\":0,\"conditionParams\":\"{}\"}]"; + "\"preTaskVersion\":1,\"postTaskCode\":123451234,\"postTaskVersion\":1,\"conditionType\":0,\"conditionParams\":\"{}\"}]";
private static final String taskDefinitionJson = "[{\"code\":123456789,\"name\":\"test1\",\"version\":1,\"description\":\"\",\"delayTime\":0,\"taskType\":\"SHELL\"," +
"\"taskParams\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo 1\",\"dependence\":{},\"conditionResult\":{\"successNode\":[],\"failedNode\":[]},\"waitStartTimeout\":{}," +
"\"switchResult\":{}},\"flag\":\"YES\",\"taskPriority\":\"MEDIUM\",\"workerGroup\":\"default\",\"failRetryTimes\":0,\"failRetryInterval\":1,\"timeoutFlag\":\"CLOSE\"," +
"\"timeoutNotifyStrategy\":null,\"timeout\":0,\"environmentCode\":-1},{\"code\":123451234,\"name\":\"test2\",\"version\":1,\"description\":\"\",\"delayTime\":0,\"taskType\":\"SHELL\"," +
"\"taskParams\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo 2\",\"dependence\":{},\"conditionResult\":{\"successNode\":[],\"failedNode\":[]},\"waitStartTimeout\":{}," +
"\"switchResult\":{}},\"flag\":\"YES\",\"taskPriority\":\"MEDIUM\",\"workerGroup\":\"default\",\"failRetryTimes\":0,\"failRetryInterval\":1,\"timeoutFlag\":\"CLOSE\"," +
"\"timeoutNotifyStrategy\":\"WARN\",\"timeout\":0,\"environmentCode\":-1}]";
@InjectMocks @InjectMocks
private ProcessDefinitionServiceImpl processDefinitionService; private ProcessDefinitionServiceImpl processDefinitionService;
@ -483,10 +493,12 @@ public class ProcessDefinitionServiceTest {
@Test @Test
public void testCheckProcessNodeList() { public void testCheckProcessNodeList() {
Map<String, Object> dataNotValidRes = processDefinitionService.checkProcessNodeList(null); Map<String, Object> dataNotValidRes = processDefinitionService.checkProcessNodeList(null, null);
Assert.assertEquals(Status.DATA_IS_NOT_VALID, dataNotValidRes.get(Constants.STATUS)); Assert.assertEquals(Status.DATA_IS_NOT_VALID, dataNotValidRes.get(Constants.STATUS));
Map<String, Object> taskEmptyRes = processDefinitionService.checkProcessNodeList(taskRelationJson); List<TaskDefinitionLog> taskDefinitionLogs = JSONUtils.toList(taskDefinitionJson, TaskDefinitionLog.class);
Map<String, Object> taskEmptyRes = processDefinitionService.checkProcessNodeList(taskRelationJson, taskDefinitionLogs);
Assert.assertEquals(Status.PROCESS_DAG_IS_EMPTY, taskEmptyRes.get(Constants.STATUS)); Assert.assertEquals(Status.PROCESS_DAG_IS_EMPTY, taskEmptyRes.get(Constants.STATUS));
} }

22
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java

@ -36,11 +36,13 @@ import org.apache.dolphinscheduler.common.graph.DAG;
import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNode;
import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog; import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
@ -128,6 +130,17 @@ public class ProcessInstanceServiceTest {
+ ":[\"\"],\"failedNode\":[\"\"]},\"dependence\":{}},\"flag\":\"NORMAL\",\"taskPriority\":\"MEDIUM\",\"workerGroup\":\"default\"," + ":[\"\"],\"failedNode\":[\"\"]},\"dependence\":{}},\"flag\":\"NORMAL\",\"taskPriority\":\"MEDIUM\",\"workerGroup\":\"default\","
+ "\"failRetryTimes\":\"0\",\"failRetryInterval\":\"1\",\"timeoutFlag\":\"CLOSE\",\"timeoutNotifyStrategy\":\"\",\"timeout\":null,\"delayTime\":\"0\"}]"; + "\"failRetryTimes\":\"0\",\"failRetryInterval\":\"1\",\"timeoutFlag\":\"CLOSE\",\"timeoutNotifyStrategy\":\"\",\"timeout\":null,\"delayTime\":\"0\"}]";
private String taskRelationJson = "[{\"name\":\"\",\"preTaskCode\":4254865123776,\"preTaskVersion\":1,\"postTaskCode\":4254862762304,\"postTaskVersion\":1,\"conditionType\":0," +
"\"conditionParams\":{}},{\"name\":\"\",\"preTaskCode\":0,\"preTaskVersion\":0,\"postTaskCode\":4254865123776,\"postTaskVersion\":1,\"conditionType\":0,\"conditionParams\":{}}]";
private String taskDefinitionJson = "[{\"code\":4254862762304,\"name\":\"test1\",\"version\":1,\"description\":\"\",\"delayTime\":0,\"taskType\":\"SHELL\",\"taskParams\":{\"resourceList\":[]," +
"\"localParams\":[],\"rawScript\":\"echo 1\",\"dependence\":{},\"conditionResult\":{\"successNode\":[],\"failedNode\":[]},\"waitStartTimeout\":{},\"switchResult\":{}},\"flag\":\"YES\"," +
"\"taskPriority\":\"MEDIUM\",\"workerGroup\":\"default\",\"failRetryTimes\":0,\"failRetryInterval\":1,\"timeoutFlag\":\"CLOSE\",\"timeoutNotifyStrategy\":null,\"timeout\":0," +
"\"environmentCode\":-1},{\"code\":4254865123776,\"name\":\"test2\",\"version\":1,\"description\":\"\",\"delayTime\":0,\"taskType\":\"SHELL\",\"taskParams\":{\"resourceList\":[]," +
"\"localParams\":[],\"rawScript\":\"echo 2\",\"dependence\":{},\"conditionResult\":{\"successNode\":[],\"failedNode\":[]},\"waitStartTimeout\":{},\"switchResult\":{}},\"flag\":\"YES\"," +
"\"taskPriority\":\"MEDIUM\",\"workerGroup\":\"default\",\"failRetryTimes\":0,\"failRetryInterval\":1,\"timeoutFlag\":\"CLOSE\",\"timeoutNotifyStrategy\":\"WARN\",\"timeout\":0," +
"\"environmentCode\":-1}]";
@Test @Test
public void testQueryProcessInstanceList() { public void testQueryProcessInstanceList() {
long projectCode = 1L; long projectCode = 1L;
@ -425,10 +438,12 @@ public class ProcessInstanceServiceTest {
when(processService.getTenantForProcess(Mockito.anyInt(), Mockito.anyInt())).thenReturn(tenant); when(processService.getTenantForProcess(Mockito.anyInt(), Mockito.anyInt())).thenReturn(tenant);
when(processService.updateProcessInstance(processInstance)).thenReturn(1); when(processService.updateProcessInstance(processInstance)).thenReturn(1);
when(processService.saveProcessDefine(loginUser, processDefinition, Boolean.TRUE, Boolean.FALSE)).thenReturn(1); when(processService.saveProcessDefine(loginUser, processDefinition, Boolean.TRUE, Boolean.FALSE)).thenReturn(1);
when(processDefinitionService.checkProcessNodeList(shellJson)).thenReturn(result);
List<TaskDefinitionLog> taskDefinitionLogs = JSONUtils.toList(taskDefinitionJson, TaskDefinitionLog.class);
when(processDefinitionService.checkProcessNodeList(taskRelationJson, taskDefinitionLogs)).thenReturn(result);
putMsg(result, Status.SUCCESS, projectCode); putMsg(result, Status.SUCCESS, projectCode);
Map<String, Object> processInstanceFinishRes = processInstanceService.updateProcessInstance(loginUser, projectCode, 1, Map<String, Object> processInstanceFinishRes = processInstanceService.updateProcessInstance(loginUser, projectCode, 1,
shellJson, taskJson,"2020-02-21 00:00:00", true, "", "", 0, "root"); taskRelationJson, taskDefinitionJson,"2020-02-21 00:00:00", true, "", "", 0, "root");
Assert.assertEquals(Status.SUCCESS, processInstanceFinishRes.get(Constants.STATUS)); Assert.assertEquals(Status.SUCCESS, processInstanceFinishRes.get(Constants.STATUS));
//success //success
@ -437,7 +452,7 @@ public class ProcessInstanceServiceTest {
when(processService.saveProcessDefine(loginUser, processDefinition, Boolean.FALSE, Boolean.FALSE)).thenReturn(1); when(processService.saveProcessDefine(loginUser, processDefinition, Boolean.FALSE, Boolean.FALSE)).thenReturn(1);
Map<String, Object> successRes = processInstanceService.updateProcessInstance(loginUser, projectCode, 1, Map<String, Object> successRes = processInstanceService.updateProcessInstance(loginUser, projectCode, 1,
shellJson, taskJson,"2020-02-21 00:00:00", Boolean.FALSE, "", "", 0, "root"); taskRelationJson, taskDefinitionJson,"2020-02-21 00:00:00", Boolean.FALSE, "", "", 0, "root");
Assert.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS)); Assert.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS));
} }
@ -638,5 +653,4 @@ public class ProcessInstanceServiceTest {
result.put(Constants.MSG, status.getMsg()); result.put(Constants.MSG, status.getMsg());
} }
} }
} }

10
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/AbstractCycle.java

@ -174,6 +174,16 @@ public abstract class AbstractCycle {
return (dayOfWeekFieldExpression instanceof Every || dayOfWeekFieldExpression instanceof Always); return (dayOfWeekFieldExpression instanceof Every || dayOfWeekFieldExpression instanceof Always);
} }
/**
* whether the year field has a value of every or always
*
* @return if year field has a value of every or always return trueelse return false
*/
protected boolean yearFieldIsEvery() {
FieldExpression yearFieldExpression = yearField.getExpression();
return (yearFieldExpression instanceof Every || yearFieldExpression instanceof Always);
}
/** /**
* get cycle enum * get cycle enum
* *

7
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtils.java

@ -22,6 +22,7 @@ import static org.apache.dolphinscheduler.service.quartz.cron.CycleFactory.hour;
import static org.apache.dolphinscheduler.service.quartz.cron.CycleFactory.min; import static org.apache.dolphinscheduler.service.quartz.cron.CycleFactory.min;
import static org.apache.dolphinscheduler.service.quartz.cron.CycleFactory.month; import static org.apache.dolphinscheduler.service.quartz.cron.CycleFactory.month;
import static org.apache.dolphinscheduler.service.quartz.cron.CycleFactory.week; import static org.apache.dolphinscheduler.service.quartz.cron.CycleFactory.week;
import static org.apache.dolphinscheduler.service.quartz.cron.CycleFactory.year;
import static com.cronutils.model.CronType.QUARTZ; import static com.cronutils.model.CronType.QUARTZ;
@ -90,7 +91,7 @@ public class CronUtils {
* @return CycleEnum * @return CycleEnum
*/ */
public static CycleEnum getMaxCycle(Cron cron) { public static CycleEnum getMaxCycle(Cron cron) {
return min(cron).addCycle(hour(cron)).addCycle(day(cron)).addCycle(week(cron)).addCycle(month(cron)).getCycle(); return min(cron).addCycle(hour(cron)).addCycle(day(cron)).addCycle(week(cron)).addCycle(month(cron)).addCycle(year(cron)).getCycle();
} }
/** /**
@ -100,7 +101,7 @@ public class CronUtils {
* @return CycleEnum * @return CycleEnum
*/ */
public static CycleEnum getMiniCycle(Cron cron) { public static CycleEnum getMiniCycle(Cron cron) {
return min(cron).addCycle(hour(cron)).addCycle(day(cron)).addCycle(week(cron)).addCycle(month(cron)).getMiniCycle(); return min(cron).addCycle(hour(cron)).addCycle(day(cron)).addCycle(week(cron)).addCycle(month(cron)).addCycle(year(cron)).getMiniCycle();
} }
/** /**
@ -186,7 +187,7 @@ public class CronUtils {
*/ */
public static List<Date> getSelfFireDateList(final Date startTime, final Date endTime, final List<Schedule> schedules) { public static List<Date> getSelfFireDateList(final Date startTime, final Date endTime, final List<Schedule> schedules) {
List<Date> result = new ArrayList<>(); List<Date> result = new ArrayList<>();
if(startTime.equals(endTime)){ if (startTime.equals(endTime)) {
result.add(startTime); result.add(startTime);
return result; return result;
} }

57
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CycleFactory.java

@ -73,6 +73,15 @@ public class CycleFactory {
return new MonthCycle(cron); return new MonthCycle(cron);
} }
/**
* year
* @param cron cron
* @return AbstractCycle
*/
public static AbstractCycle year(Cron cron) {
return new YearCycle(cron);
}
/** /**
* day cycle * day cycle
*/ */
@ -272,6 +281,54 @@ public class CycleFactory {
return CycleEnum.WEEK; return CycleEnum.WEEK;
} }
return null;
}
}
/**
* year cycle
*/
public static class YearCycle extends AbstractCycle {
public YearCycle(Cron cron) {
super(cron);
}
/**
* get cycle
* @return CycleEnum
*/
@Override
protected CycleEnum getCycle() {
boolean flag = (minFiledIsSetAll()
&& hourFiledIsSetAll()
&& dayOfMonthFieldIsSetAll()
&& dayOfWeekField.getExpression() instanceof QuestionMark
&& monthFieldIsSetAll())
&& yearFieldIsEvery() ||
(minFiledIsSetAll()
&& hourFiledIsSetAll()
&& dayOfMonthField.getExpression() instanceof QuestionMark
&& dayofWeekFieldIsSetAll()
&& monthFieldIsSetAll()
&& yearFieldIsEvery());
if (flag) {
return CycleEnum.YEAR;
}
return null;
}
/**
* get mini cycle
* @return CycleEnum
*/
@Override
protected CycleEnum getMiniCycle() {
if (yearFieldIsEvery()) {
return CycleEnum.YEAR;
}
return null; return null;
} }
} }

24
dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtilsTest.java

@ -95,6 +95,20 @@ public class CronUtilsTest {
CycleEnum cycleEnum3 = CronUtils.getMiniCycle(CronUtils.parse2Cron("0 * * * * ? *")); CycleEnum cycleEnum3 = CronUtils.getMiniCycle(CronUtils.parse2Cron("0 * * * * ? *"));
Assert.assertEquals("MINUTE", cycleEnum3.name()); Assert.assertEquals("MINUTE", cycleEnum3.name());
CycleEnum cycleEnum4 = CronUtils.getMaxCycle(CronUtils.parse2Cron("0 0 7 * 1 ? *"));
Assert.assertEquals("YEAR", cycleEnum4.name());
cycleEnum4 = CronUtils.getMiniCycle(CronUtils.parse2Cron("0 0 7 * 1 ? *"));
Assert.assertEquals("DAY", cycleEnum4.name());
CycleEnum cycleEnum5 = CronUtils.getMaxCycle(CronUtils.parse2Cron("0 0 7 * 1/1 ? *"));
Assert.assertEquals("MONTH", cycleEnum5.name());
CycleEnum cycleEnum6 = CronUtils.getMaxCycle(CronUtils.parse2Cron("0 0 7 * 1-2 ? *"));
Assert.assertEquals("YEAR", cycleEnum6.name());
CycleEnum cycleEnum7 = CronUtils.getMaxCycle(CronUtils.parse2Cron("0 0 7 * 1,2 ? *"));
Assert.assertEquals("YEAR", cycleEnum7.name());
} }
/** /**
@ -113,7 +127,7 @@ public class CronUtilsTest {
.instance(); .instance();
// minute cycle // minute cycle
String[] cronArayy = new String[]{"* * * * * ? *","* 0 * * * ? *", String[] cronArayy = new String[]{"* * * * * ? *","* 0 * * * ? *",
"* 5 * * 3/5 ? *","0 0 * * * ? *"}; "* 5 * * 3/5 ? *","0 0 * * * ? *", "0 0 7 * 1 ? *", "0 0 7 * 1/1 ? *", "0 0 7 * 1-2 ? *" , "0 0 7 * 1,2 ? *"};
for(String minCrontab:cronArayy){ for(String minCrontab:cronArayy){
if (!org.quartz.CronExpression.isValidExpression(minCrontab)) { if (!org.quartz.CronExpression.isValidExpression(minCrontab)) {
throw new RuntimeException(minCrontab+" verify failure, cron expression not valid"); throw new RuntimeException(minCrontab+" verify failure, cron expression not valid");
@ -156,6 +170,14 @@ public class CronUtilsTest {
logger.info("dayOfWeekField instanceof And:"+(dayOfWeekField.getExpression() instanceof And)); logger.info("dayOfWeekField instanceof And:"+(dayOfWeekField.getExpression() instanceof And));
logger.info("dayOfWeekField instanceof QuestionMark:"+(dayOfWeekField.getExpression() instanceof QuestionMark)); logger.info("dayOfWeekField instanceof QuestionMark:"+(dayOfWeekField.getExpression() instanceof QuestionMark));
CronField yearField = cron.retrieve(CronFieldName.YEAR);
logger.info("yearField instanceof Between:"+(yearField.getExpression() instanceof Between));
logger.info("yearField instanceof Always:"+(yearField.getExpression() instanceof Always));
logger.info("yearField instanceof Every:"+(yearField.getExpression() instanceof Every));
logger.info("yearField instanceof On:"+(yearField.getExpression() instanceof On));
logger.info("yearField instanceof And:"+(yearField.getExpression() instanceof And));
logger.info("yearField instanceof QuestionMark:"+(yearField.getExpression() instanceof QuestionMark));
CycleEnum cycleEnum = CronUtils.getMaxCycle(minCrontab); CycleEnum cycleEnum = CronUtils.getMaxCycle(minCrontab);
if(cycleEnum !=null){ if(cycleEnum !=null){
logger.info(cycleEnum.name()); logger.info(cycleEnum.name());

4
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java

@ -122,10 +122,12 @@ public abstract class AbstractCommandExecutor {
// merge error information to standard output stream // merge error information to standard output stream
processBuilder.redirectErrorStream(true); processBuilder.redirectErrorStream(true);
// setting up user to run commands // if sudo.enable=true,setting up user to run commands
if (OSUtils.isSudoEnable()) {
command.add("sudo"); command.add("sudo");
command.add("-u"); command.add("-u");
command.add(taskRequest.getTenantCode()); command.add(taskRequest.getTenantCode());
}
command.add(commandInterpreter()); command.add(commandInterpreter());
command.addAll(Collections.emptyList()); command.addAll(Collections.emptyList());
command.add(commandFile); command.add(commandFile);

10
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/util/OSUtils.java

@ -18,6 +18,7 @@
package org.apache.dolphinscheduler.plugin.task.util; package org.apache.dolphinscheduler.plugin.task.util;
import org.apache.dolphinscheduler.plugin.task.api.ShellExecutor; import org.apache.dolphinscheduler.plugin.task.api.ShellExecutor;
import org.apache.dolphinscheduler.spi.utils.PropertyUtils;
import org.apache.dolphinscheduler.spi.utils.StringUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils;
import java.io.IOException; import java.io.IOException;
@ -40,6 +41,15 @@ public class OSUtils {
return StringUtils.isEmpty(tenantCode) ? command : "sudo -u " + tenantCode + " " + command; return StringUtils.isEmpty(tenantCode) ? command : "sudo -u " + tenantCode + " " + command;
} }
/**
* use sudo or not
*
* @return true is use sudo
*/
public static boolean isSudoEnable() {
return PropertyUtils.getBoolean("sudo.enable", Boolean.TRUE);
}
/** /**
* whether is macOS * whether is macOS
* *

3
dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java

@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.task.datax;
import org.apache.dolphinscheduler.spi.enums.DbType; import org.apache.dolphinscheduler.spi.enums.DbType;
import com.alibaba.druid.sql.dialect.clickhouse.parser.ClickhouseStatementParser;
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser; import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser;
import com.alibaba.druid.sql.dialect.postgresql.parser.PGSQLStatementParser; import com.alibaba.druid.sql.dialect.postgresql.parser.PGSQLStatementParser;
@ -91,6 +92,8 @@ public class DataxUtils {
return new OracleStatementParser(sql); return new OracleStatementParser(sql);
case SQLSERVER: case SQLSERVER:
return new SQLServerStatementParser(sql); return new SQLServerStatementParser(sql);
case CLICKHOUSE:
return new ClickhouseStatementParser(sql);
default: default:
return null; return null;
} }

4
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue

@ -330,11 +330,11 @@
}) })
if (this.type === 'instance') { if (this.type === 'instance') {
this.$router.push({ this.$router.push({
path: `/projects/${this.projectCode}/instance/list` path: `/projects/${this.projectCode}/instance/list/${methodParam}`
}) })
} else { } else {
this.$router.push({ this.$router.push({
path: `/projects/${this.projectCode}/definition/list` path: `/projects/${this.projectCode}/definition/list/${methodParam}`
}) })
} }
}) })

10
dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/_source/list.vue

@ -51,8 +51,11 @@
<span>{{scope.row.updateTime | formatDate}}</span> <span>{{scope.row.updateTime | formatDate}}</span>
</template> </template>
</el-table-column> </el-table-column>
<el-table-column :label="$t('Operation')" width="150"> <el-table-column :label="$t('Operation')" width="180">
<template slot-scope="scope"> <template slot-scope="scope">
<el-tooltip :content="$t('ReUpload File')" placement="top" :enterable="false">
<span><el-button type="primary" size="mini" icon="el-icon-refresh-right" @click="_reUploadFile(scope.row)" v-show="scope.row.directory? false: true" circle></el-button></span>
</el-tooltip>
<el-tooltip :content="$t('Edit')" placement="top"> <el-tooltip :content="$t('Edit')" placement="top">
<span><el-button type="primary" size="mini" icon="el-icon-edit-outline" @click="_edit(scope.row)" :disabled="_rtDisb(scope.row)" circle></el-button></span> <span><el-button type="primary" size="mini" icon="el-icon-edit-outline" @click="_edit(scope.row)" :disabled="_rtDisb(scope.row)" circle></el-button></span>
</el-tooltip> </el-tooltip>
@ -92,7 +95,9 @@
import { filtTypeArr } from '../../_source/common' import { filtTypeArr } from '../../_source/common'
import { bytesToSize } from '@/module/util/util' import { bytesToSize } from '@/module/util/util'
import { downloadFile } from '@/module/download' import { downloadFile } from '@/module/download'
import { findComponentDownward } from '@/module/util/'
import localStore from '@/module/util/localStorage' import localStore from '@/module/util/localStorage'
export default { export default {
name: 'file-manage-list', name: 'file-manage-list',
data () { data () {
@ -123,6 +128,9 @@
this.$router.push({ path: `/resource/file/list/${item.id}` }) this.$router.push({ path: `/resource/file/list/${item.id}` })
} }
}, },
_reUploadFile (item) {
findComponentDownward(this.$root, 'roof-nav')._fileReUpload(item)
},
_downloadFile (item) { _downloadFile (item) {
downloadFile(`resources/${item.id}/download`) downloadFile(`resources/${item.id}/download`)
}, },

9
dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/_source/list.vue

@ -50,8 +50,11 @@
<span>{{scope.row.updateTime | formatDate}}</span> <span>{{scope.row.updateTime | formatDate}}</span>
</template> </template>
</el-table-column> </el-table-column>
<el-table-column :label="$t('Operation')" width="150"> <el-table-column :label="$t('Operation')" width="180">
<template slot-scope="scope"> <template slot-scope="scope">
<el-tooltip :content="$t('ReUpload File')" placement="top" :enterable="false">
<span><el-button type="primary" size="mini" icon="el-icon-refresh-right" @click="_reUploadFile(scope.row)" v-show="scope.row.directory? false: true" circle></el-button></span>
</el-tooltip>
<el-tooltip :content="$t('Edit')" placement="top" :enterable="false"> <el-tooltip :content="$t('Edit')" placement="top" :enterable="false">
<span><el-button type="primary" size="mini" icon="el-icon-edit-outline" @click="_edit(scope.row)" :disabled="_rtDisb(scope.row)" circle></el-button></span> <span><el-button type="primary" size="mini" icon="el-icon-edit-outline" @click="_edit(scope.row)" :disabled="_rtDisb(scope.row)" circle></el-button></span>
</el-tooltip> </el-tooltip>
@ -90,6 +93,7 @@
import { mapActions } from 'vuex' import { mapActions } from 'vuex'
import { filtTypeArr } from '../../_source/common' import { filtTypeArr } from '../../_source/common'
import { bytesToSize } from '@/module/util/util' import { bytesToSize } from '@/module/util/util'
import { findComponentDownward } from '@/module/util/'
import { downloadFile } from '@/module/download' import { downloadFile } from '@/module/download'
import localStore from '@/module/util/localStorage' import localStore from '@/module/util/localStorage'
@ -122,6 +126,9 @@
this.$router.push({ path: `/resource/file/list/${item.id}` }) this.$router.push({ path: `/resource/file/list/${item.id}` })
} }
}, },
_reUploadFile (item) {
findComponentDownward(this.$root, 'roof-nav')._fileReUpload(item)
},
_downloadFile (item) { _downloadFile (item) {
downloadFile(`resources/${item.id}/download`) downloadFile(`resources/${item.id}/download`)
}, },

82
dolphinscheduler-ui/src/js/module/components/fileUpdate/fileReUpload.vue

@ -1,25 +1,26 @@
/* /*
* Licensed to the Apache Software Foundation (ASF) under one or more * Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with * contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership. * this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0 * The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with * (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at * the License. You may obtain a copy of the License at
* *
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* *
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
<template> <template>
<m-popup <m-popup
ref="popup" ref="popup"
:ok-text="$t('Upload')" :ok-text="$t('Upload')"
:nameText="$t('ReUpload File')" :nameText="$t('ReUpload File')"
@ok="_ok" @ok="_ok"
@close="_close"
:disabled="progress === 0 ? false : true"> :disabled="progress === 0 ? false : true">
<template slot="content"> <template slot="content">
<form name="files" enctype="multipart/form-data" method="post"> <form name="files" enctype="multipart/form-data" method="post">
@ -47,7 +48,7 @@
type="input" type="input"
size="small" size="small"
v-model="name" v-model="name"
:disabled="progress !== 0" :disabled="true"
:placeholder="$t('Please enter name')"> :placeholder="$t('Please enter name')">
</el-input> </el-input>
</template> </template>
@ -92,7 +93,7 @@
import mProgressBar from '@/module/components/progressBar/progressBar' import mProgressBar from '@/module/components/progressBar/progressBar'
export default { export default {
name: 'file-update', name: 'file-upload',
data () { data () {
return { return {
store, store,
@ -105,17 +106,21 @@
// file // file
file: null, file: null,
currentDir: '/', currentDir: '/',
id: null,
// Whether to drag upload // Whether to drag upload
dragOver: false dragOver: false
} }
}, },
watch: { watch: {
originalFileData: {
deep: true,
handler () {
this._init()
}
}
}, },
props: { props: {
type: String, originalFileData: Object
fileName: String,
desc: String,
id: Number
}, },
methods: { methods: {
/** /**
@ -124,7 +129,7 @@
_ok () { _ok () {
this.$refs.popup.spinnerLoading = true this.$refs.popup.spinnerLoading = true
if (this._validation()) { if (this._validation()) {
if (this.fileName === this.name) { if (this.originalFileData.fileName === this.name) {
const isLt1024M = this.file.size / 1024 / 1024 < 1024 const isLt1024M = this.file.size / 1024 / 1024 < 1024
if (isLt1024M) { if (isLt1024M) {
this._formDataUpdate().then(res => { this._formDataUpdate().then(res => {
@ -139,10 +144,8 @@
this.$refs.popup.spinnerLoading = false this.$refs.popup.spinnerLoading = false
} }
} else { } else {
this.store.dispatch('resource/resourceVerifyName', { const params = { fullName: this.currentDir + this.name, type: 'FILE' }
fullName: '/' + this.name, this.store.dispatch('resource/resourceVerifyName', params).then(res => {
type: this.type
}).then(res => {
const isLt1024M = this.file.size / 1024 / 1024 < 1024 const isLt1024M = this.file.size / 1024 / 1024 < 1024
if (isLt1024M) { if (isLt1024M) {
this._formDataUpdate().then(res => { this._formDataUpdate().then(res => {
@ -165,6 +168,9 @@
this.$refs.popup.spinnerLoading = false this.$refs.popup.spinnerLoading = false
} }
}, },
_close () {
this.$emit('closeFileUpload')
},
/** /**
* validation * validation
*/ */
@ -186,19 +192,19 @@
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let self = this let self = this
let formData = new FormData() let formData = new FormData()
formData.append('file', this.file) formData.append('file', this.file)
formData.append('name', this.name) formData.append('name', this.name)
formData.append('description', this.description) formData.append('description', this.description)
formData.append('id', this.id) formData.append('type', 'FILE')
formData.append('type', this.type) io.put('resources/' + this.id, res => {
io.post('resources/update', res => {
this.$message.success(res.msg) this.$message.success(res.msg)
resolve() resolve()
self.$emit('onUpdate') self.$emit('onUploadFile')
this.reset() this.reset()
}, e => { }, e => {
reject(e) reject(e)
self.$emit('close') self.$emit('closeFileUpload')
this.$message.error(e.msg || '') this.$message.error(e.msg || '')
this.reset() this.reset()
}, { }, {
@ -244,11 +250,21 @@
this.file = file this.file = file
this.name = file.name this.name = file.name
this.$refs.file.value = null this.$refs.file.value = null
},
_init () {
if (this.originalFileData) {
this.id = this.originalFileData.id
this.name = this.originalFileData.fileName
if (this.originalFileData.desc) {
this.description = this.originalFileData.desc
}
this.currentDir = this.originalFileData.fullName.substring(0, this.originalFileData.fullName.length - this.originalFileData.fileName.length)
}
} }
}, },
mounted () { mounted () {
this.name = this.fileName this.reset()
this.description = this.desc this._init()
}, },
components: { mPopup, mListBoxF, mProgressBar } components: { mPopup, mListBoxF, mProgressBar }
} }

46
dolphinscheduler-ui/src/js/module/components/nav/nav.vue

@ -163,6 +163,17 @@
width="auto"> width="auto">
<m-resource-child-update :type="type" :id="id" @onProgressResourceChildUpdate="onProgressResourceChildUpdate" @onUpdateResourceChildUpdate="onUpdateResourceChildUpdate" @onArchiveFileChildUpdate="onArchiveResourceChildUpdate" @closeResourceChildUpdate="closeResourceChildUpdate"></m-resource-child-update> <m-resource-child-update :type="type" :id="id" @onProgressResourceChildUpdate="onProgressResourceChildUpdate" @onUpdateResourceChildUpdate="onUpdateResourceChildUpdate" @onArchiveFileChildUpdate="onArchiveResourceChildUpdate" @closeResourceChildUpdate="closeResourceChildUpdate"></m-resource-child-update>
</el-dialog> </el-dialog>
<el-dialog
:visible.sync="fileUploadDialog"
append-to-body="true"
width="auto">
<m-file-upload
:originalFileData="originalFileData"
@onUploadFile="onUploadFile"
@closeFileUpload="closeFileUpload">
</m-file-upload>
</el-dialog>
</div> </div>
</template> </template>
<script> <script>
@ -174,6 +185,7 @@
import mFileChildUpdate from '@/module/components/fileUpdate/fileChildUpdate' import mFileChildUpdate from '@/module/components/fileUpdate/fileChildUpdate'
import mResourceChildUpdate from '@/module/components/fileUpdate/resourceChildUpdate' import mResourceChildUpdate from '@/module/components/fileUpdate/resourceChildUpdate'
import mDefinitionUpdate from '@/module/components/fileUpdate/definitionUpdate' import mDefinitionUpdate from '@/module/components/fileUpdate/definitionUpdate'
import mFileUpload from '@/module/components/fileUpdate/fileReUpload'
import mProgressBar from '@/module/components/progressBar/progressBar' import mProgressBar from '@/module/components/progressBar/progressBar'
import { findLocale, localeList } from '@/module/i18n/config' import { findLocale, localeList } from '@/module/i18n/config'
@ -202,7 +214,11 @@
fileUpdateDialog: false, fileUpdateDialog: false,
fileChildUpdateDialog: false, fileChildUpdateDialog: false,
id: null, id: null,
resourceChildUpdateDialog: false fileName: '',
description: '',
originalFileData: {},
resourceChildUpdateDialog: false,
fileUploadDialog: false
} }
}, },
@ -250,16 +266,13 @@
this.progress = 0 this.progress = 0
this.definitionUpdateDialog = false this.definitionUpdateDialog = false
}, },
onArchiveDefinition () { onArchiveDefinition () {
this.isUpdate = true this.isUpdate = true
}, },
closeDefinition () { closeDefinition () {
this.progress = 0 this.progress = 0
this.definitionUpdateDialog = false this.definitionUpdateDialog = false
}, },
onProgressFileUpdate (val) { onProgressFileUpdate (val) {
this.progress = val this.progress = val
}, },
@ -270,6 +283,17 @@
this.progress = 0 this.progress = 0
this.fileUpdateDialog = false this.fileUpdateDialog = false
}, },
onUploadFile () {
let self = this
findComponentDownward(self.$root, 'resource-list-index-FILE')._updateList()
this.isUpdate = false
this.progress = 0
this.fileUploadDialog = false
},
closeFileUpload () {
this.progress = 0
this.fileUploadDialog = false
},
onArchiveFileUpdate () { onArchiveFileUpdate () {
this.isUpdate = true this.isUpdate = true
}, },
@ -277,7 +301,6 @@
this.progress = 0 this.progress = 0
this.fileUpdateDialog = false this.fileUpdateDialog = false
}, },
_fileChildUpdate (type, data) { _fileChildUpdate (type, data) {
if (this.progress) { if (this.progress) {
this._toggleArchive() this._toggleArchive()
@ -287,7 +310,14 @@
this.id = data this.id = data
this.fileChildUpdateDialog = true this.fileChildUpdateDialog = true
}, },
_fileReUpload (item) {
if (this.progress) {
this._toggleArchive()
return
}
this.originalFileData = item
this.fileUploadDialog = true
},
onProgressFileChildUpdate (val) { onProgressFileChildUpdate (val) {
this.progress = val this.progress = val
}, },
@ -307,7 +337,6 @@
this.progress = 0 this.progress = 0
this.fileChildUpdateDialog = false this.fileChildUpdateDialog = false
}, },
_resourceChildUpdate (type, data) { _resourceChildUpdate (type, data) {
if (this.progress) { if (this.progress) {
this._toggleArchive() this._toggleArchive()
@ -350,7 +379,6 @@
* Language switching * Language switching
*/ */
_toggleLanguage (language) { _toggleLanguage (language) {
console.log(language)
cookies.set('language', language, { path: '/' }) cookies.set('language', language, { path: '/' })
setTimeout(() => { setTimeout(() => {
window.location.reload() window.location.reload()
@ -365,7 +393,7 @@
computed: { computed: {
...mapState('user', ['userInfo']) ...mapState('user', ['userInfo'])
}, },
components: { mFileUpdate, mProgressBar, mDefinitionUpdate, mFileChildUpdate, mResourceChildUpdate } components: { mFileUpdate, mProgressBar, mDefinitionUpdate, mFileChildUpdate, mResourceChildUpdate, mFileUpload }
} }
</script> </script>

4
dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js

@ -727,6 +727,10 @@ export default {
webHook: 'WebHook', webHook: 'WebHook',
Keyword: 'Keyword', Keyword: 'Keyword',
Secret: 'Secret', Secret: 'Secret',
MsgType: 'MsgType',
AtMobiles: '@Mobiles',
AtUserIds: '@UserIds',
IsAtAll: '@All',
Proxy: 'Proxy', Proxy: 'Proxy',
receivers: 'Receivers', receivers: 'Receivers',
receiverCcs: 'ReceiverCcs', receiverCcs: 'ReceiverCcs',

4
dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js

@ -726,6 +726,10 @@ export default {
webHook: 'Web钩子', webHook: 'Web钩子',
Keyword: '关键词', Keyword: '关键词',
Secret: '密钥', Secret: '密钥',
MsgType: '消息类型',
AtMobiles: '@手机号',
AtUserIds: '@用户ID',
IsAtAll: '@所有人',
Proxy: '代理', Proxy: '代理',
receivers: '收件人', receivers: '收件人',
receiverCcs: '抄送人', receiverCcs: '抄送人',

Loading…
Cancel
Save