break60 4 years ago
parent
commit
b1db5dc13a
  1. 7
      docker/build/hooks/build
  2. 2
      docker/build/hooks/push
  3. 13
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java
  4. 17
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/DingTalkManager.java
  5. 18
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java
  6. 9
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactory.java
  7. 73
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java
  8. 14
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java
  9. 57
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtils.java
  10. 30
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java
  11. 163
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java
  12. 7
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java
  13. 125
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java
  14. 55
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/PropertyUtils.java
  15. 8
      dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java
  16. 10
      dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/ExcelUtilsTest.java
  17. 50
      dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/PropertyUtilsTest.java
  18. 63
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java
  19. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
  20. 192
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java
  21. 2
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java
  22. 108
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java
  23. 14
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java
  24. 351
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java
  25. 124
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java
  26. 52
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java
  27. 45
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java
  28. 6
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
  29. 9
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue
  30. 3
      dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js
  31. 3
      dolphinscheduler-ui/src/js/conf/home/store/dag/mutations.js
  32. 2
      dolphinscheduler-ui/src/js/conf/home/store/dag/state.js
  33. 1
      dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js
  34. 1
      dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js
  35. 258
      mvnw

7
docker/build/hooks/build

@ -24,7 +24,8 @@ printenv
if [ -z "${VERSION}" ] if [ -z "${VERSION}" ]
then then
echo "set default environment variable [VERSION]" echo "set default environment variable [VERSION]"
export VERSION=$(cat $(pwd)/pom.xml | grep '<version>' -m 1 | awk '{print $1}' | sed 's/<version>//' | sed 's/<\/version>//') VERSION=$(grep '<version>' -m 1 "$(pwd)"/pom.xml | awk '{print $1}' | sed 's/<version>//' | sed 's/<\/version>//')
export VERSION
fi fi
if [ "${DOCKER_REPO}x" = "x" ] if [ "${DOCKER_REPO}x" = "x" ]
@ -44,10 +45,10 @@ mvn -B clean compile package -Prelease -Dmaven.test.skip=true
# mv dolphinscheduler-bin.tar.gz file to docker/build directory # mv dolphinscheduler-bin.tar.gz file to docker/build directory
echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/\n" echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/\n"
mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/ mv "$(pwd)"/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-"${VERSION}"-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/
# docker build # docker build
echo -e "docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/\n" echo -e "docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/\n"
sudo docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/ sudo docker build --build-arg VERSION="${VERSION}" -t $DOCKER_REPO:"${VERSION}" "$(pwd)/docker/build/"
echo "------ dolphinscheduler end - build -------" echo "------ dolphinscheduler end - build -------"

2
docker/build/hooks/push

@ -19,6 +19,6 @@
echo "------ push start -------" echo "------ push start -------"
printenv printenv
docker push $DOCKER_REPO:${VERSION} docker push "$DOCKER_REPO:${VERSION}"
echo "------ push end -------" echo "------ push end -------"

13
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.alert; package org.apache.dolphinscheduler.alert;
import org.apache.dolphinscheduler.alert.plugin.EmailAlertPlugin; import org.apache.dolphinscheduler.alert.plugin.EmailAlertPlugin;
@ -25,11 +26,12 @@ import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.DaoFactory; import org.apache.dolphinscheduler.dao.DaoFactory;
import org.apache.dolphinscheduler.dao.entity.Alert; import org.apache.dolphinscheduler.dao.entity.Alert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List; import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* alert of start * alert of start
*/ */
@ -40,8 +42,6 @@ public class AlertServer {
*/ */
private AlertDao alertDao = DaoFactory.getDaoInstance(AlertDao.class); private AlertDao alertDao = DaoFactory.getDaoInstance(AlertDao.class);
private AlertSender alertSender;
private static AlertServer instance; private static AlertServer instance;
private FilePluginManager alertPluginManager; private FilePluginManager alertPluginManager;
@ -61,7 +61,7 @@ public class AlertServer {
alertPluginManager.addPlugin(new EmailAlertPlugin()); alertPluginManager.addPlugin(new EmailAlertPlugin());
} }
public synchronized static AlertServer getInstance() { public static synchronized AlertServer getInstance() {
if (null == instance) { if (null == instance) {
instance = new AlertServer(); instance = new AlertServer();
} }
@ -78,12 +78,11 @@ public class AlertServer {
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
} }
List<Alert> alerts = alertDao.listWaitExecutionAlert(); List<Alert> alerts = alertDao.listWaitExecutionAlert();
alertSender = new AlertSender(alerts, alertDao, alertPluginManager); AlertSender alertSender = new AlertSender(alerts, alertDao, alertPluginManager);
alertSender.run(); alertSender.run();
} }
} }
public static void main(String[] args) { public static void main(String[] args) {
AlertServer alertServer = AlertServer.getInstance(); AlertServer alertServer = AlertServer.getInstance();
alertServer.start(); alertServer.start();

17
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/DingTalkManager.java

@ -14,40 +14,41 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.alert.manager; package org.apache.dolphinscheduler.alert.manager;
import org.apache.dolphinscheduler.alert.utils.Constants; import org.apache.dolphinscheduler.alert.utils.Constants;
import org.apache.dolphinscheduler.alert.utils.DingTalkUtils; import org.apache.dolphinscheduler.alert.utils.DingTalkUtils;
import org.apache.dolphinscheduler.plugin.model.AlertInfo; import org.apache.dolphinscheduler.plugin.model.AlertInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException; import java.io.IOException;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* Ding Talk Manager * Ding Talk Manager
*/ */
public class DingTalkManager { public class DingTalkManager {
private static final Logger logger = LoggerFactory.getLogger(EnterpriseWeChatManager.class); private static final Logger logger = LoggerFactory.getLogger(DingTalkManager.class);
public Map<String,Object> send(AlertInfo alert) { public Map<String, Object> send(AlertInfo alert) {
Map<String,Object> retMap = new HashMap<>(); Map<String, Object> retMap = new HashMap<>();
retMap.put(Constants.STATUS, false); retMap.put(Constants.STATUS, false);
logger.info("send message {}", alert.getAlertData().getTitle()); logger.info("send message {}", alert.getAlertData().getTitle());
try { try {
String msg = buildMessage(alert); String msg = buildMessage(alert);
DingTalkUtils.sendDingTalkMsg(msg, Constants.UTF_8); DingTalkUtils.sendDingTalkMsg(msg, Constants.UTF_8);
} catch (IOException e) { } catch (IOException e) {
logger.error(e.getMessage(),e); logger.error(e.getMessage(), e);
} }
retMap.put(Constants.STATUS, true); retMap.put(Constants.STATUS, true);
return retMap; return retMap;
} }
private String buildMessage(AlertInfo alert) { private String buildMessage(AlertInfo alert) {
String msg = alert.getAlertData().getContent(); return alert.getAlertData().getContent();
return msg;
} }
} }

18
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.alert.plugin; package org.apache.dolphinscheduler.alert.plugin;
import org.apache.dolphinscheduler.alert.manager.DingTalkManager; import org.apache.dolphinscheduler.alert.manager.DingTalkManager;
@ -28,14 +29,19 @@ import org.apache.dolphinscheduler.plugin.api.AlertPlugin;
import org.apache.dolphinscheduler.plugin.model.AlertData; import org.apache.dolphinscheduler.plugin.model.AlertData;
import org.apache.dolphinscheduler.plugin.model.AlertInfo; import org.apache.dolphinscheduler.plugin.model.AlertInfo;
import org.apache.dolphinscheduler.plugin.model.PluginName; import org.apache.dolphinscheduler.plugin.model.PluginName;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.util.*;
/** /**
* EmailAlertPlugin * EmailAlertPlugin
* * <p>
* This plugin is a default plugin, and mix up email and enterprise wechat, because adapt with former alert behavior * This plugin is a default plugin, and mix up email and enterprise wechat, because adapt with former alert behavior
*/ */
public class EmailAlertPlugin implements AlertPlugin { public class EmailAlertPlugin implements AlertPlugin {
@ -125,10 +131,10 @@ public class EmailAlertPlugin implements AlertPlugin {
} }
} }
if (DingTalkUtils.isEnableDingTalk) { if (DingTalkUtils.IS_ENABLE_DING_TALK) {
logger.info("Ding Talk is enable."); logger.info("Ding Talk is enable.");
dingTalkManager.send(info); dingTalkManager.send(info);
} }
} else { } else {
retMaps.put(Constants.MESSAGE, "alert send error."); retMaps.put(Constants.MESSAGE, "alert send error.");

9
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactory.java

@ -14,23 +14,22 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.alert.template; package org.apache.dolphinscheduler.alert.template;
import org.apache.dolphinscheduler.alert.template.impl.DefaultHTMLTemplate; import org.apache.dolphinscheduler.alert.template.impl.DefaultHTMLTemplate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* the alert template factory * the alert template factory
*/ */
public class AlertTemplateFactory { public class AlertTemplateFactory {
private static final Logger logger = LoggerFactory.getLogger(AlertTemplateFactory.class); private AlertTemplateFactory() {
}
private AlertTemplateFactory(){}
/** /**
* get a template from alert.properties conf file * get a template from alert.properties conf file
*
* @return a template, default is DefaultHTMLTemplate * @return a template, default is DefaultHTMLTemplate
*/ */
public static AlertTemplate getMessageTemplate() { public static AlertTemplate getMessageTemplate() {

73
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java

@ -14,21 +14,29 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.alert.template.impl; package org.apache.dolphinscheduler.alert.template.impl;
import com.fasterxml.jackson.databind.JsonNode; import static org.apache.dolphinscheduler.common.utils.Preconditions.checkNotNull;
import com.fasterxml.jackson.databind.node.ArrayNode;
import org.apache.dolphinscheduler.alert.template.AlertTemplate; import org.apache.dolphinscheduler.alert.template.AlertTemplate;
import org.apache.dolphinscheduler.alert.utils.Constants; import org.apache.dolphinscheduler.alert.utils.Constants;
import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.enums.ShowType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.StringUtils;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.apache.dolphinscheduler.common.utils.*;
import java.util.*; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import static org.apache.dolphinscheduler.common.utils.Preconditions.*;
/** /**
* the default html alert message template * the default html alert message template
@ -37,33 +45,33 @@ public class DefaultHTMLTemplate implements AlertTemplate {
public static final Logger logger = LoggerFactory.getLogger(DefaultHTMLTemplate.class); public static final Logger logger = LoggerFactory.getLogger(DefaultHTMLTemplate.class);
@Override @Override
public String getMessageFromTemplate(String content, ShowType showType,boolean showAll) { public String getMessageFromTemplate(String content, ShowType showType, boolean showAll) {
switch (showType){ switch (showType) {
case TABLE: case TABLE:
return getTableTypeMessage(content,showAll); return getTableTypeMessage(content, showAll);
case TEXT: case TEXT:
return getTextTypeMessage(content,showAll); return getTextTypeMessage(content);
default: default:
throw new IllegalArgumentException(String.format("not support showType: %s in DefaultHTMLTemplate",showType)); throw new IllegalArgumentException(String.format("not support showType: %s in DefaultHTMLTemplate", showType));
} }
} }
/** /**
* get alert message which type is TABLE * get alert message which type is TABLE
*
* @param content message content * @param content message content
* @param showAll weather to show all * @param showAll weather to show all
* @return alert message * @return alert message
*/ */
private String getTableTypeMessage(String content,boolean showAll){ private String getTableTypeMessage(String content, boolean showAll) {
if (StringUtils.isNotEmpty(content)){ if (StringUtils.isNotEmpty(content)) {
List<LinkedHashMap> mapItemsList = JSONUtils.toList(content, LinkedHashMap.class); List<LinkedHashMap> mapItemsList = JSONUtils.toList(content, LinkedHashMap.class);
if(!showAll && mapItemsList.size() > Constants.NUMBER_1000){ if (!showAll && mapItemsList.size() > Constants.NUMBER_1000) {
mapItemsList = mapItemsList.subList(0,Constants.NUMBER_1000); mapItemsList = mapItemsList.subList(0, Constants.NUMBER_1000);
} }
StringBuilder contents = new StringBuilder(200); StringBuilder contents = new StringBuilder(200);
@ -71,15 +79,15 @@ public class DefaultHTMLTemplate implements AlertTemplate {
boolean flag = true; boolean flag = true;
String title = ""; String title = "";
for (LinkedHashMap mapItems : mapItemsList){ for (LinkedHashMap mapItems : mapItemsList) {
Set<Map.Entry<String, Object>> entries = mapItems.entrySet(); Set<Entry<String, Object>> entries = mapItems.entrySet();
Iterator<Map.Entry<String, Object>> iterator = entries.iterator(); Iterator<Entry<String, Object>> iterator = entries.iterator();
StringBuilder t = new StringBuilder(Constants.TR); StringBuilder t = new StringBuilder(Constants.TR);
StringBuilder cs = new StringBuilder(Constants.TR); StringBuilder cs = new StringBuilder(Constants.TR);
while (iterator.hasNext()){ while (iterator.hasNext()) {
Map.Entry<String, Object> entry = iterator.next(); Map.Entry<String, Object> entry = iterator.next();
t.append(Constants.TH).append(entry.getKey()).append(Constants.TH_END); t.append(Constants.TH).append(entry.getKey()).append(Constants.TH_END);
@ -88,14 +96,14 @@ public class DefaultHTMLTemplate implements AlertTemplate {
} }
t.append(Constants.TR_END); t.append(Constants.TR_END);
cs.append(Constants.TR_END); cs.append(Constants.TR_END);
if (flag){ if (flag) {
title = t.toString(); title = t.toString();
} }
flag = false; flag = false;
contents.append(cs); contents.append(cs);
} }
return getMessageFromHtmlTemplate(title,contents.toString()); return getMessageFromHtmlTemplate(title, contents.toString());
} }
return content; return content;
@ -103,22 +111,22 @@ public class DefaultHTMLTemplate implements AlertTemplate {
/** /**
* get alert message which type is TEXT * get alert message which type is TEXT
*
* @param content message content * @param content message content
* @param showAll weather to show all
* @return alert message * @return alert message
*/ */
private String getTextTypeMessage(String content,boolean showAll){ private String getTextTypeMessage(String content) {
if (StringUtils.isNotEmpty(content)){ if (StringUtils.isNotEmpty(content)) {
ArrayNode list = JSONUtils.parseArray(content); ArrayNode list = JSONUtils.parseArray(content);
StringBuilder contents = new StringBuilder(100); StringBuilder contents = new StringBuilder(100);
for (JsonNode jsonNode : list){ for (JsonNode jsonNode : list) {
contents.append(Constants.TR); contents.append(Constants.TR);
contents.append(Constants.TD).append(jsonNode.toString()).append(Constants.TD_END); contents.append(Constants.TD).append(jsonNode.toString()).append(Constants.TD_END);
contents.append(Constants.TR_END); contents.append(Constants.TR_END);
} }
return getMessageFromHtmlTemplate(null,contents.toString()); return getMessageFromHtmlTemplate(null, contents.toString());
} }
@ -127,16 +135,17 @@ public class DefaultHTMLTemplate implements AlertTemplate {
/** /**
* get alert message from a html template * get alert message from a html template
* @param title message title *
* @param content message content * @param title message title
* @param content message content
* @return alert message which use html template * @return alert message which use html template
*/ */
private String getMessageFromHtmlTemplate(String title,String content){ private String getMessageFromHtmlTemplate(String title, String content) {
checkNotNull(content); checkNotNull(content);
String htmlTableThead = StringUtils.isEmpty(title) ? "" : String.format("<thead>%s</thead>\n",title); String htmlTableThead = StringUtils.isEmpty(title) ? "" : String.format("<thead>%s</thead>%n", title);
return Constants.HTML_HEADER_PREFIX +htmlTableThead + content + Constants.TABLE_BODY_HTML_TAIL; return Constants.HTML_HEADER_PREFIX + htmlTableThead + content + Constants.TABLE_BODY_HTML_TAIL;
} }
} }

14
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.alert.utils; package org.apache.dolphinscheduler.alert.utils;
/** /**
@ -23,6 +24,7 @@ public class Constants {
private Constants() { private Constants() {
throw new IllegalStateException("Constants class"); throw new IllegalStateException("Constants class");
} }
/** /**
* alert properties path * alert properties path
*/ */
@ -67,7 +69,7 @@ public class Constants {
public static final String MAIL_SMTP_SSL_ENABLE = "mail.smtp.ssl.enable"; public static final String MAIL_SMTP_SSL_ENABLE = "mail.smtp.ssl.enable";
public static final String MAIL_SMTP_SSL_TRUST="mail.smtp.ssl.trust"; public static final String MAIL_SMTP_SSL_TRUST = "mail.smtp.ssl.trust";
public static final String TEXT_HTML_CHARSET_UTF_8 = "text/html;charset=utf-8"; public static final String TEXT_HTML_CHARSET_UTF_8 = "text/html;charset=utf-8";
@ -156,7 +158,7 @@ public class Constants {
public static final String ENTERPRISE_WECHAT_AGENT_ID = "enterprise.wechat.agent.id"; public static final String ENTERPRISE_WECHAT_AGENT_ID = "enterprise.wechat.agent.id";
public static final String ENTERPRISE_WECHAT_USERS = "enterprise.wechat.users"; public static final String ENTERPRISE_WECHAT_USERS = "enterprise.wechat.users";
public static final String DINGTALK_WEBHOOK = "dingtalk.webhook"; public static final String DINGTALK_WEBHOOK = "dingtalk.webhook";
@ -174,7 +176,13 @@ public class Constants {
public static final String DINGTALK_ENABLE = "dingtalk.isEnable"; public static final String DINGTALK_ENABLE = "dingtalk.isEnable";
public static final String HTML_HEADER_PREFIX = "<!DOCTYPE HTML PUBLIC '-//W3C//DTD HTML 4.01 Transitional//EN' 'http://www.w3.org/TR/html4/loose.dtd'><html><head><title>dolphinscheduler</title><meta name='Keywords' content=''><meta name='Description' content=''><style type=\"text/css\">table {margin-top:0px;padding-top:0px;border:1px solid;font-size: 14px;color: #333333;border-width: 1px;border-color: #666666;border-collapse: collapse;}table th {border-width: 1px;padding: 8px;border-style: solid;border-color: #666666;background-color: #dedede;text-align: left;}table td {border-width: 1px;padding: 8px;border-style: solid;border-color: #666666;background-color: #ffffff;text-align: left;}</style></head><body style=\"margin:0;padding:0\"><table border=\"1px\" cellpadding=\"5px\" cellspacing=\"-10px\"> "; public static final String HTML_HEADER_PREFIX = "<!DOCTYPE HTML PUBLIC '-//W3C//DTD HTML 4.01 Transitional//EN' 'http://www.w3.org/TR/html4/loose.dtd'>"
+ "<html><head><title>dolphinscheduler</title><meta name='Keywords' content=''>"
+ "<meta name='Description' content=''>"
+ "<style type=\"text/css\">table {margin-top:0px;padding-top:0px;border:1px solid;font-size: 14px;color: #333333;border-width: 1px;border-color: #666666;border-collapse: collapse;}"
+ "table th {border-width: 1px;padding: 8px;border-style: solid;border-color: #666666;background-color: #dedede;text-align: left;}"
+ "table td {border-width: 1px;padding: 8px;border-style: solid;border-color: #666666;background-color: #ffffff;text-align: left;}</style>"
+ "/head><body style=\"margin:0;padding:0\"><table border=\"1px\" cellpadding=\"5px\" cellspacing=\"-10px\"> ";
public static final String TABLE_BODY_HTML_TAIL = "</table></body></html>"; public static final String TABLE_BODY_HTML_TAIL = "</table></body></html>";

57
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtils.java

@ -14,10 +14,11 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.alert.utils;
package org.apache.dolphinscheduler.alert.utils;
import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.*;
import org.apache.commons.codec.binary.StringUtils; import org.apache.commons.codec.binary.StringUtils;
import org.apache.http.HttpEntity; import org.apache.http.HttpEntity;
import org.apache.http.HttpHost; import org.apache.http.HttpHost;
@ -32,44 +33,50 @@ import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils; import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException; import java.io.IOException;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* DingTalkUtils utils * DingTalkUtils utils
* support send msg to ding talk by robot message push function. * support send msg to ding talk by robot message push function.
* support proxy setting * support PROXY setting
*/ */
public class DingTalkUtils { public class DingTalkUtils {
public static final Logger logger = LoggerFactory.getLogger(DingTalkUtils.class); public static final Logger logger = LoggerFactory.getLogger(DingTalkUtils.class);
public static final boolean isEnableDingTalk = PropertyUtils.getBoolean(Constants.DINGTALK_ENABLE); public static final boolean IS_ENABLE_DING_TALK = PropertyUtils.getBoolean(Constants.DINGTALK_ENABLE);
private static final String dingTaskUrl = PropertyUtils.getString(Constants.DINGTALK_WEBHOOK); private static final String DING_TASK_URL = PropertyUtils.getString(Constants.DINGTALK_WEBHOOK);
private static final String keyword = PropertyUtils.getString(Constants.DINGTALK_KEYWORD); private static final String KEYWORD = PropertyUtils.getString(Constants.DINGTALK_KEYWORD);
private static final Boolean isEnableProxy = PropertyUtils.getBoolean(Constants.DINGTALK_PROXY_ENABLE); private static final Boolean IS_ENABLE_PROXY = PropertyUtils.getBoolean(Constants.DINGTALK_PROXY_ENABLE);
private static final String proxy = PropertyUtils.getString(Constants.DINGTALK_PROXY); private static final String PROXY = PropertyUtils.getString(Constants.DINGTALK_PROXY);
private static final String user = PropertyUtils.getString(Constants.DINGTALK_USER); private static final String USER = PropertyUtils.getString(Constants.DINGTALK_USER);
private static final String passwd = PropertyUtils.getString(Constants.DINGTALK_PASSWORD); private static final String PASSWD = PropertyUtils.getString(Constants.DINGTALK_PASSWORD);
private static final Integer port = PropertyUtils.getInt(Constants.DINGTALK_PORT); private static final Integer PORT = PropertyUtils.getInt(Constants.DINGTALK_PORT);
private DingTalkUtils() {
throw new IllegalStateException(DingTalkUtils.class.getName());
}
/** /**
* send message interface * send message interface
* only support text message format now. * only support text message format now.
*
* @param msg message context to send * @param msg message context to send
* @param charset charset type * @param charset charset type
* @return result of sending msg * @return result of sending msg
* @throws IOException the IOException * @throws IOException the IOException
*/ */
public static String sendDingTalkMsg(String msg, String charset) throws IOException { public static String sendDingTalkMsg(String msg, String charset) throws IOException {
String msgToJson = textToJsonString(msg + "#" + keyword); String msgToJson = textToJsonString(msg + "#" + KEYWORD);
HttpPost httpPost = constructHttpPost(msgToJson, charset); HttpPost httpPost = constructHttpPost(msgToJson, charset);
CloseableHttpClient httpClient; CloseableHttpClient httpClient;
if (isEnableProxy) { if (Boolean.TRUE.equals(IS_ENABLE_PROXY)) {
httpClient = getProxyClient(); httpClient = getProxyClient();
RequestConfig rcf = getProxyConfig(); RequestConfig rcf = getProxyConfig();
httpPost.setConfig(rcf); httpPost.setConfig(rcf);
@ -87,28 +94,26 @@ public class DingTalkUtils {
} finally { } finally {
response.close(); response.close();
} }
logger.info("Ding Talk send [{}], resp:{%s}", msg, resp); logger.info("Ding Talk send [{}], resp:{%s}", msg);
return resp; return resp;
} finally { } finally {
httpClient.close(); httpClient.close();
} }
} }
public static HttpPost constructHttpPost(String msg, String charset) { public static HttpPost constructHttpPost(String msg, String charset) {
HttpPost post = new HttpPost(dingTaskUrl); HttpPost post = new HttpPost(DING_TASK_URL);
StringEntity entity = new StringEntity(msg, charset); StringEntity entity = new StringEntity(msg, charset);
post.setEntity(entity); post.setEntity(entity);
post.addHeader("Content-Type", "application/json; charset=utf-8"); post.addHeader("Content-Type", "application/json; charset=utf-8");
return post; return post;
} }
public static CloseableHttpClient getProxyClient() { public static CloseableHttpClient getProxyClient() {
HttpHost httpProxy = new HttpHost(proxy, port); HttpHost httpProxy = new HttpHost(PROXY, PORT);
CredentialsProvider provider = new BasicCredentialsProvider(); CredentialsProvider provider = new BasicCredentialsProvider();
provider.setCredentials(new AuthScope(httpProxy), new UsernamePasswordCredentials(user, passwd)); provider.setCredentials(new AuthScope(httpProxy), new UsernamePasswordCredentials(USER, PASSWD));
CloseableHttpClient httpClient = HttpClients.custom().setDefaultCredentialsProvider(provider).build(); return HttpClients.custom().setDefaultCredentialsProvider(provider).build();
return httpClient;
} }
public static CloseableHttpClient getDefaultClient() { public static CloseableHttpClient getDefaultClient() {
@ -116,14 +121,14 @@ public class DingTalkUtils {
} }
public static RequestConfig getProxyConfig() { public static RequestConfig getProxyConfig() {
HttpHost httpProxy = new HttpHost(proxy, port); HttpHost httpProxy = new HttpHost(PROXY, PORT);
return RequestConfig.custom().setProxy(httpProxy).build(); return RequestConfig.custom().setProxy(httpProxy).build();
} }
public static String textToJsonString(String text) { public static String textToJsonString(String text) {
Map<String, Object> items = new HashMap<String, Object>(); Map<String, Object> items = new HashMap<>();
items.put("msgtype", "text"); items.put("msgtype", "text");
Map<String, String> textContent = new HashMap<String, String>(); Map<String, String> textContent = new HashMap<>();
byte[] byt = StringUtils.getBytesUtf8(text); byte[] byt = StringUtils.getBytesUtf8(text);
String txt = StringUtils.newStringUtf8(byt); String txt = StringUtils.newStringUtf8(byt);
textContent.put("content", txt); textContent.put("content", txt);

30
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java

@ -60,9 +60,13 @@ public class EnterpriseWeChatUtils {
private static final String ENTERPRISE_WE_CHAT_TEAM_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TEAM_SEND_MSG); private static final String ENTERPRISE_WE_CHAT_TEAM_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TEAM_SEND_MSG);
private static final String ENTERPRISE_WE_CHAT_USER_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USER_SEND_MSG); private static final String ENTERPRISE_WE_CHAT_USER_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USER_SEND_MSG);
private static final String agentIdRegExp = "\\{agentId}"; private static final String AGENT_ID_REG_EXP = "\\{agentId}";
private static final String msgRegExp = "\\{msg}"; private static final String MSG_REG_EXP = "\\{msg}";
private static final String userRegExp = "\\{toUser}"; private static final String USER_REG_EXP = "\\{toUser}";
private EnterpriseWeChatUtils() {
throw new IllegalStateException(EnterpriseWeChatUtils.class.getName());
}
/** /**
* get Enterprise WeChat is enable * get Enterprise WeChat is enable
@ -120,8 +124,8 @@ public class EnterpriseWeChatUtils {
*/ */
public static String makeTeamSendMsg(String toParty, String agentId, String msg) { public static String makeTeamSendMsg(String toParty, String agentId, String msg) {
return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty}", toParty) return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty}", toParty)
.replaceAll(agentIdRegExp, agentId) .replaceAll(AGENT_ID_REG_EXP, agentId)
.replaceAll(msgRegExp, msg); .replaceAll(MSG_REG_EXP, msg);
} }
/** /**
@ -135,8 +139,8 @@ public class EnterpriseWeChatUtils {
public static String makeTeamSendMsg(Collection<String> toParty, String agentId, String msg) { public static String makeTeamSendMsg(Collection<String> toParty, String agentId, String msg) {
String listParty = FuncUtils.mkString(toParty, "|"); String listParty = FuncUtils.mkString(toParty, "|");
return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty}", listParty) return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty}", listParty)
.replaceAll(agentIdRegExp, agentId) .replaceAll(AGENT_ID_REG_EXP, agentId)
.replaceAll(msgRegExp, msg); .replaceAll(MSG_REG_EXP, msg);
} }
/** /**
@ -148,9 +152,9 @@ public class EnterpriseWeChatUtils {
* @return Enterprise WeChat send message * @return Enterprise WeChat send message
*/ */
public static String makeUserSendMsg(String toUser, String agentId, String msg) { public static String makeUserSendMsg(String toUser, String agentId, String msg) {
return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\{toUser}", toUser) return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll(USER_REG_EXP, toUser)
.replaceAll(agentIdRegExp, agentId) .replaceAll(AGENT_ID_REG_EXP, agentId)
.replaceAll(msgRegExp, msg); .replaceAll(MSG_REG_EXP, msg);
} }
/** /**
@ -163,9 +167,9 @@ public class EnterpriseWeChatUtils {
*/ */
public static String makeUserSendMsg(Collection<String> toUser, String agentId, String msg) { public static String makeUserSendMsg(Collection<String> toUser, String agentId, String msg) {
String listUser = FuncUtils.mkString(toUser, "|"); String listUser = FuncUtils.mkString(toUser, "|");
return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll(userRegExp, listUser) return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll(USER_REG_EXP, listUser)
.replaceAll(agentIdRegExp, agentId) .replaceAll(AGENT_ID_REG_EXP, agentId)
.replaceAll(msgRegExp, msg); .replaceAll(MSG_REG_EXP, msg);
} }
/** /**

163
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java

@ -14,23 +14,29 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.alert.utils; package org.apache.dolphinscheduler.alert.utils;
import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.poi.hssf.usermodel.HSSFCell; import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFRow; import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet; import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook; import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.CellStyle; import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.HorizontalAlignment; import org.apache.poi.ss.usermodel.HorizontalAlignment;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File; import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.util.*; import java.util.ArrayList;
import org.apache.dolphinscheduler.common.utils.*; import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* excel utils * excel utils
@ -38,19 +44,25 @@ import org.apache.dolphinscheduler.common.utils.*;
public class ExcelUtils { public class ExcelUtils {
private static final Logger logger = LoggerFactory.getLogger(ExcelUtils.class); private static final Logger logger = LoggerFactory.getLogger(ExcelUtils.class);
private ExcelUtils() {
throw new IllegalStateException(ExcelUtils.class.getName());
}
/** /**
* generate excel file * generate excel file
*
* @param content the content * @param content the content
* @param title the title * @param title the title
* @param xlsFilePath the xls path * @param xlsFilePath the xls path
*/ */
public static void genExcelFile(String content,String title,String xlsFilePath){ public static void genExcelFile(String content, String title, String xlsFilePath) {
List<LinkedHashMap> itemsList; List<LinkedHashMap> itemsList;
//The JSONUtils.toList has been try catch ex //The JSONUtils.toList has been try catch ex
itemsList = JSONUtils.toList(content, LinkedHashMap.class); itemsList = JSONUtils.toList(content, LinkedHashMap.class);
if (CollectionUtils.isEmpty(itemsList)){ if (CollectionUtils.isEmpty(itemsList)) {
logger.error("itemsList is null"); logger.error("itemsList is null");
throw new RuntimeException("itemsList is null"); throw new RuntimeException("itemsList is null");
} }
@ -59,82 +71,79 @@ public class ExcelUtils {
List<String> headerList = new ArrayList<>(); List<String> headerList = new ArrayList<>();
Iterator<Map.Entry<String, Object>> iter = headerMap.entrySet().iterator(); for (Map.Entry<String, Object> en : headerMap.entrySet()) {
while (iter.hasNext()){
Map.Entry<String, Object> en = iter.next();
headerList.add(en.getKey()); headerList.add(en.getKey());
} }
HSSFWorkbook wb = null; HSSFWorkbook wb = null;
FileOutputStream fos = null; FileOutputStream fos = null;
try { try {
// declare a workbook // declare a workbook
wb = new HSSFWorkbook(); wb = new HSSFWorkbook();
// generate a table // generate a table
HSSFSheet sheet = wb.createSheet(); HSSFSheet sheet = wb.createSheet();
HSSFRow row = sheet.createRow(0); HSSFRow row = sheet.createRow(0);
//set the height of the first line //set the height of the first line
row.setHeight((short)500); row.setHeight((short) 500);
//set Horizontal right //set Horizontal right
CellStyle cellStyle = wb.createCellStyle(); CellStyle cellStyle = wb.createCellStyle();
cellStyle.setAlignment(HorizontalAlignment.RIGHT); cellStyle.setAlignment(HorizontalAlignment.RIGHT);
//setting excel headers //setting excel headers
for (int i = 0; i < headerList.size(); i++) { for (int i = 0; i < headerList.size(); i++) {
HSSFCell cell = row.createCell(i); HSSFCell cell = row.createCell(i);
cell.setCellStyle(cellStyle); cell.setCellStyle(cellStyle);
cell.setCellValue(headerList.get(i)); cell.setCellValue(headerList.get(i));
} }
//setting excel body //setting excel body
int rowIndex = 1; int rowIndex = 1;
for (LinkedHashMap<String, Object> itemsMap : itemsList){ for (LinkedHashMap<String, Object> itemsMap : itemsList) {
Object[] values = itemsMap.values().toArray(); Object[] values = itemsMap.values().toArray();
row = sheet.createRow(rowIndex); row = sheet.createRow(rowIndex);
//setting excel body height //setting excel body height
row.setHeight((short)500); row.setHeight((short) 500);
rowIndex++; rowIndex++;
for (int j = 0 ; j < values.length ; j++){ for (int j = 0; j < values.length; j++) {
HSSFCell cell1 = row.createCell(j); HSSFCell cell1 = row.createCell(j);
cell1.setCellStyle(cellStyle); cell1.setCellStyle(cellStyle);
cell1.setCellValue(String.valueOf(values[j])); cell1.setCellValue(String.valueOf(values[j]));
} }
} }
for (int i = 0; i < headerList.size(); i++) { for (int i = 0; i < headerList.size(); i++) {
sheet.setColumnWidth(i, headerList.get(i).length() * 800); sheet.setColumnWidth(i, headerList.get(i).length() * 800);
} }
File file = new File(xlsFilePath); File file = new File(xlsFilePath);
if (!file.exists()) { if (!file.exists()) {
file.mkdirs(); file.mkdirs();
} }
//setting file output //setting file output
fos = new FileOutputStream(xlsFilePath + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS); fos = new FileOutputStream(xlsFilePath + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS);
wb.write(fos); wb.write(fos);
}catch (Exception e){ } catch (Exception e) {
logger.error("generate excel error",e); throw new RuntimeException("generate excel error", e);
throw new RuntimeException("generate excel error",e); } finally {
}finally { if (wb != null) {
if (wb != null){ try {
try { wb.close();
wb.close(); } catch (IOException e) {
} catch (IOException e) { logger.error(e.getMessage(), e);
logger.error(e.getMessage(),e); }
} }
} if (fos != null) {
if (fos != null){ try {
try { fos.close();
fos.close(); } catch (IOException e) {
} catch (IOException e) { logger.error(e.getMessage(), e);
logger.error(e.getMessage(),e); }
} }
} }
}
} }
} }

7
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java

@ -14,15 +14,20 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.alert.utils; package org.apache.dolphinscheduler.alert.utils;
import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.StringUtils;
public class FuncUtils { public class FuncUtils {
private FuncUtils() {
throw new IllegalStateException(FuncUtils.class.getName());
}
public static String mkString(Iterable<String> list, String split) { public static String mkString(Iterable<String> list, String split) {
if (null == list || StringUtils.isEmpty(split)){ if (null == list || StringUtils.isEmpty(split)) {
return null; return null;
} }

125
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java

@ -14,23 +14,39 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.alert.utils; package org.apache.dolphinscheduler.alert.utils;
import org.apache.dolphinscheduler.alert.template.AlertTemplate; import org.apache.dolphinscheduler.alert.template.AlertTemplate;
import org.apache.dolphinscheduler.alert.template.AlertTemplateFactory; import org.apache.dolphinscheduler.alert.template.AlertTemplateFactory;
import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.enums.ShowType;
import org.apache.commons.mail.EmailException;
import org.apache.commons.mail.HtmlEmail;
import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.mail.*; import org.apache.commons.mail.EmailException;
import javax.mail.internet.*; import org.apache.commons.mail.HtmlEmail;
import java.io.*;
import java.util.*; import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import javax.mail.Authenticator;
import javax.mail.Message;
import javax.mail.MessagingException;
import javax.mail.PasswordAuthentication;
import javax.mail.Session;
import javax.mail.Transport;
import javax.mail.internet.InternetAddress;
import javax.mail.internet.MimeBodyPart;
import javax.mail.internet.MimeMessage;
import javax.mail.internet.MimeMultipart;
import javax.mail.internet.MimeUtility;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* mail utils * mail utils
@ -55,7 +71,7 @@ public class MailUtils {
public static final Boolean MAIL_USE_SSL = PropertyUtils.getBoolean(Constants.MAIL_SMTP_SSL_ENABLE); public static final Boolean MAIL_USE_SSL = PropertyUtils.getBoolean(Constants.MAIL_SMTP_SSL_ENABLE);
public static final String xlsFilePath = PropertyUtils.getString(Constants.XLS_FILE_PATH,"/tmp/xls"); public static final String XLS_FILE_PATH = PropertyUtils.getString(Constants.XLS_FILE_PATH, "/tmp/xls");
public static final String STARTTLS_ENABLE = PropertyUtils.getString(Constants.MAIL_SMTP_STARTTLS_ENABLE); public static final String STARTTLS_ENABLE = PropertyUtils.getString(Constants.MAIL_SMTP_STARTTLS_ENABLE);
@ -67,23 +83,29 @@ public class MailUtils {
//Solve the problem of messy Chinese name in excel attachment //Solve the problem of messy Chinese name in excel attachment
static { static {
System.setProperty("mail.mime.splitlongparameters","false"); System.setProperty("mail.mime.splitlongparameters", "false");
}
private MailUtils() {
throw new IllegalStateException(MailUtils.class.getName());
} }
/** /**
* send mail to receivers * send mail to receivers
*
* @param receivers the receiver list * @param receivers the receiver list
* @param title the title * @param title the title
* @param content the content * @param content the content
* @param showType the show type * @param showType the show type
* @return the result map * @return the result map
*/ */
public static Map<String,Object> sendMails(Collection<String> receivers, String title, String content,String showType) { public static Map<String, Object> sendMails(Collection<String> receivers, String title, String content, String showType) {
return sendMails(receivers, null, title, content, showType); return sendMails(receivers, null, title, content, showType);
} }
/** /**
* send mail * send mail
*
* @param receivers the receiver list * @param receivers the receiver list
* @param receiversCc cc list * @param receiversCc cc list
* @param title the title * @param title the title
@ -91,8 +113,8 @@ public class MailUtils {
* @param showType the show type * @param showType the show type
* @return the send result * @return the send result
*/ */
public static Map<String,Object> sendMails(Collection<String> receivers, Collection<String> receiversCc, String title, String content, String showType) { public static Map<String, Object> sendMails(Collection<String> receivers, Collection<String> receiversCc, String title, String content, String showType) {
Map<String,Object> retMap = new HashMap<>(); Map<String, Object> retMap = new HashMap<>();
retMap.put(Constants.STATUS, false); retMap.put(Constants.STATUS, false);
// if there is no receivers && no receiversCc, no need to process // if there is no receivers && no receiversCc, no need to process
@ -111,14 +133,14 @@ public class MailUtils {
email.setMailSession(session); email.setMailSession(session);
email.setFrom(MAIL_SENDER); email.setFrom(MAIL_SENDER);
email.setCharset(Constants.UTF_8); email.setCharset(Constants.UTF_8);
if (CollectionUtils.isNotEmpty(receivers)){ if (CollectionUtils.isNotEmpty(receivers)) {
// receivers mail // receivers mail
for (String receiver : receivers) { for (String receiver : receivers) {
email.addTo(receiver); email.addTo(receiver);
} }
} }
if (CollectionUtils.isNotEmpty(receiversCc)){ if (CollectionUtils.isNotEmpty(receiversCc)) {
//cc //cc
for (String receiverCc : receiversCc) { for (String receiverCc : receiversCc) {
email.addCc(receiverCc); email.addCc(receiverCc);
@ -129,16 +151,16 @@ public class MailUtils {
} catch (Exception e) { } catch (Exception e) {
handleException(receivers, retMap, e); handleException(receivers, retMap, e);
} }
}else if (showType.equals(ShowType.ATTACHMENT.getDescp()) || showType.equals(ShowType.TABLEATTACHMENT.getDescp())) { } else if (showType.equals(ShowType.ATTACHMENT.getDescp()) || showType.equals(ShowType.TABLEATTACHMENT.getDescp())) {
try { try {
String partContent = (showType.equals(ShowType.ATTACHMENT.getDescp()) ? "Please see the attachment " + title + Constants.EXCEL_SUFFIX_XLS : htmlTable(content,false)); String partContent = (showType.equals(ShowType.ATTACHMENT.getDescp()) ? "Please see the attachment " + title + Constants.EXCEL_SUFFIX_XLS : htmlTable(content, false));
attachment(receivers,receiversCc,title,content,partContent); attachment(receivers, receiversCc, title, content, partContent);
retMap.put(Constants.STATUS, true); retMap.put(Constants.STATUS, true);
return retMap; return retMap;
}catch (Exception e){ } catch (Exception e) {
handleException(receivers, retMap, e); handleException(receivers, retMap, e);
return retMap; return retMap;
} }
@ -149,49 +171,52 @@ public class MailUtils {
/** /**
* html table content * html table content
*
* @param content the content * @param content the content
* @param showAll if show the whole content * @param showAll if show the whole content
* @return the html table form * @return the html table form
*/ */
private static String htmlTable(String content, boolean showAll){ private static String htmlTable(String content, boolean showAll) {
return alertTemplate.getMessageFromTemplate(content,ShowType.TABLE,showAll); return alertTemplate.getMessageFromTemplate(content, ShowType.TABLE, showAll);
} }
/** /**
* html table content * html table content
*
* @param content the content * @param content the content
* @return the html table form * @return the html table form
*/ */
private static String htmlTable(String content){ private static String htmlTable(String content) {
return htmlTable(content,true); return htmlTable(content, true);
} }
/** /**
* html text content * html text content
*
* @param content the content * @param content the content
* @return text in html form * @return text in html form
*/ */
private static String htmlText(String content){ private static String htmlText(String content) {
return alertTemplate.getMessageFromTemplate(content,ShowType.TEXT); return alertTemplate.getMessageFromTemplate(content, ShowType.TEXT);
} }
/** /**
* send mail as Excel attachment * send mail as Excel attachment
*
* @param receivers the receiver list * @param receivers the receiver list
* @param title the title * @param title the title
* @throws Exception
*/ */
private static void attachment(Collection<String> receivers,Collection<String> receiversCc,String title,String content,String partContent)throws Exception{ private static void attachment(Collection<String> receivers, Collection<String> receiversCc, String title, String content, String partContent) throws Exception {
MimeMessage msg = getMimeMessage(receivers); MimeMessage msg = getMimeMessage(receivers);
attachContent(receiversCc, title, content,partContent, msg); attachContent(receiversCc, title, content, partContent, msg);
} }
/** /**
* get MimeMessage * get MimeMessage
*
* @param receivers receivers * @param receivers receivers
* @return the MimeMessage * @return the MimeMessage
* @throws MessagingException
*/ */
private static MimeMessage getMimeMessage(Collection<String> receivers) throws MessagingException { private static MimeMessage getMimeMessage(Collection<String> receivers) throws MessagingException {
@ -223,7 +248,7 @@ public class MailUtils {
props.setProperty(Constants.MAIL_SMTP_AUTH, Constants.STRING_TRUE); props.setProperty(Constants.MAIL_SMTP_AUTH, Constants.STRING_TRUE);
props.setProperty(Constants.MAIL_TRANSPORT_PROTOCOL, MAIL_PROTOCOL); props.setProperty(Constants.MAIL_TRANSPORT_PROTOCOL, MAIL_PROTOCOL);
props.setProperty(Constants.MAIL_SMTP_STARTTLS_ENABLE, STARTTLS_ENABLE); props.setProperty(Constants.MAIL_SMTP_STARTTLS_ENABLE, STARTTLS_ENABLE);
if (SSL_ENABLE) { if (Boolean.TRUE.equals(SSL_ENABLE)) {
props.setProperty(Constants.MAIL_SMTP_SSL_ENABLE, "true"); props.setProperty(Constants.MAIL_SMTP_SSL_ENABLE, "true");
props.setProperty(Constants.MAIL_SMTP_SSL_TRUST, SSL_TRUST); props.setProperty(Constants.MAIL_SMTP_SSL_TRUST, SSL_TRUST);
} }
@ -241,20 +266,19 @@ public class MailUtils {
/** /**
* attach content * attach content
*
* @param receiversCc the cc list * @param receiversCc the cc list
* @param title the title * @param title the title
* @param content the content * @param content the content
* @param partContent the partContent * @param partContent the partContent
* @param msg the message * @param msg the message
* @throws MessagingException
* @throws IOException
*/ */
private static void attachContent(Collection<String> receiversCc, String title, String content, String partContent,MimeMessage msg) throws MessagingException, IOException { private static void attachContent(Collection<String> receiversCc, String title, String content, String partContent, MimeMessage msg) throws MessagingException, IOException {
/** /*
* set receiverCc * set receiverCc
*/ */
if(CollectionUtils.isNotEmpty(receiversCc)){ if (CollectionUtils.isNotEmpty(receiversCc)) {
for (String receiverCc : receiversCc){ for (String receiverCc : receiversCc) {
msg.addRecipients(Message.RecipientType.CC, InternetAddress.parse(receiverCc)); msg.addRecipients(Message.RecipientType.CC, InternetAddress.parse(receiverCc));
} }
} }
@ -267,16 +291,16 @@ public class MailUtils {
part1.setContent(partContent, Constants.TEXT_HTML_CHARSET_UTF_8); part1.setContent(partContent, Constants.TEXT_HTML_CHARSET_UTF_8);
// set attach file // set attach file
MimeBodyPart part2 = new MimeBodyPart(); MimeBodyPart part2 = new MimeBodyPart();
File file = new File(xlsFilePath + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS); File file = new File(XLS_FILE_PATH + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS);
if (!file.getParentFile().exists()) { if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs(); file.getParentFile().mkdirs();
} }
// make excel file // make excel file
ExcelUtils.genExcelFile(content,title,xlsFilePath); ExcelUtils.genExcelFile(content, title, XLS_FILE_PATH);
part2.attachFile(file); part2.attachFile(file);
part2.setFileName(MimeUtility.encodeText(title + Constants.EXCEL_SUFFIX_XLS,Constants.UTF_8,"B")); part2.setFileName(MimeUtility.encodeText(title + Constants.EXCEL_SUFFIX_XLS, Constants.UTF_8, "B"));
// add components to collection // add components to collection
partList.addBodyPart(part1); partList.addBodyPart(part1);
partList.addBodyPart(part2); partList.addBodyPart(part2);
@ -289,21 +313,21 @@ public class MailUtils {
/** /**
* the string object map * the string object map
*
* @param title the title * @param title the title
* @param content the content * @param content the content
* @param showType the showType * @param showType the showType
* @param retMap the result map * @param retMap the result map
* @param email the email * @param email the email
* @return the result map * @return the result map
* @throws EmailException
*/ */
private static Map<String, Object> getStringObjectMap(String title, String content, String showType, Map<String, Object> retMap, HtmlEmail email) throws EmailException { private static Map<String, Object> getStringObjectMap(String title, String content, String showType, Map<String, Object> retMap, HtmlEmail email) throws EmailException {
/** /*
* the subject of the message to be sent * the subject of the message to be sent
*/ */
email.setSubject(title); email.setSubject(title);
/** /*
* to send information, you can use HTML tags in mail content because of the use of HtmlEmail * to send information, you can use HTML tags in mail content because of the use of HtmlEmail
*/ */
if (showType.equals(ShowType.TABLE.getDescp())) { if (showType.equals(ShowType.TABLE.getDescp())) {
@ -322,23 +346,24 @@ public class MailUtils {
/** /**
* file delete * file delete
*
* @param file the file to delete * @param file the file to delete
*/ */
public static void deleteFile(File file){ public static void deleteFile(File file) {
if(file.exists()){ if (file.exists()) {
if(file.delete()){ if (file.delete()) {
logger.info("delete success: {}",file.getAbsolutePath() + file.getName()); logger.info("delete success: {}", file.getAbsolutePath() + file.getName());
}else{ } else {
logger.info("delete fail: {}", file.getAbsolutePath() + file.getName()); logger.info("delete fail: {}", file.getAbsolutePath() + file.getName());
} }
}else{ } else {
logger.info("file not exists: {}", file.getAbsolutePath() + file.getName()); logger.info("file not exists: {}", file.getAbsolutePath() + file.getName());
} }
} }
/** /**
* handle exception * handle exception
*
* @param receivers the receiver list * @param receivers the receiver list
* @param retMap the result map * @param retMap the result map
* @param e the exception * @param e the exception
@ -347,6 +372,4 @@ public class MailUtils {
logger.error("Send email to {} failed", receivers, e); logger.error("Send email to {} failed", receivers, e);
retMap.put(Constants.MESSAGE, "Send email to {" + String.join(",", receivers) + "} failed," + e.toString()); retMap.put(Constants.MESSAGE, "Send email to {" + String.join(",", receivers) + "} failed," + e.toString());
} }
} }

55
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/PropertyUtils.java

@ -14,19 +14,21 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.alert.utils; package org.apache.dolphinscheduler.alert.utils;
import static org.apache.dolphinscheduler.alert.utils.Constants.ALERT_PROPERTIES_PATH;
import org.apache.dolphinscheduler.common.utils.IOUtils; import org.apache.dolphinscheduler.common.utils.IOUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.Properties; import java.util.Properties;
import java.util.regex.PatternSyntaxException; import java.util.regex.PatternSyntaxException;
import static org.apache.dolphinscheduler.alert.utils.Constants.ALERT_PROPERTIES_PATH; import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* property utils * property utils
@ -41,13 +43,11 @@ public class PropertyUtils {
private static final Properties properties = new Properties(); private static final Properties properties = new Properties();
private static final PropertyUtils propertyUtils = new PropertyUtils(); private PropertyUtils() {
private PropertyUtils(){
init(); init();
} }
private void init(){ private void init() {
String[] propertyFiles = new String[]{ALERT_PROPERTIES_PATH}; String[] propertyFiles = new String[]{ALERT_PROPERTIES_PATH};
for (String fileName : propertyFiles) { for (String fileName : propertyFiles) {
InputStream fis = null; InputStream fis = null;
@ -69,6 +69,7 @@ public class PropertyUtils {
/** /**
* get property value * get property value
*
* @param key property name * @param key property name
* @return the value * @return the value
*/ */
@ -95,7 +96,7 @@ public class PropertyUtils {
* get property value * get property value
* *
* @param key property name * @param key property name
* @return get property int value , if key == null, then return -1 * @return get property int value , if key == null, then return -1
*/ */
public static int getInt(String key) { public static int getInt(String key) {
@ -104,6 +105,7 @@ public class PropertyUtils {
/** /**
* get int value * get int value
*
* @param key the key * @param key the key
* @param defaultValue the default value * @param defaultValue the default value
* @return the value related the key or the default value if the key not existed * @return the value related the key or the default value if the key not existed
@ -117,15 +119,16 @@ public class PropertyUtils {
try { try {
return Integer.parseInt(value); return Integer.parseInt(value);
} catch (NumberFormatException e) { } catch (NumberFormatException e) {
logger.info(e.getMessage(),e); logger.info(e.getMessage(), e);
} }
return defaultValue; return defaultValue;
} }
/** /**
* get property value * get property value
*
* @param key property name * @param key property name
* @return the boolean result value * @return the boolean result value
*/ */
public static Boolean getBoolean(String key) { public static Boolean getBoolean(String key) {
@ -134,7 +137,7 @@ public class PropertyUtils {
} }
String value = properties.getProperty(key.trim()); String value = properties.getProperty(key.trim());
if(null != value){ if (null != value) {
return Boolean.parseBoolean(value); return Boolean.parseBoolean(value);
} }
@ -143,15 +146,17 @@ public class PropertyUtils {
/** /**
* get long value * get long value
*
* @param key the key * @param key the key
* @return if the value not existed, return -1, or will return the related value * @return if the value not existed, return -1, or will return the related value
*/ */
public static long getLong(String key) { public static long getLong(String key) {
return getLong(key,-1); return getLong(key, -1);
} }
/** /**
* get long value * get long value
*
* @param key the key * @param key the key
* @param defaultVal the default value * @param defaultVal the default value
* @return the value related the key or the default value if the key not existed * @return the value related the key or the default value if the key not existed
@ -166,7 +171,7 @@ public class PropertyUtils {
try { try {
return Long.parseLong(val); return Long.parseLong(val);
} catch (NumberFormatException e) { } catch (NumberFormatException e) {
logger.info(e.getMessage(),e); logger.info(e.getMessage(), e);
} }
return defaultVal; return defaultVal;
@ -174,16 +179,17 @@ public class PropertyUtils {
/** /**
* get double value * get double value
*
* @param key the key * @param key the key
* @return if the value not existed, return -1.0, or will return the related value * @return if the value not existed, return -1.0, or will return the related value
*/ */
public static double getDouble(String key) { public static double getDouble(String key) {
String val = getString(key); return getDouble(key, -1.0);
return getDouble(key,-1.0);
} }
/** /**
* get double value * get double value
*
* @param key the key * @param key the key
* @param defaultVal the default value * @param defaultVal the default value
* @return the value related the key or the default value if the key not existed * @return the value related the key or the default value if the key not existed
@ -198,17 +204,17 @@ public class PropertyUtils {
try { try {
return Double.parseDouble(val); return Double.parseDouble(val);
} catch (NumberFormatException e) { } catch (NumberFormatException e) {
logger.info(e.getMessage(),e); logger.info(e.getMessage(), e);
} }
return defaultVal; return defaultVal;
} }
/** /**
* get array * get array
* @param key property name *
* @param splitStr separator * @param key property name
* @param splitStr separator
* @return the result array * @return the result array
*/ */
public static String[] getArray(String key, String splitStr) { public static String[] getArray(String key, String splitStr) {
@ -219,21 +225,22 @@ public class PropertyUtils {
try { try {
return value.split(splitStr); return value.split(splitStr);
} catch (PatternSyntaxException e) { } catch (PatternSyntaxException e) {
logger.info(e.getMessage(),e); logger.info(e.getMessage(), e);
} }
return null; return null;
} }
/** /**
* get enum * get enum
*
* @param key the key * @param key the key
* @param type the class type * @param type the class type
* @param defaultValue the default value * @param defaultValue the default value
* @param <T> the generic class type * @param <T> the generic class type
* @return get enum value * @return get enum value
*/ */
public static <T extends Enum<T>> T getEnum(String key, Class<T> type, public static <T extends Enum<T>> T getEnum(String key, Class<T> type,
T defaultValue) { T defaultValue) {
String val = getString(key); String val = getString(key);
if (val == null) { if (val == null) {
return defaultValue; return defaultValue;
@ -242,7 +249,7 @@ public class PropertyUtils {
try { try {
return Enum.valueOf(type, val); return Enum.valueOf(type, val);
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
logger.info(e.getMessage(),e); logger.info(e.getMessage(), e);
} }
return defaultValue; return defaultValue;

8
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java

@ -32,8 +32,6 @@ import org.slf4j.LoggerFactory;
import java.io.IOException; import java.io.IOException;
import static org.junit.Assert.*;
@PrepareForTest(PropertyUtils.class) @PrepareForTest(PropertyUtils.class)
@RunWith(PowerMockRunner.class) @RunWith(PowerMockRunner.class)
@PowerMockIgnore("javax.net.ssl.*") @PowerMockIgnore("javax.net.ssl.*")
@ -97,8 +95,8 @@ public class DingTalkUtilsTest {
@Test @Test
public void testProxyConfig() { public void testProxyConfig() {
RequestConfig rc = DingTalkUtils.getProxyConfig(); RequestConfig rc = DingTalkUtils.getProxyConfig();
Assert.assertEquals(rc.getProxy().getPort(), 80); Assert.assertEquals(80, rc.getProxy().getPort());
Assert.assertEquals(rc.getProxy().getHostName(), "proxy.com.cn"); Assert.assertEquals("proxy.com.cn", rc.getProxy().getHostName());
} }
@Test @Test
@ -114,7 +112,7 @@ public class DingTalkUtilsTest {
String msg = DingTalkUtils.textToJsonString("this is test:中文"); String msg = DingTalkUtils.textToJsonString("this is test:中文");
logger.info("test support utf8, actual:" + msg); logger.info("test support utf8, actual:" + msg);
logger.info("test support utf8, actual:" + DingTalkUtils.isEnableDingTalk); logger.info("test support utf8, actual:" + DingTalkUtils.IS_ENABLE_DING_TALK);
String expect = "{\"text\":{\"content\":\"this is test:中文\"},\"msgtype\":\"text\"}"; String expect = "{\"text\":{\"content\":\"this is test:中文\"},\"msgtype\":\"text\"}";
Assert.assertEquals(expect, msg); Assert.assertEquals(expect, msg);
} }

10
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/ExcelUtilsTest.java

@ -17,6 +17,10 @@
package org.apache.dolphinscheduler.alert.utils; package org.apache.dolphinscheduler.alert.utils;
import static org.junit.Assert.assertTrue;
import java.io.File;
import org.junit.After; import org.junit.After;
import org.junit.Before; import org.junit.Before;
import org.junit.Rule; import org.junit.Rule;
@ -25,8 +29,6 @@ import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder; import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.File;
import static org.junit.Assert.assertTrue;
public class ExcelUtilsTest { public class ExcelUtilsTest {
@ -61,7 +63,7 @@ public class ExcelUtilsTest {
//Define dest file path //Define dest file path
String xlsFilePath = rootPath + System.getProperty("file.separator"); String xlsFilePath = rootPath + System.getProperty("file.separator");
logger.info("xlsFilePath: "+xlsFilePath); logger.info("XLS_FILE_PATH: " + xlsFilePath);
//Define correctContent //Define correctContent
String correctContent = "[{\"name\":\"ds name\",\"value\":\"ds value\"}]"; String correctContent = "[{\"name\":\"ds name\",\"value\":\"ds value\"}]";
@ -99,4 +101,4 @@ public class ExcelUtilsTest {
File file = new File("/tmp/xls" + Constants.SINGLE_SLASH + "t" + Constants.EXCEL_SUFFIX_XLS); File file = new File("/tmp/xls" + Constants.SINGLE_SLASH + "t" + Constants.EXCEL_SUFFIX_XLS);
file.delete(); file.delete();
} }
} }

50
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/PropertyUtilsTest.java

@ -41,11 +41,11 @@ public class PropertyUtilsTest {
//Expected "EMAIL" //Expected "EMAIL"
String result = PropertyUtils.getString("alert.type"); String result = PropertyUtils.getString("alert.type");
logger.info(result); logger.info(result);
assertEquals(result, "EMAIL"); assertEquals("EMAIL", result);
//Expected "xxx.xxx.test" //Expected "xxx.xxx.test"
result = PropertyUtils.getString("mail.server.host"); result = PropertyUtils.getString("mail.server.host");
assertEquals(result, "xxx.xxx.test"); assertEquals("xxx.xxx.test", result);
//If key is undefine in alert.properties, then return null //If key is undefine in alert.properties, then return null
result = PropertyUtils.getString("abc"); result = PropertyUtils.getString("abc");
@ -88,23 +88,23 @@ public class PropertyUtilsTest {
//Expected 25 //Expected 25
long result = PropertyUtils.getLong("mail.server.port"); long result = PropertyUtils.getLong("mail.server.port");
assertSame(result, 25L); assertSame(25L, result);
//If key is null, then return -1 //If key is null, then return -1
result = PropertyUtils.getLong(null); result = PropertyUtils.getLong(null);
assertSame(result, -1L); assertSame(-1L, result);
//If key is undefine in alert.properties, then return -1 //If key is undefine in alert.properties, then return -1
result = PropertyUtils.getLong("abc"); result = PropertyUtils.getLong("abc");
assertSame(result, -1L); assertSame(-1L, result);
//If key is undefine in alert.properties, and there is a defaultval, then return defaultval //If key is undefine in alert.properties, and there is a defaultval, then return defaultval
result = PropertyUtils.getLong("abc", 200); result = PropertyUtils.getLong("abc", 200);
assertEquals(result, 200L); assertEquals(200L, result);
//If the value can not parse to long ,it will log the error and return -1L //If the value can not parse to long ,it will log the error and return -1L
result = PropertyUtils.getLong("test.server.testnumber"); result = PropertyUtils.getLong("test.server.testnumber");
assertSame(result, -1L); assertSame(-1L, result);
} }
/** /**
@ -115,23 +115,23 @@ public class PropertyUtilsTest {
//Expected 3.0 //Expected 3.0
double result = PropertyUtils.getDouble("test.server.factor"); double result = PropertyUtils.getDouble("test.server.factor");
assertEquals(result, 3.0, 0); assertEquals(3.0, result, 0);
//If key is null, then return -1.0 //If key is null, then return -1.0
result = PropertyUtils.getDouble(null); result = PropertyUtils.getDouble(null);
assertEquals(result, -1.0, 0); assertEquals(-1.0, result, 0);
//If key is undefine in alert.properties, then return -1 //If key is undefine in alert.properties, then return -1
result = PropertyUtils.getDouble("abc"); result = PropertyUtils.getDouble("abc");
assertEquals(result, -1.0, 0); assertEquals(-1.0, result, 0);
//If key is undefine in alert.properties, and there is a defaultval, then return defaultval //If key is undefine in alert.properties, and there is a defaultval, then return defaultval
result = PropertyUtils.getDouble("abc", 5.0); result = PropertyUtils.getDouble("abc", 5.0);
assertEquals(result, 5.0, 0); assertEquals(5.0, result, 0);
//If the value can not parse to double ,it will log the error and return -1.0 //If the value can not parse to double ,it will log the error and return -1.0
result = PropertyUtils.getDouble("test.server.testnumber"); result = PropertyUtils.getDouble("test.server.testnumber");
assertEquals(result, -1.0, 0); assertEquals(-1.0, result, 0);
} }
/** /**
@ -145,9 +145,9 @@ public class PropertyUtilsTest {
assertEquals(result.length, 3); assertEquals(result.length, 3);
//Equal array values //Equal array values
assertEquals(result[0], "xxx.xxx.test1"); assertEquals("xxx.xxx.test1", result[0]);
assertEquals(result[1], "xxx.xxx.test2"); assertEquals("xxx.xxx.test2", result[1]);
assertEquals(result[2], "xxx.xxx.test3"); assertEquals("xxx.xxx.test3", result[2]);
//If key is null, then return -1 //If key is null, then return -1
result = PropertyUtils.getArray(null, ","); result = PropertyUtils.getArray(null, ",");
@ -170,23 +170,23 @@ public class PropertyUtilsTest {
//Expected 25 //Expected 25
int result = PropertyUtils.getInt("mail.server.port"); int result = PropertyUtils.getInt("mail.server.port");
assertSame(result, 25); assertSame(25, result);
//If key is null, then return -1 //If key is null, then return -1
result = PropertyUtils.getInt(null); result = PropertyUtils.getInt(null);
assertSame(result, -1); assertSame(-1, result);
//If key is undefine in alert.properties, then return -1 //If key is undefine in alert.properties, then return -1
result = PropertyUtils.getInt("abc"); result = PropertyUtils.getInt("abc");
assertSame(result, -1); assertSame(-1, result);
//If key is undefine in alert.properties, and there is a defaultval, then return defaultval //If key is undefine in alert.properties, and there is a defaultval, then return defaultval
result = PropertyUtils.getInt("abc", 300); result = PropertyUtils.getInt("abc", 300);
assertEquals(result, 300); assertEquals(300, result);
//If the value can not parse to int ,it will log the error and return -1 //If the value can not parse to int ,it will log the error and return -1
result = PropertyUtils.getInt("test.server.testnumber"); result = PropertyUtils.getInt("test.server.testnumber");
assertSame(result, -1); assertSame(-1, result);
} }
/** /**
@ -197,19 +197,19 @@ public class PropertyUtilsTest {
//Expected MASTER //Expected MASTER
ZKNodeType zkNodeType = PropertyUtils.getEnum("test.server.enum1", ZKNodeType.class,ZKNodeType.WORKER); ZKNodeType zkNodeType = PropertyUtils.getEnum("test.server.enum1", ZKNodeType.class,ZKNodeType.WORKER);
assertEquals(zkNodeType, ZKNodeType.MASTER); assertEquals(ZKNodeType.MASTER, zkNodeType);
//Expected DEAD_SERVER //Expected DEAD_SERVER
zkNodeType = PropertyUtils.getEnum("test.server.enum2", ZKNodeType.class,ZKNodeType.WORKER); zkNodeType = PropertyUtils.getEnum("test.server.enum2", ZKNodeType.class,ZKNodeType.WORKER);
assertEquals(zkNodeType, ZKNodeType.DEAD_SERVER); assertEquals(ZKNodeType.DEAD_SERVER, zkNodeType);
//If key is null, then return defaultval //If key is null, then return defaultval
zkNodeType = PropertyUtils.getEnum(null, ZKNodeType.class,ZKNodeType.WORKER); zkNodeType = PropertyUtils.getEnum(null, ZKNodeType.class,ZKNodeType.WORKER);
assertEquals(zkNodeType, ZKNodeType.WORKER); assertEquals(ZKNodeType.WORKER, zkNodeType);
//If the value doesn't define in enum ,it will log the error and return -1 //If the value doesn't define in enum ,it will log the error and return -1
zkNodeType = PropertyUtils.getEnum("test.server.enum3", ZKNodeType.class,ZKNodeType.WORKER); zkNodeType = PropertyUtils.getEnum("test.server.enum3", ZKNodeType.class,ZKNodeType.WORKER);
assertEquals(zkNodeType, ZKNodeType.WORKER); assertEquals(ZKNodeType.WORKER, zkNodeType);
} }
} }

63
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java

@ -14,12 +14,20 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.api.controller; package org.apache.dolphinscheduler.api.controller;
import io.swagger.annotations.Api; import static org.apache.dolphinscheduler.api.enums.Status.AUTHORIZED_DATA_SOURCE;
import io.swagger.annotations.ApiImplicitParam; import static org.apache.dolphinscheduler.api.enums.Status.CONNECTION_TEST_FAILURE;
import io.swagger.annotations.ApiImplicitParams; import static org.apache.dolphinscheduler.api.enums.Status.CONNECT_DATASOURCE_FAILURE;
import io.swagger.annotations.ApiOperation; import static org.apache.dolphinscheduler.api.enums.Status.CREATE_DATASOURCE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.DELETE_DATA_SOURCE_FAILURE;
import static org.apache.dolphinscheduler.api.enums.Status.KERBEROS_STARTUP_STATE;
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_DATASOURCE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.UNAUTHORIZED_DATASOURCE;
import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_DATASOURCE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_DATASOURCE_NAME_FAILURE;
import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.exceptions.ApiException;
import org.apache.dolphinscheduler.api.service.DataSourceService; import org.apache.dolphinscheduler.api.service.DataSourceService;
@ -30,16 +38,26 @@ import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import java.util.Map;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.GetMapping;
import springfox.documentation.annotations.ApiIgnore; import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestAttribute;
import java.util.Map; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestController;
import static org.apache.dolphinscheduler.api.enums.Status.*; import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import springfox.documentation.annotations.ApiIgnore;
/** /**
* data source controller * data source controller
@ -101,8 +119,7 @@ public class DataSourceController extends BaseController {
logger.info("login user {} create datasource name: {}, note: {}, type: {}, host: {}, port: {}, database : {}, principal: {}, userName : {}, connectType: {}, other: {}", logger.info("login user {} create datasource name: {}, note: {}, type: {}, host: {}, port: {}, database : {}, principal: {}, userName : {}, connectType: {}, other: {}",
loginUser.getUserName(), name, note, type, host, port, database, principal, userName, connectType, other); loginUser.getUserName(), name, note, type, host, port, database, principal, userName, connectType, other);
String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other); String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other);
Map<String, Object> result = dataSourceService.createDataSource(loginUser, name, note, type, parameter); return dataSourceService.createDataSource(loginUser, name, note, type, parameter);
return returnDataList(result);
} }
@ -156,8 +173,7 @@ public class DataSourceController extends BaseController {
logger.info("login user {} updateProcessInstance datasource name: {}, note: {}, type: {}, connectType: {}, other: {}", logger.info("login user {} updateProcessInstance datasource name: {}, note: {}, type: {}, connectType: {}, other: {}",
loginUser.getUserName(), name, note, type, connectType, other); loginUser.getUserName(), name, note, type, connectType, other);
String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other); String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other);
Map<String, Object> dataSource = dataSourceService.updateDataSource(id, loginUser, name, note, type, parameter); return dataSourceService.updateDataSource(id, loginUser, name, note, type, parameter);
return returnDataList(dataSource);
} }
/** /**
@ -281,15 +297,7 @@ public class DataSourceController extends BaseController {
logger.info("login user {}, connect datasource: {}, note: {}, type: {}, connectType: {}, other: {}", logger.info("login user {}, connect datasource: {}, note: {}, type: {}, connectType: {}, other: {}",
loginUser.getUserName(), name, note, type, connectType, other); loginUser.getUserName(), name, note, type, connectType, other);
String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other); String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other);
Boolean isConnection = dataSourceService.checkConnection(type, parameter); return dataSourceService.checkConnection(type, parameter);
Result result = new Result();
if (isConnection) {
putMsg(result, SUCCESS);
} else {
putMsg(result, CONNECT_DATASOURCE_FAILURE);
}
return result;
} }
/** /**
@ -309,16 +317,7 @@ public class DataSourceController extends BaseController {
public Result connectionTest(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, public Result connectionTest(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam("id") int id) { @RequestParam("id") int id) {
logger.info("connection test, login user:{}, id:{}", loginUser.getUserName(), id); logger.info("connection test, login user:{}, id:{}", loginUser.getUserName(), id);
return dataSourceService.connectionTest(id);
Boolean isConnection = dataSourceService.connectionTest(id);
Result result = new Result();
if (isConnection) {
putMsg(result, SUCCESS);
} else {
putMsg(result, CONNECTION_TEST_FAILURE);
}
return result;
} }
/** /**

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java

@ -198,7 +198,7 @@ public enum Status {
CHECK_OS_TENANT_CODE_ERROR(10164, "Please enter the English os tenant code", "请输入英文操作系统租户"), CHECK_OS_TENANT_CODE_ERROR(10164, "Please enter the English os tenant code", "请输入英文操作系统租户"),
FORCE_TASK_SUCCESS_ERROR(10165, "force task success error", "强制成功任务实例错误"), FORCE_TASK_SUCCESS_ERROR(10165, "force task success error", "强制成功任务实例错误"),
TASK_INSTANCE_STATE_OPERATION_ERROR(10166, "the status of task instance {0} is {1},Cannot perform force success operation", "任务实例[{0}]的状态是[{1}],无法执行强制成功操作"), TASK_INSTANCE_STATE_OPERATION_ERROR(10166, "the status of task instance {0} is {1},Cannot perform force success operation", "任务实例[{0}]的状态是[{1}],无法执行强制成功操作"),
DATASOURCE_TYPE_NOT_EXIST(10167, "data source type not exist", "数据源类型不存在"),
UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found", "UDF函数不存在"), UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found", "UDF函数不存在"),
UDF_FUNCTION_EXISTS(20002, "UDF function already exists", "UDF函数已存在"), UDF_FUNCTION_EXISTS(20002, "UDF function already exists", "UDF函数已存在"),

192
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java

@ -14,12 +14,9 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.api.service; package org.apache.dolphinscheduler.api.service;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
@ -27,27 +24,36 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbConnectType; import org.apache.dolphinscheduler.common.enums.DbConnectType;
import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.datasource.*; import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.datasource.BaseDataSource;
import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory;
import org.apache.dolphinscheduler.dao.datasource.OracleDataSource;
import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper; import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation; import java.sql.Connection;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import java.sql.Connection; import com.baomidou.mybatisplus.core.metadata.IPage;
import java.sql.DriverManager; import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import java.sql.SQLException; import com.fasterxml.jackson.databind.node.ObjectNode;
import java.util.*;
import static org.apache.dolphinscheduler.common.utils.PropertyUtils.getString;
/** /**
* datasource service * datasource service
@ -67,11 +73,9 @@ public class DataSourceService extends BaseService {
public static final String USER_NAME = "userName"; public static final String USER_NAME = "userName";
public static final String OTHER = "other"; public static final String OTHER = "other";
@Autowired @Autowired
private DataSourceMapper dataSourceMapper; private DataSourceMapper dataSourceMapper;
@Autowired @Autowired
private DataSourceUserMapper datasourceUserMapper; private DataSourceUserMapper datasourceUserMapper;
@ -85,24 +89,16 @@ public class DataSourceService extends BaseService {
* @param parameter datasource parameters * @param parameter datasource parameters
* @return create result code * @return create result code
*/ */
public Map<String, Object> createDataSource(User loginUser, String name, String desc, DbType type, String parameter) { public Result<Object> createDataSource(User loginUser, String name, String desc, DbType type, String parameter) {
Map<String, Object> result = new HashMap<>(); Result<Object> result = new Result<>();
// check name can use or not // check name can use or not
if (checkName(name)) { if (checkName(name)) {
putMsg(result, Status.DATASOURCE_EXIST); putMsg(result, Status.DATASOURCE_EXIST);
return result; return result;
} }
Boolean isConnection = checkConnection(type, parameter); Result<Object> isConnection = checkConnection(type, parameter);
if (!isConnection) { if (Status.SUCCESS.getCode() != isConnection.getCode()) {
logger.info("connect failed, type:{}, parameter:{}", type, parameter);
putMsg(result, Status.DATASOURCE_CONNECT_FAILED);
return result;
}
BaseDataSource datasource = DataSourceFactory.getDatasource(type, parameter);
if (datasource == null) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, parameter);
return result; return result;
} }
@ -125,7 +121,6 @@ public class DataSourceService extends BaseService {
return result; return result;
} }
/** /**
* updateProcessInstance datasource * updateProcessInstance datasource
* *
@ -137,9 +132,9 @@ public class DataSourceService extends BaseService {
* @param id data source id * @param id data source id
* @return update result code * @return update result code
*/ */
public Map<String, Object> updateDataSource(int id, User loginUser, String name, String desc, DbType type, String parameter) { public Result<Object> updateDataSource(int id, User loginUser, String name, String desc, DbType type, String parameter) {
Map<String, Object> result = new HashMap<>(); Result<Object> result = new Result<>();
// determine whether the data source exists // determine whether the data source exists
DataSource dataSource = dataSourceMapper.selectById(id); DataSource dataSource = dataSourceMapper.selectById(id);
if (dataSource == null) { if (dataSource == null) {
@ -168,12 +163,11 @@ public class DataSourceService extends BaseService {
// connectionParams json // connectionParams json
String connectionParams = paramObject.toString(); String connectionParams = paramObject.toString();
Boolean isConnection = checkConnection(type, connectionParams); Result<Object> isConnection = checkConnection(type, parameter);
if (!isConnection) { if (Status.SUCCESS.getCode() != isConnection.getCode()) {
logger.info("connect failed, type:{}, parameter:{}", type, parameter);
putMsg(result, Status.DATASOURCE_CONNECT_FAILED);
return result; return result;
} }
Date now = new Date(); Date now = new Date();
dataSource.setName(name.trim()); dataSource.setName(name.trim());
@ -192,7 +186,6 @@ public class DataSourceService extends BaseService {
return queryDataSource != null && queryDataSource.size() > 0; return queryDataSource != null && queryDataSource.size() > 0;
} }
/** /**
* updateProcessInstance datasource * updateProcessInstance datasource
* *
@ -360,8 +353,8 @@ public class DataSourceService extends BaseService {
* @param name datasource name * @param name datasource name
* @return true if data datasource not exists, otherwise return false * @return true if data datasource not exists, otherwise return false
*/ */
public Result verifyDataSourceName(String name) { public Result<Object> verifyDataSourceName(String name) {
Result result = new Result(); Result<Object> result = new Result<>();
List<DataSource> dataSourceList = dataSourceMapper.queryDataSourceByName(name); List<DataSource> dataSourceList = dataSourceMapper.queryDataSourceByName(name);
if (dataSourceList != null && dataSourceList.size() > 0) { if (dataSourceList != null && dataSourceList.size() > 0) {
logger.error("datasource name:{} has exist, can't create again.", name); logger.error("datasource name:{} has exist, can't create again.", name);
@ -373,77 +366,6 @@ public class DataSourceService extends BaseService {
return result; return result;
} }
/**
* get connection
*
* @param dbType datasource type
* @param parameter parameter
* @return connection for datasource
*/
private Connection getConnection(DbType dbType, String parameter) {
Connection connection = null;
BaseDataSource datasource = null;
try {
switch (dbType) {
case POSTGRESQL:
datasource = JSONUtils.parseObject(parameter, PostgreDataSource.class);
Class.forName(Constants.ORG_POSTGRESQL_DRIVER);
break;
case MYSQL:
datasource = JSONUtils.parseObject(parameter, MySQLDataSource.class);
Class.forName(Constants.COM_MYSQL_JDBC_DRIVER);
break;
case HIVE:
case SPARK:
if (CommonUtils.getKerberosStartupState()) {
System.setProperty(org.apache.dolphinscheduler.common.Constants.JAVA_SECURITY_KRB5_CONF,
getString(org.apache.dolphinscheduler.common.Constants.JAVA_SECURITY_KRB5_CONF_PATH));
Configuration configuration = new Configuration();
configuration.set(org.apache.dolphinscheduler.common.Constants.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
UserGroupInformation.setConfiguration(configuration);
UserGroupInformation.loginUserFromKeytab(getString(org.apache.dolphinscheduler.common.Constants.LOGIN_USER_KEY_TAB_USERNAME),
getString(org.apache.dolphinscheduler.common.Constants.LOGIN_USER_KEY_TAB_PATH));
}
if (dbType == DbType.HIVE) {
datasource = JSONUtils.parseObject(parameter, HiveDataSource.class);
} else if (dbType == DbType.SPARK) {
datasource = JSONUtils.parseObject(parameter, SparkDataSource.class);
}
Class.forName(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER);
break;
case CLICKHOUSE:
datasource = JSONUtils.parseObject(parameter, ClickHouseDataSource.class);
Class.forName(Constants.COM_CLICKHOUSE_JDBC_DRIVER);
break;
case ORACLE:
datasource = JSONUtils.parseObject(parameter, OracleDataSource.class);
Class.forName(Constants.COM_ORACLE_JDBC_DRIVER);
break;
case SQLSERVER:
datasource = JSONUtils.parseObject(parameter, SQLServerDataSource.class);
Class.forName(Constants.COM_SQLSERVER_JDBC_DRIVER);
break;
case DB2:
datasource = JSONUtils.parseObject(parameter, DB2ServerDataSource.class);
Class.forName(Constants.COM_DB2_JDBC_DRIVER);
break;
case PRESTO:
datasource = JSONUtils.parseObject(parameter, PrestoDataSource.class);
Class.forName(Constants.COM_PRESTO_JDBC_DRIVER);
break;
default:
break;
}
if (datasource != null) {
connection = DriverManager.getConnection(datasource.getJdbcUrl(), datasource.getUser(), datasource.getPassword());
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
return connection;
}
/** /**
* check connection * check connection
* *
@ -451,18 +373,24 @@ public class DataSourceService extends BaseService {
* @param parameter data source parameters * @param parameter data source parameters
* @return true if connect successfully, otherwise false * @return true if connect successfully, otherwise false
*/ */
public boolean checkConnection(DbType type, String parameter) { public Result<Object> checkConnection(DbType type, String parameter) {
Boolean isConnection = false; Result<Object> result = new Result<>();
Connection con = getConnection(type, parameter); BaseDataSource datasource = DataSourceFactory.getDatasource(type, parameter);
if (con != null) { if (datasource == null) {
isConnection = true; putMsg(result, Status.DATASOURCE_TYPE_NOT_EXIST, type);
try { return result;
con.close(); }
} catch (SQLException e) { try (Connection connection = datasource.getConnection()) {
logger.error("close connection fail at DataSourceService::checkConnection()", e); if (connection == null) {
putMsg(result, Status.CONNECTION_TEST_FAILURE);
return result;
} }
putMsg(result, Status.SUCCESS);
return result;
} catch (Exception e) {
logger.error("datasource test connection error, dbType:{}, jdbcUrl:{}, message:{}.", type, datasource.getJdbcUrl(), e.getMessage());
return new Result<>(Status.CONNECTION_TEST_FAILURE.getCode(),e.getMessage());
} }
return isConnection;
} }
/** /**
@ -471,13 +399,14 @@ public class DataSourceService extends BaseService {
* @param id datasource id * @param id datasource id
* @return connect result code * @return connect result code
*/ */
public boolean connectionTest(int id) { public Result<Object> connectionTest(int id) {
DataSource dataSource = dataSourceMapper.selectById(id); DataSource dataSource = dataSourceMapper.selectById(id);
if (dataSource != null) { if (dataSource == null) {
return checkConnection(dataSource.getType(), dataSource.getConnectionParams()); Result<Object> result = new Result<>();
} else { putMsg(result, Status.RESOURCE_NOT_EXIST);
return false; return result;
} }
return checkConnection(dataSource.getType(), dataSource.getConnectionParams());
} }
/** /**
@ -510,8 +439,8 @@ public class DataSourceService extends BaseService {
parameterMap.put(Constants.ORACLE_DB_CONNECT_TYPE, connectType); parameterMap.put(Constants.ORACLE_DB_CONNECT_TYPE, connectType);
} }
if (CommonUtils.getKerberosStartupState() && if (CommonUtils.getKerberosStartupState()
(type == DbType.HIVE || type == DbType.SPARK)) { && (type == DbType.HIVE || type == DbType.SPARK)) {
jdbcUrl += ";principal=" + principal; jdbcUrl += ";principal=" + principal;
} }
@ -535,8 +464,8 @@ public class DataSourceService extends BaseService {
parameterMap.put(Constants.JDBC_URL, jdbcUrl); parameterMap.put(Constants.JDBC_URL, jdbcUrl);
parameterMap.put(Constants.USER, userName); parameterMap.put(Constants.USER, userName);
parameterMap.put(Constants.PASSWORD, CommonUtils.encodePassword(password)); parameterMap.put(Constants.PASSWORD, CommonUtils.encodePassword(password));
if (CommonUtils.getKerberosStartupState() && if (CommonUtils.getKerberosStartupState()
(type == DbType.HIVE || type == DbType.SPARK)) { && (type == DbType.HIVE || type == DbType.SPARK)) {
parameterMap.put(Constants.PRINCIPAL, principal); parameterMap.put(Constants.PRINCIPAL, principal);
} }
@ -557,7 +486,6 @@ public class DataSourceService extends BaseService {
} }
return JSONUtils.toJsonString(parameterMap); return JSONUtils.toJsonString(parameterMap);
} }
private String buildAddress(DbType type, String host, String port, DbConnectType connectType) { private String buildAddress(DbType type, String host, String port, DbConnectType connectType) {
@ -609,8 +537,8 @@ public class DataSourceService extends BaseService {
* @return delete result code * @return delete result code
*/ */
@Transactional(rollbackFor = RuntimeException.class) @Transactional(rollbackFor = RuntimeException.class)
public Result delete(User loginUser, int datasourceId) { public Result<Object> delete(User loginUser, int datasourceId) {
Result result = new Result(); Result<Object> result = new Result<>();
try { try {
//query datasource by id //query datasource by id
DataSource dataSource = dataSourceMapper.selectById(datasourceId); DataSource dataSource = dataSourceMapper.selectById(datasourceId);
@ -673,7 +601,6 @@ public class DataSourceService extends BaseService {
return result; return result;
} }
/** /**
* authorized datasource * authorized datasource
* *
@ -695,7 +622,6 @@ public class DataSourceService extends BaseService {
return result; return result;
} }
/** /**
* get host and port by address * get host and port by address
* *

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java

@ -72,6 +72,8 @@ public class BaseServiceTest {
} }
@Test @Test
public void testPutMsg(){ public void testPutMsg(){

108
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.api.service; package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.enums.Status;
@ -24,12 +25,19 @@ import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.dao.datasource.BaseDataSource;
import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory;
import org.apache.dolphinscheduler.dao.datasource.MySQLDataSource; import org.apache.dolphinscheduler.dao.datasource.MySQLDataSource;
import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper; import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper;
import java.sql.Connection;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
@ -38,16 +46,15 @@ import org.mockito.Mock;
import org.mockito.Mockito; import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito; import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner; import org.powermock.modules.junit4.PowerMockRunner;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@RunWith(PowerMockRunner.class) @RunWith(PowerMockRunner.class)
@PowerMockIgnore({"sun.security.*", "javax.net.*"}) @PowerMockIgnore({"sun.security.*", "javax.net.*"})
@PrepareForTest({DataSourceFactory.class})
public class DataSourceServiceTest { public class DataSourceServiceTest {
@InjectMocks @InjectMocks
private DataSourceService dataSourceService; private DataSourceService dataSourceService;
@Mock @Mock
@ -69,28 +76,31 @@ public class DataSourceServiceTest {
dataSource.setName(dataSourceName); dataSource.setName(dataSourceName);
dataSourceList.add(dataSource); dataSourceList.add(dataSource);
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(dataSourceList); PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(dataSourceList);
Map<String, Object> dataSourceExitsResult = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); Result dataSourceExitsResult = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.DATASOURCE_EXIST, dataSourceExitsResult.get(Constants.STATUS)); Assert.assertEquals(Status.DATASOURCE_EXIST.getCode(), dataSourceExitsResult.getCode().intValue());
// data source exits // data source exits
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null); PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null);
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(false); Result connectionResult = new Result(Status.DATASOURCE_CONNECT_FAILED.getCode(),Status.DATASOURCE_CONNECT_FAILED.getMsg());
Map<String, Object> connectFailedResult = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); //PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult);
Assert.assertEquals(Status.DATASOURCE_CONNECT_FAILED, connectFailedResult.get(Constants.STATUS)); PowerMockito.doReturn(connectionResult).when(dataSourceService).checkConnection(dataSourceType, parameter);
Result connectFailedResult = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.DATASOURCE_CONNECT_FAILED.getCode(), connectFailedResult.getCode().intValue());
// data source exits // data source exits
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null); PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null);
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(true); connectionResult = new Result(Status.SUCCESS.getCode(),Status.SUCCESS.getMsg());
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult);
PowerMockito.when(DataSourceFactory.getDatasource(dataSourceType, parameter)).thenReturn(null); PowerMockito.when(DataSourceFactory.getDatasource(dataSourceType, parameter)).thenReturn(null);
Map<String, Object> notValidError = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); Result notValidError = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, notValidError.get(Constants.STATUS)); Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR.getCode(), notValidError.getCode().intValue());
// success // success
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null); PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null);
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(true); PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult);
PowerMockito.when(DataSourceFactory.getDatasource(dataSourceType, parameter)).thenReturn(JSONUtils.parseObject(parameter, MySQLDataSource.class)); PowerMockito.when(DataSourceFactory.getDatasource(dataSourceType, parameter)).thenReturn(JSONUtils.parseObject(parameter, MySQLDataSource.class));
Map<String, Object> success = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); Result success = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.SUCCESS, success.get(Constants.STATUS)); Assert.assertEquals(Status.SUCCESS.getCode(), success.getCode().intValue());
} }
public void updateDataSourceTest() { public void updateDataSourceTest() {
@ -104,14 +114,14 @@ public class DataSourceServiceTest {
// data source not exits // data source not exits
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(null); PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(null);
Map<String, Object> resourceNotExits = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); Result resourceNotExits = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.RESOURCE_NOT_EXIST, resourceNotExits.get(Constants.STATUS)); Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getCode(), resourceNotExits.getCode().intValue());
// user no operation perm // user no operation perm
DataSource dataSource = new DataSource(); DataSource dataSource = new DataSource();
dataSource.setUserId(0); dataSource.setUserId(0);
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource); PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource);
Map<String, Object> userNoOperationPerm = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); Result userNoOperationPerm = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, userNoOperationPerm.get(Constants.STATUS)); Assert.assertEquals(Status.USER_NO_OPERATION_PERM.getCode(), userNoOperationPerm.getCode().intValue());
// data source name exits // data source name exits
dataSource.setUserId(-1); dataSource.setUserId(-1);
@ -119,22 +129,24 @@ public class DataSourceServiceTest {
dataSourceList.add(dataSource); dataSourceList.add(dataSource);
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource); PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource);
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(dataSourceList); PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(dataSourceList);
Map<String, Object> dataSourceNameExist = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); Result dataSourceNameExist = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.DATASOURCE_EXIST, dataSourceNameExist.get(Constants.STATUS)); Assert.assertEquals(Status.DATASOURCE_EXIST.getCode(), dataSourceNameExist.getCode().intValue());
// data source connect failed // data source connect failed
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource); PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource);
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(null); PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(null);
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(true); Result connectionResult = new Result(Status.SUCCESS.getCode(),Status.SUCCESS.getMsg());
Map<String, Object> connectFailed = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult);
Assert.assertEquals(Status.DATASOURCE_CONNECT_FAILED, connectFailed.get(Constants.STATUS)); Result connectFailed = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.DATASOURCE_CONNECT_FAILED.getCode(), connectFailed.getCode().intValue());
//success //success
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource); PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource);
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(null); PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(null);
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(false); connectionResult = new Result(Status.DATASOURCE_CONNECT_FAILED.getCode(),Status.DATASOURCE_CONNECT_FAILED.getMsg());
Map<String, Object> success = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult);
Assert.assertEquals(Status.SUCCESS, connectFailed.get(Constants.STATUS)); Result success = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.SUCCESS.getCode(), success.getCode().intValue());
} }
@ -152,7 +164,8 @@ public class DataSourceServiceTest {
public void connectionTest() { public void connectionTest() {
int dataSourceId = -1; int dataSourceId = -1;
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(null); PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(null);
Assert.assertFalse(dataSourceService.connectionTest(dataSourceId)); Result result = dataSourceService.connectionTest(dataSourceId);
Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getCode(),result.getCode().intValue());
} }
@Test @Test
@ -252,7 +265,8 @@ public class DataSourceServiceTest {
dataSource.setName("test"); dataSource.setName("test");
dataSource.setNote("Note"); dataSource.setNote("Note");
dataSource.setType(DbType.ORACLE); dataSource.setType(DbType.ORACLE);
dataSource.setConnectionParams("{\"connectType\":\"ORACLE_SID\",\"address\":\"jdbc:oracle:thin:@192.168.xx.xx:49161\",\"database\":\"XE\",\"jdbcUrl\":\"jdbc:oracle:thin:@192.168.xx.xx:49161/XE\",\"user\":\"system\",\"password\":\"oracle\"}"); dataSource.setConnectionParams("{\"connectType\":\"ORACLE_SID\",\"address\":\"jdbc:oracle:thin:@192.168.xx.xx:49161\",\"database\":\"XE\","
+ "\"jdbcUrl\":\"jdbc:oracle:thin:@192.168.xx.xx:49161/XE\",\"user\":\"system\",\"password\":\"oracle\"}");
return dataSource; return dataSource;
} }
@ -261,7 +275,8 @@ public class DataSourceServiceTest {
public void buildParameter() { public void buildParameter() {
String param = dataSourceService.buildParameter(DbType.ORACLE, "192.168.9.1", "1521", "im" String param = dataSourceService.buildParameter(DbType.ORACLE, "192.168.9.1", "1521", "im"
, "", "test", "test", DbConnectType.ORACLE_SERVICE_NAME, ""); , "", "test", "test", DbConnectType.ORACLE_SERVICE_NAME, "");
String expected = "{\"connectType\":\"ORACLE_SERVICE_NAME\",\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}"; String expected = "{\"connectType\":\"ORACLE_SERVICE_NAME\",\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\","
+ "\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}";
Assert.assertEquals(expected, param); Assert.assertEquals(expected, param);
} }
@ -270,10 +285,10 @@ public class DataSourceServiceTest {
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "true"); PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "true");
String param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im" String param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im"
, "", "test", "123456", null, ""); , "", "test", "123456", null, "");
String expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\"}"; String expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\","
+ "\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\"}";
Assert.assertEquals(expected, param); Assert.assertEquals(expected, param);
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "false"); PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "false");
param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im" param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im"
, "", "test", "123456", null, ""); , "", "test", "123456", null, "");
@ -294,4 +309,31 @@ public class DataSourceServiceTest {
return loginUser; return loginUser;
} }
} /**
* test check connection
* @throws Exception
*/
@Test
public void testCheckConnection() throws Exception {
DbType dataSourceType = DbType.POSTGRESQL;
String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null);
PowerMockito.mockStatic(DataSourceFactory.class);
PowerMockito.when(DataSourceFactory.getDatasource(Mockito.any(), Mockito.anyString())).thenReturn(null);
Result result = dataSourceService.checkConnection(dataSourceType, parameter);
Assert.assertEquals(Status.DATASOURCE_TYPE_NOT_EXIST.getCode(), result.getCode().intValue());
BaseDataSource dataSource = PowerMockito.mock(BaseDataSource.class);
PowerMockito.when(DataSourceFactory.getDatasource(Mockito.any(), Mockito.anyString())).thenReturn(dataSource);
PowerMockito.when(dataSource.getConnection()).thenReturn(null);
result = dataSourceService.checkConnection(dataSourceType, parameter);
Assert.assertEquals(Status.CONNECTION_TEST_FAILURE.getCode(), result.getCode().intValue());
Connection connection = PowerMockito.mock(Connection.class);
PowerMockito.when(dataSource.getConnection()).thenReturn(connection);
result = dataSourceService.checkConnection(dataSourceType, parameter);
Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
}
}

14
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java

@ -33,11 +33,17 @@ public class StringUtils {
return !isEmpty(cs); return !isEmpty(cs);
} }
public static boolean isBlank(String s) { public static boolean isBlank(String str) {
if (isEmpty(s)) { int strLen;
return true; if (str != null && (strLen = str.length()) != 0) {
for (int i = 0; i < strLen; ++i) {
if (!Character.isWhitespace(str.charAt(i))) {
return false;
}
}
} }
return s.trim().length() == 0; return true;
} }
public static boolean isNotBlank(String s) { public static boolean isNotBlank(String s) {

351
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java

@ -14,14 +14,16 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.dao.datasource; package org.apache.dolphinscheduler.dao.datasource;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.StringUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -30,194 +32,183 @@ import org.slf4j.LoggerFactory;
*/ */
public abstract class BaseDataSource { public abstract class BaseDataSource {
private static final Logger logger = LoggerFactory.getLogger(BaseDataSource.class); private static final Logger logger = LoggerFactory.getLogger(BaseDataSource.class);
/** /**
* user name * user name
*/ */
protected String user; protected String user;
/** /**
* user password * user password
*/ */
protected String password; protected String password;
/** /**
* data source address * data source address
*/ */
private String address; private String address;
/** /**
* database name * database name
*/ */
private String database; private String database;
/** /**
* other connection parameters for the data source * other connection parameters for the data source
*/ */
private String other; private String other;
/** /**
* principal * principal
*/ */
private String principal; private String principal;
public String getPrincipal() { public String getPrincipal() {
return principal; return principal;
} }
public void setPrincipal(String principal) { public void setPrincipal(String principal) {
this.principal = principal; this.principal = principal;
} }
/** /**
* @return driver class * @return driver class
*/ */
public abstract String driverClassSelector(); public abstract String driverClassSelector();
/** /**
* @return db type * @return db type
*/ */
public abstract DbType dbTypeSelector(); public abstract DbType dbTypeSelector();
/** /**
* gets the JDBC url for the data source connection * gets the JDBC url for the data source connection
* @return getJdbcUrl * @return getJdbcUrl
*/ */
public String getJdbcUrl() { public String getJdbcUrl() {
StringBuilder jdbcUrl = new StringBuilder(getAddress()); StringBuilder jdbcUrl = new StringBuilder(getAddress());
appendDatabase(jdbcUrl); appendDatabase(jdbcUrl);
appendPrincipal(jdbcUrl); appendPrincipal(jdbcUrl);
appendOther(jdbcUrl); appendOther(jdbcUrl);
return jdbcUrl.toString(); return jdbcUrl.toString();
} }
/** /**
* append database * append database
* @param jdbcUrl jdbc url * @param jdbcUrl jdbc url
*/ */
protected void appendDatabase(StringBuilder jdbcUrl) { protected void appendDatabase(StringBuilder jdbcUrl) {
if (dbTypeSelector() == DbType.SQLSERVER) { if (dbTypeSelector() == DbType.SQLSERVER) {
jdbcUrl.append(";databaseName=").append(getDatabase()); jdbcUrl.append(";databaseName=").append(getDatabase());
} else { } else {
if (getAddress().lastIndexOf('/') != (jdbcUrl.length() - 1)) { if (getAddress().lastIndexOf('/') != (jdbcUrl.length() - 1)) {
jdbcUrl.append("/"); jdbcUrl.append("/");
} }
jdbcUrl.append(getDatabase()); jdbcUrl.append(getDatabase());
} }
} }
/** /**
* append principal * append principal
* @param jdbcUrl jdbc url * @param jdbcUrl jdbc url
*/ */
private void appendPrincipal(StringBuilder jdbcUrl) { private void appendPrincipal(StringBuilder jdbcUrl) {
boolean tag = dbTypeSelector() == DbType.HIVE || dbTypeSelector() == DbType.SPARK; boolean tag = dbTypeSelector() == DbType.HIVE || dbTypeSelector() == DbType.SPARK;
if (tag && StringUtils.isNotEmpty(getPrincipal())) { if (tag && StringUtils.isNotEmpty(getPrincipal())) {
jdbcUrl.append(";principal=").append(getPrincipal()); jdbcUrl.append(";principal=").append(getPrincipal());
}
}
/**
* append other
* @param jdbcUrl jdbc url
*/
private void appendOther(StringBuilder jdbcUrl) {
String otherParams = filterOther(getOther());
if (StringUtils.isNotEmpty(otherParams)) {
String separator = "";
switch (dbTypeSelector()) {
case CLICKHOUSE:
case MYSQL:
case ORACLE:
case POSTGRESQL:
case PRESTO:
separator = "?";
break;
case DB2:
separator = ":";
break;
case HIVE:
case SPARK:
case SQLSERVER:
separator = ";";
break;
default:
logger.error("Db type mismatch!");
}
jdbcUrl.append(separator).append(otherParams);
}
}
protected String filterOther(String otherParams){
return otherParams;
}
/**
* test whether the data source can be connected successfully
*/
public void isConnectable() {
Connection con = null;
try {
Class.forName(driverClassSelector());
con = DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword());
} catch (ClassNotFoundException | SQLException e) {
logger.error("Get connection error: {}", e.getMessage());
} finally {
if (con != null) {
try {
con.close();
} catch (SQLException e) {
logger.error(e.getMessage(), e);
} }
}
} }
}
public String getUser() { /**
return user; * append other
} * @param jdbcUrl jdbc url
*/
private void appendOther(StringBuilder jdbcUrl) {
String otherParams = filterOther(getOther());
if (StringUtils.isNotEmpty(otherParams)) {
String separator = "";
switch (dbTypeSelector()) {
case CLICKHOUSE:
case MYSQL:
case ORACLE:
case POSTGRESQL:
case PRESTO:
separator = "?";
break;
case DB2:
separator = ":";
break;
case HIVE:
case SPARK:
case SQLSERVER:
separator = ";";
break;
default:
logger.error("Db type mismatch!");
}
jdbcUrl.append(separator).append(otherParams);
}
}
/**
* the data source test connection
* @return Connection Connection
* @throws Exception Exception
*/
public Connection getConnection() throws Exception {
Class.forName(driverClassSelector());
return DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword());
}
protected String filterOther(String otherParams) {
return otherParams;
}
public String getUser() {
return user;
}
public void setUser(String user) { public void setUser(String user) {
this.user = user; this.user = user;
} }
/** /**
* password need decode * password need decode
* @return * @return
*/ */
public String getPassword() { public String getPassword() {
return CommonUtils.decodePassword(password); return CommonUtils.decodePassword(password);
} }
public void setPassword(String password) { public void setPassword(String password) {
this.password = password; this.password = password;
} }
public void setAddress(String address) { public void setAddress(String address) {
this.address = address; this.address = address;
} }
public String getAddress() { public String getAddress() {
return address; return address;
} }
public String getDatabase() { public String getDatabase() {
return database; return database;
} }
public void setDatabase(String database) { public void setDatabase(String database) {
this.database = database; this.database = database;
} }
public String getOther() { public String getOther() {
return other; return other;
} }
public void setOther(String other) { public void setOther(String other) {
this.other = other; this.other = other;
} }
} }

124
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java

@ -19,78 +19,92 @@ package org.apache.dolphinscheduler.dao.datasource;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import java.sql.Connection;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
/** /**
* data source of hive * data source of hive
*/ */
public class HiveDataSource extends BaseDataSource { public class HiveDataSource extends BaseDataSource {
/** /**
* gets the JDBC url for the data source connection * gets the JDBC url for the data source connection
* @return jdbc url * @return jdbc url
*/ */
@Override @Override
public String driverClassSelector() { public String driverClassSelector() {
return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER;
}
/**
* @return db type
*/
@Override
public DbType dbTypeSelector() {
return DbType.HIVE;
}
/**
* build hive jdbc params,append : ?hive_conf_list
*
* hive jdbc url template:
*
* jdbc:hive2://<host1>:<port1>,<host2>:<port2>/dbName;initFile=<file>;sess_var_list?hive_conf_list#hive_var_list
*
* @param otherParams otherParams
* @return filter otherParams
*/
@Override
protected String filterOther(String otherParams) {
if (StringUtils.isBlank(otherParams)) {
return "";
} }
StringBuilder hiveConfListSb = new StringBuilder(); /**
hiveConfListSb.append("?"); * @return db type
StringBuilder sessionVarListSb = new StringBuilder(); */
@Override
public DbType dbTypeSelector() {
return DbType.HIVE;
}
String[] otherArray = otherParams.split(";", -1); /**
* build hive jdbc params,append : ?hive_conf_list
*
* hive jdbc url template:
*
* jdbc:hive2://<host1>:<port1>,<host2>:<port2>/dbName;initFile=<file>;sess_var_list?hive_conf_list#hive_var_list
*
* @param otherParams otherParams
* @return filter otherParams
*/
@Override
protected String filterOther(String otherParams) {
if (StringUtils.isBlank(otherParams)) {
return "";
}
// get the default hive conf var name StringBuilder hiveConfListSb = new StringBuilder();
Set<String> hiveConfSet = Stream.of(ConfVars.values()).map(confVars -> confVars.varname) hiveConfListSb.append("?");
.collect(Collectors.toSet()); StringBuilder sessionVarListSb = new StringBuilder();
for (String conf : otherArray) { String[] otherArray = otherParams.split(";", -1);
if (hiveConfSet.contains(conf.split("=")[0])) {
hiveConfListSb.append(conf).append(";");
} else {
sessionVarListSb.append(conf).append(";");
}
}
// remove the last ";" // get the default hive conf var name
if (sessionVarListSb.length() > 0) { Set<String> hiveConfSet = Stream.of(ConfVars.values()).map(confVars -> confVars.varname)
sessionVarListSb.deleteCharAt(sessionVarListSb.length() - 1); .collect(Collectors.toSet());
}
for (String conf : otherArray) {
if (hiveConfSet.contains(conf.split("=")[0])) {
hiveConfListSb.append(conf).append(";");
} else {
sessionVarListSb.append(conf).append(";");
}
}
// remove the last ";"
if (sessionVarListSb.length() > 0) {
sessionVarListSb.deleteCharAt(sessionVarListSb.length() - 1);
}
if (hiveConfListSb.length() > 0) { if (hiveConfListSb.length() > 0) {
hiveConfListSb.deleteCharAt(hiveConfListSb.length() - 1); hiveConfListSb.deleteCharAt(hiveConfListSb.length() - 1);
}
return sessionVarListSb.toString() + hiveConfListSb.toString();
} }
return sessionVarListSb.toString() + hiveConfListSb.toString(); /**
} * the data source test connection
* @return Connection Connection
* @throws Exception Exception
*/
@Override
public Connection getConnection() throws Exception {
CommonUtils.loadKerberosConf();
return super.getConnection();
}
} }

52
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java

@ -14,25 +14,18 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.dao.datasource; package org.apache.dolphinscheduler.dao.datasource;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
/** /**
* data source of SQL Server * data source of SQL Server
*/ */
public class SQLServerDataSource extends BaseDataSource { public class SQLServerDataSource extends BaseDataSource {
private static final Logger logger = LoggerFactory.getLogger(SQLServerDataSource.class);
/** /**
* gets the JDBC url for the data source connection * gets the JDBC url for the data source connection
* @return jdbc url * @return jdbc url
@ -50,39 +43,18 @@ public class SQLServerDataSource extends BaseDataSource {
} }
/** /**
* test whether the data source can be connected successfully * @return driver class
*/ */
@Override @Override
public void isConnectable() { public String driverClassSelector() {
Connection con = null; return Constants.COM_SQLSERVER_JDBC_DRIVER;
try {
Class.forName(Constants.COM_SQLSERVER_JDBC_DRIVER);
con = DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword());
} catch (Exception e) {
logger.error("error", e);
} finally {
if (con != null) {
try {
con.close();
} catch (SQLException e) {
logger.error("SQL Server datasource try conn close conn error", e);
}
}
}
} }
/**
* @return driver class
*/
@Override
public String driverClassSelector() {
return Constants.COM_SQLSERVER_JDBC_DRIVER;
}
/** /**
* @return db type * @return db type
*/ */
@Override @Override
public DbType dbTypeSelector() { public DbType dbTypeSelector() {
return DbType.SQLSERVER; return DbType.SQLSERVER;
} }
} }

45
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java

@ -14,30 +14,45 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.dao.datasource; package org.apache.dolphinscheduler.dao.datasource;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import java.sql.Connection;
/** /**
* data source of spark * data source of spark
*/ */
public class SparkDataSource extends BaseDataSource { public class SparkDataSource extends BaseDataSource {
/** /**
* gets the JDBC url for the data source connection * gets the JDBC url for the data source connection
* @return jdbc url * @return jdbc url
*/ */
@Override @Override
public String driverClassSelector() { public String driverClassSelector() {
return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER;
} }
/**
* @return db type
*/
@Override
public DbType dbTypeSelector() {
return DbType.SPARK;
}
/** /**
* @return db type * the data source test connection
*/ * @return Connection Connection
@Override * @throws Exception Exception
public DbType dbTypeSelector() { */
return DbType.SPARK; @Override
} public Connection getConnection() throws Exception {
CommonUtils.loadKerberosConf();
return super.getConnection();
}
} }

6
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java

@ -687,6 +687,12 @@ public class ProcessService {
processInstance = generateNewProcessInstance(processDefinition, command, cmdParam); processInstance = generateNewProcessInstance(processDefinition, command, cmdParam);
} else { } else {
processInstance = this.findProcessInstanceDetailById(processInstanceId); processInstance = this.findProcessInstanceDetailById(processInstanceId);
// Recalculate global parameters after rerun.
processInstance.setGlobalParams(ParameterUtils.curingGlobalParams(
processDefinition.getGlobalParamMap(),
processDefinition.getGlobalParamList(),
getCommandTypeIfComplement(processInstance, command),
processInstance.getScheduleTime()));
} }
processDefinition = processDefineMapper.selectById(processInstance.getProcessDefinitionId()); processDefinition = processDefineMapper.selectById(processInstance.getProcessDefinitionId());
processInstance.setProcessDefinition(processDefinition); processInstance.setProcessDefinition(processDefinition);

9
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue

@ -74,6 +74,11 @@
</div> </div>
<div class="bottom"> <div class="bottom">
<div class="submit"> <div class="submit">
<template v-if="router.history.current.name === 'projects-definition-details'">
<div class="lint-pt">
<el-checkbox v-model="releaseState" size="small" :false-label="0" :true-label="1">{{$t('Whether to go online the process definition')}}</el-checkbox>
</div>
</template>
<template v-if="router.history.current.name === 'projects-instance-details'"> <template v-if="router.history.current.name === 'projects-instance-details'">
<div class="lint-pt"> <div class="lint-pt">
<el-checkbox v-model="syncDefine" size="small">{{$t('Whether to update the process definition')}}</el-checkbox> <el-checkbox v-model="syncDefine" size="small">{{$t('Whether to update the process definition')}}</el-checkbox>
@ -106,6 +111,8 @@
udpList: [], udpList: [],
// Global custom parameters // Global custom parameters
udpListCache: [], udpListCache: [],
// Whether to go online the process definition
releaseState: 1,
// Whether to update the process definition // Whether to update the process definition
syncDefine: true, syncDefine: true,
// Timeout alarm // Timeout alarm
@ -141,6 +148,7 @@
this.store.commit('dag/setTenantId', _.cloneDeep(this.tenantId)) this.store.commit('dag/setTenantId', _.cloneDeep(this.tenantId))
this.store.commit('dag/setDesc', _.cloneDeep(this.description)) this.store.commit('dag/setDesc', _.cloneDeep(this.description))
this.store.commit('dag/setSyncDefine', this.syncDefine) this.store.commit('dag/setSyncDefine', this.syncDefine)
this.store.commit('dag/setReleaseState', this.releaseState)
}, },
/** /**
* submit * submit
@ -201,6 +209,7 @@
this.originalName = dag.name this.originalName = dag.name
this.description = dag.description this.description = dag.description
this.syncDefine = dag.syncDefine this.syncDefine = dag.syncDefine
this.releaseState = dag.releaseState
this.timeout = dag.timeout || 0 this.timeout = dag.timeout || 0
this.checkedTimeout = this.timeout !== 0 this.checkedTimeout = this.timeout !== 0
this.$nextTick(() => { this.$nextTick(() => {

3
dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js

@ -318,7 +318,8 @@ export default {
connects: JSON.stringify(state.connects), connects: JSON.stringify(state.connects),
name: _.trim(state.name), name: _.trim(state.name),
description: _.trim(state.description), description: _.trim(state.description),
id: payload id: payload,
releaseState: state.releaseState
}, res => { }, res => {
resolve(res) resolve(res)
state.isEditDag = false state.isEditDag = false

3
dolphinscheduler-ui/src/js/conf/home/store/dag/mutations.js

@ -75,6 +75,9 @@ export default {
setDesc (state, payload) { setDesc (state, payload) {
state.description = payload state.description = payload
}, },
setReleaseState (state, payload) {
state.releaseState = payload
},
/** /**
* Whether to update the process definition * Whether to update the process definition
*/ */

2
dolphinscheduler-ui/src/js/conf/home/store/dag/state.js

@ -45,6 +45,8 @@ export default {
isEditDag: false, isEditDag: false,
// Current project // Current project
projectName: projectName || '', projectName: projectName || '',
// Whether to go online the process definition
releaseState: 1,
// Whether to update the process definition // Whether to update the process definition
syncDefine: true, syncDefine: true,
// tasks processList // tasks processList

1
dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js

@ -64,6 +64,7 @@ export default {
'Set the DAG diagram name': 'Set the DAG diagram name', 'Set the DAG diagram name': 'Set the DAG diagram name',
'Please enter description(optional)': 'Please enter description(optional)', 'Please enter description(optional)': 'Please enter description(optional)',
'Set global': 'Set global', 'Set global': 'Set global',
'Whether to go online the process definition': 'Whether to go online the process definition',
'Whether to update the process definition': 'Whether to update the process definition', 'Whether to update the process definition': 'Whether to update the process definition',
Add: 'Add', Add: 'Add',
'DAG graph name cannot be empty': 'DAG graph name cannot be empty', 'DAG graph name cannot be empty': 'DAG graph name cannot be empty',

1
dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js

@ -64,6 +64,7 @@ export default {
'Set the DAG diagram name': '设置DAG图名称', 'Set the DAG diagram name': '设置DAG图名称',
'Please enter description(optional)': '请输入描述(选填)', 'Please enter description(optional)': '请输入描述(选填)',
'Set global': '设置全局', 'Set global': '设置全局',
'Whether to go online the process definition': '是否上线流程定义',
'Whether to update the process definition': '是否更新流程定义', 'Whether to update the process definition': '是否更新流程定义',
Add: '添加', Add: '添加',
'DAG graph name cannot be empty': 'DAG图名称不能为空', 'DAG graph name cannot be empty': 'DAG图名称不能为空',

258
mvnw vendored

@ -34,128 +34,137 @@
# MAVEN_SKIP_RC - flag to disable loading of mavenrc files # MAVEN_SKIP_RC - flag to disable loading of mavenrc files
# ---------------------------------------------------------------------------- # ----------------------------------------------------------------------------
if [ -z "$MAVEN_SKIP_RC" ] ; then if [ -z "$MAVEN_SKIP_RC" ]; then
if [ -f /etc/mavenrc ] ; then if [ -f /etc/mavenrc ]; then
. /etc/mavenrc . /etc/mavenrc
fi fi
if [ -f "$HOME/.mavenrc" ] ; then if [ -f "$HOME/.mavenrc" ]; then
. "$HOME/.mavenrc" . "$HOME/.mavenrc"
fi fi
fi fi
# OS specific support. $var _must_ be set to either true or false. # OS specific support. $var _must_ be set to either true or false.
cygwin=false; cygwin=false
darwin=false; darwin=false
mingw=false mingw=false
case "`uname`" in case "$(uname)" in
CYGWIN*) cygwin=true ;; CYGWIN*) cygwin=true ;;
MINGW*) mingw=true;; MINGW*) mingw=true ;;
Darwin*) darwin=true Darwin*)
# Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home darwin=true
# See https://developer.apple.com/library/mac/qa/qa1170/_index.html # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
if [ -z "$JAVA_HOME" ]; then # See https://developer.apple.com/library/mac/qa/qa1170/_index.html
if [ -x "/usr/libexec/java_home" ]; then if [ -z "$JAVA_HOME" ]; then
export JAVA_HOME="`/usr/libexec/java_home`" if [ -x "/usr/libexec/java_home" ]; then
else JAVA_HOME="$(/usr/libexec/java_home)"
export JAVA_HOME="/Library/Java/Home" export JAVA_HOME
fi else
JAVA_HOME="/Library/Java/Home"
export JAVA_HOME
fi fi
;; fi
;;
esac esac
if [ -z "$JAVA_HOME" ] ; then if [ -z "$JAVA_HOME" ]; then
if [ -r /etc/gentoo-release ] ; then if [ -r /etc/gentoo-release ]; then
JAVA_HOME=`java-config --jre-home` JAVA_HOME=$(java-config --jre-home)
fi fi
fi fi
if [ -z "$M2_HOME" ] ; then if [ -z "$M2_HOME" ]; then
## resolve links - $0 may be a link to maven's home ## resolve links - $0 may be a link to maven's home
PRG="$0" PRG="$0"
# need this for relative symlinks # need this for relative symlinks
while [ -h "$PRG" ] ; do while [ -h "$PRG" ]; do
ls=`ls -ld "$PRG"` ls=$(ls -ld "$PRG")
link=`expr "$ls" : '.*-> \(.*\)$'` link=$(expr "$ls" : '.*-> \(.*\)$')
if expr "$link" : '/.*' > /dev/null; then if expr "$link" : '/.*' >/dev/null; then
PRG="$link" PRG="$link"
else else
PRG="`dirname "$PRG"`/$link" PRG="$(dirname "$PRG")/$link"
fi fi
done done
saveddir=`pwd` saveddir=$(pwd)
M2_HOME=`dirname "$PRG"`/.. M2_HOME=$(dirname "$PRG")/..
# make it fully qualified # make it fully qualified
M2_HOME=`cd "$M2_HOME" && pwd` M2_HOME=$(cd "$M2_HOME" && pwd)
cd "$saveddir" cd "$saveddir" || exit
# echo Using m2 at $M2_HOME # echo Using m2 at $M2_HOME
fi fi
# For Cygwin, ensure paths are in UNIX format before anything is touched # For Cygwin, ensure paths are in UNIX format before anything is touched
if $cygwin ; then if $cygwin; then
[ -n "$M2_HOME" ] && [ -n "$M2_HOME" ] &&
M2_HOME=`cygpath --unix "$M2_HOME"` M2_HOME=$(cygpath --unix "$M2_HOME")
[ -n "$JAVA_HOME" ] && [ -n "$JAVA_HOME" ] &&
JAVA_HOME=`cygpath --unix "$JAVA_HOME"` JAVA_HOME=$(cygpath --unix "$JAVA_HOME")
[ -n "$CLASSPATH" ] && [ -n "$CLASSPATH" ] &&
CLASSPATH=`cygpath --path --unix "$CLASSPATH"` CLASSPATH=$(cygpath --path --unix "$CLASSPATH")
fi fi
# For Mingw, ensure paths are in UNIX format before anything is touched # For Mingw, ensure paths are in UNIX format before anything is touched
if $mingw ; then if $mingw; then
[ -n "$M2_HOME" ] && [ -n "$M2_HOME" ] &&
M2_HOME="`(cd "$M2_HOME"; pwd)`" M2_HOME="$( (
cd "$M2_HOME" || exit
pwd
))"
[ -n "$JAVA_HOME" ] && [ -n "$JAVA_HOME" ] &&
JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" JAVA_HOME="$( (
cd "$JAVA_HOME" || exit
pwd
))"
fi fi
if [ -z "$JAVA_HOME" ]; then if [ -z "$JAVA_HOME" ]; then
javaExecutable="`which javac`" javaExecutable="$(command -v javac)"
if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then if [ -n "$javaExecutable" ] && ! [ "$(expr "$javaExecutable" : '\([^ ]*\)')" = "no" ]; then
# readlink(1) is not available as standard on Solaris 10. # readlink(1) is not available as standard on Solaris 10.
readLink=`which readlink` readLink=$(command -v readlink)
if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then if [ ! "$(expr "$readLink" : '\([^ ]*\)')" = "no" ]; then
if $darwin ; then if $darwin; then
javaHome="`dirname \"$javaExecutable\"`" javaHome="$(dirname "$javaExecutable")"
javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" javaExecutable="$(cd "$javaHome" && pwd -P)/javac"
else else
javaExecutable="`readlink -f \"$javaExecutable\"`" javaExecutable="$(readlink -f "$javaExecutable")"
fi fi
javaHome="`dirname \"$javaExecutable\"`" javaHome="$(dirname "$javaExecutable")"
javaHome=`expr "$javaHome" : '\(.*\)/bin'` javaHome=$(expr "$javaHome" : '\(.*\)/bin')
JAVA_HOME="$javaHome" JAVA_HOME="$javaHome"
export JAVA_HOME export JAVA_HOME
fi fi
fi fi
fi fi
if [ -z "$JAVACMD" ] ; then if [ -z "$JAVACMD" ]; then
if [ -n "$JAVA_HOME" ] ; then if [ -n "$JAVA_HOME" ]; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ]; then
# IBM's JDK on AIX uses strange locations for the executables # IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java" JAVACMD="$JAVA_HOME/jre/sh/java"
else else
JAVACMD="$JAVA_HOME/bin/java" JAVACMD="$JAVA_HOME/bin/java"
fi fi
else else
JAVACMD="`which java`" JAVACMD="$(command -v java)"
fi fi
fi fi
if [ ! -x "$JAVACMD" ] ; then if [ ! -x "$JAVACMD" ]; then
echo "Error: JAVA_HOME is not defined correctly." >&2 echo "Error: JAVA_HOME is not defined correctly." >&2
echo " We cannot execute $JAVACMD" >&2 echo " We cannot execute $JAVACMD" >&2
exit 1 exit 1
fi fi
if [ -z "$JAVA_HOME" ] ; then if [ -z "$JAVA_HOME" ]; then
echo "Warning: JAVA_HOME environment variable is not set." echo "Warning: JAVA_HOME environment variable is not set."
fi fi
@ -165,22 +174,24 @@ CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher
# first directory with .mvn subdirectory is considered project base directory # first directory with .mvn subdirectory is considered project base directory
find_maven_basedir() { find_maven_basedir() {
if [ -z "$1" ] if [ -z "$1" ]; then
then
echo "Path not specified to find_maven_basedir" echo "Path not specified to find_maven_basedir"
return 1 return 1
fi fi
basedir="$1" basedir="$1"
wdir="$1" wdir="$1"
while [ "$wdir" != '/' ] ; do while [ "$wdir" != '/' ]; do
if [ -d "$wdir"/.mvn ] ; then if [ -d "$wdir"/.mvn ]; then
basedir=$wdir basedir=$wdir
break break
fi fi
# workaround for JBEAP-8937 (on Solaris 10/Sparc) # workaround for JBEAP-8937 (on Solaris 10/Sparc)
if [ -d "${wdir}" ]; then if [ -d "${wdir}" ]; then
wdir=`cd "$wdir/.."; pwd` wdir=$(
cd "$wdir/.." || exit
pwd
)
fi fi
# end of workaround # end of workaround
done done
@ -190,13 +201,13 @@ find_maven_basedir() {
# concatenates all lines of a file # concatenates all lines of a file
concat_lines() { concat_lines() {
if [ -f "$1" ]; then if [ -f "$1" ]; then
echo "$(tr -s '\n' ' ' < "$1")" tr -s '\n' ' ' <"$1"
fi fi
} }
BASE_DIR=`find_maven_basedir "$(pwd)"` BASE_DIR=$(find_maven_basedir "$(pwd)")
if [ -z "$BASE_DIR" ]; then if [ -z "$BASE_DIR" ]; then
exit 1; exit 1
fi fi
########################################################################################## ##########################################################################################
@ -204,75 +215,78 @@ fi
# This allows using the maven wrapper in projects that prohibit checking in binary data. # This allows using the maven wrapper in projects that prohibit checking in binary data.
########################################################################################## ##########################################################################################
if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then
if [ "$MVNW_VERBOSE" = true ]; then
echo "Found .mvn/wrapper/maven-wrapper.jar"
fi
else
if [ "$MVNW_VERBOSE" = true ]; then
echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..."
fi
if [ -n "$MVNW_REPOURL" ]; then
jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
else
jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
fi
while IFS="=" read -r key value; do
case "$key" in wrapperUrl)
jarUrl="$value"
break
;;
esac
done <"$BASE_DIR/.mvn/wrapper/maven-wrapper.properties"
if [ "$MVNW_VERBOSE" = true ]; then
echo "Downloading from: $jarUrl"
fi
wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar"
if $cygwin; then
wrapperJarPath=$(cygpath --path --windows "$wrapperJarPath")
fi
if command -v wget >/dev/null; then
if [ "$MVNW_VERBOSE" = true ]; then if [ "$MVNW_VERBOSE" = true ]; then
echo "Found .mvn/wrapper/maven-wrapper.jar" echo "Found wget ... using wget"
fi fi
else if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
wget "$jarUrl" -O "$wrapperJarPath"
else
wget --http-user="$MVNW_USERNAME" --http-password="$MVNW_PASSWORD" "$jarUrl" -O "$wrapperJarPath"
fi
elif command -v curl >/dev/null; then
if [ "$MVNW_VERBOSE" = true ]; then if [ "$MVNW_VERBOSE" = true ]; then
echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." echo "Found curl ... using curl"
fi fi
if [ -n "$MVNW_REPOURL" ]; then if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" curl -o "$wrapperJarPath" "$jarUrl" -f
else else
jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" curl --user "$MVNW_USERNAME":"$MVNW_PASSWORD" -o "$wrapperJarPath" "$jarUrl" -f
fi fi
while IFS="=" read key value; do
case "$key" in (wrapperUrl) jarUrl="$value"; break ;; else
esac
done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties"
if [ "$MVNW_VERBOSE" = true ]; then if [ "$MVNW_VERBOSE" = true ]; then
echo "Downloading from: $jarUrl" echo "Falling back to using Java to download"
fi fi
wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java"
# For Cygwin, switch paths to Windows format before running javac
if $cygwin; then if $cygwin; then
wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"` javaClass=$(cygpath --path --windows "$javaClass")
fi fi
if [ -e "$javaClass" ]; then
if command -v wget > /dev/null; then if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
if [ "$MVNW_VERBOSE" = true ]; then
echo "Found wget ... using wget"
fi
if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
wget "$jarUrl" -O "$wrapperJarPath"
else
wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath"
fi
elif command -v curl > /dev/null; then
if [ "$MVNW_VERBOSE" = true ]; then if [ "$MVNW_VERBOSE" = true ]; then
echo "Found curl ... using curl" echo " - Compiling MavenWrapperDownloader.java ..."
fi
if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
curl -o "$wrapperJarPath" "$jarUrl" -f
else
curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f
fi fi
# Compiling the Java class
else ("$JAVA_HOME/bin/javac" "$javaClass")
fi
if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
# Running the downloader
if [ "$MVNW_VERBOSE" = true ]; then if [ "$MVNW_VERBOSE" = true ]; then
echo "Falling back to using Java to download" echo " - Running MavenWrapperDownloader.java ..."
fi
javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java"
# For Cygwin, switch paths to Windows format before running javac
if $cygwin; then
javaClass=`cygpath --path --windows "$javaClass"`
fi
if [ -e "$javaClass" ]; then
if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
if [ "$MVNW_VERBOSE" = true ]; then
echo " - Compiling MavenWrapperDownloader.java ..."
fi
# Compiling the Java class
("$JAVA_HOME/bin/javac" "$javaClass")
fi
if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
# Running the downloader
if [ "$MVNW_VERBOSE" = true ]; then
echo " - Running MavenWrapperDownloader.java ..."
fi
("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR")
fi
fi fi
("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR")
fi
fi fi
fi
fi fi
########################################################################################## ##########################################################################################
# End of extension # End of extension
@ -280,25 +294,25 @@ fi
export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}
if [ "$MVNW_VERBOSE" = true ]; then if [ "$MVNW_VERBOSE" = true ]; then
echo $MAVEN_PROJECTBASEDIR echo "$MAVEN_PROJECTBASEDIR"
fi fi
MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
# For Cygwin, switch paths to Windows format before running java # For Cygwin, switch paths to Windows format before running java
if $cygwin; then if $cygwin; then
[ -n "$M2_HOME" ] && [ -n "$M2_HOME" ] &&
M2_HOME=`cygpath --path --windows "$M2_HOME"` M2_HOME=$(cygpath --path --windows "$M2_HOME")
[ -n "$JAVA_HOME" ] && [ -n "$JAVA_HOME" ] &&
JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` JAVA_HOME=$(cygpath --path --windows "$JAVA_HOME")
[ -n "$CLASSPATH" ] && [ -n "$CLASSPATH" ] &&
CLASSPATH=`cygpath --path --windows "$CLASSPATH"` CLASSPATH=$(cygpath --path --windows "$CLASSPATH")
[ -n "$MAVEN_PROJECTBASEDIR" ] && [ -n "$MAVEN_PROJECTBASEDIR" ] &&
MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` MAVEN_PROJECTBASEDIR=$(cygpath --path --windows "$MAVEN_PROJECTBASEDIR")
fi fi
# Provide a "standardized" way to retrieve the CLI args that will # Provide a "standardized" way to retrieve the CLI args that will
# work with both Windows and non-Windows executions. # work with both Windows and non-Windows executions.
MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $*"
export MAVEN_CMD_LINE_ARGS export MAVEN_CMD_LINE_ARGS
WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain

Loading…
Cancel
Save