github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Merge branch 'dev' of https://github.com/apache/incubator-dolphinscheduler into dev

pull/3/MERGE
break60 4 years ago
parent
394c5f29d2 91dfe0913f
commit
b1db5dc13a
35 changed files with 987 additions and 907 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 7
      docker/build/hooks/build
  2. 2
      docker/build/hooks/push
  3. 13
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java
  4. 17
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/DingTalkManager.java
  5. 18
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java
  6. 9
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactory.java
  7. 73
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java
  8. 14
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java
  9. 57
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtils.java
  10. 30
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java
  11. 163
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java
  12. 7
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java
  13. 125
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java
  14. 55
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/PropertyUtils.java
  15. 8
      dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java
  16. 10
      dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/ExcelUtilsTest.java
  17. 50
      dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/PropertyUtilsTest.java
  18. 63
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java
  19. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
  20. 192
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java
  21. 2
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java
  22. 108
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java
  23. 14
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java
  24. 351
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java
  25. 124
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java
  26. 52
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java
  27. 45
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java
  28. 6
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
  29. 9
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue
  30. 3
      dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js
  31. 3
      dolphinscheduler-ui/src/js/conf/home/store/dag/mutations.js
  32. 2
      dolphinscheduler-ui/src/js/conf/home/store/dag/state.js
  33. 1
      dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js
  34. 1
      dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js
  35. 258
      mvnw

7
docker/build/hooks/build
Unescape View File

@ -24,7 +24,8 @@ printenv
if [ -z "${VERSION}" ]
then
echo "set default environment variable [VERSION]"
export VERSION=$(cat $(pwd)/pom.xml | grep '<version>' -m 1 | awk '{print $1}' | sed 's/<version>//' | sed 's/<\/version>//')
VERSION=$(grep '<version>' -m 1 "$(pwd)"/pom.xml | awk '{print $1}' | sed 's/<version>//' | sed 's/<\/version>//')
export VERSION
fi
if [ "${DOCKER_REPO}x" = "x" ]
@ -44,10 +45,10 @@ mvn -B clean compile package -Prelease -Dmaven.test.skip=true
# mv dolphinscheduler-bin.tar.gz file to docker/build directory
echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/\n"
mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/
mv "$(pwd)"/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-"${VERSION}"-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/
# docker build
echo -e "docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/\n"
sudo docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/
sudo docker build --build-arg VERSION="${VERSION}" -t $DOCKER_REPO:"${VERSION}" "$(pwd)/docker/build/"
echo "------ dolphinscheduler end - build -------"

2
docker/build/hooks/push
Unescape View File

@ -19,6 +19,6 @@
echo "------ push start -------"
printenv
docker push $DOCKER_REPO:${VERSION}
docker push "$DOCKER_REPO:${VERSION}"
echo "------ push end -------"

13
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java
Unescape View File

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.alert;
import org.apache.dolphinscheduler.alert.plugin.EmailAlertPlugin;
@ -25,11 +26,12 @@ import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.DaoFactory;
import org.apache.dolphinscheduler.dao.entity.Alert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* alert of start
*/
@ -40,8 +42,6 @@ public class AlertServer {
*/
private AlertDao alertDao = DaoFactory.getDaoInstance(AlertDao.class);
private AlertSender alertSender;
private static AlertServer instance;
private FilePluginManager alertPluginManager;
@ -61,7 +61,7 @@ public class AlertServer {
alertPluginManager.addPlugin(new EmailAlertPlugin());
}
public synchronized static AlertServer getInstance() {
public static synchronized AlertServer getInstance() {
if (null == instance) {
instance = new AlertServer();
}
@ -78,12 +78,11 @@ public class AlertServer {
Thread.currentThread().interrupt();
}
List<Alert> alerts = alertDao.listWaitExecutionAlert();
alertSender = new AlertSender(alerts, alertDao, alertPluginManager);
AlertSender alertSender = new AlertSender(alerts, alertDao, alertPluginManager);
alertSender.run();
}
}
public static void main(String[] args) {
AlertServer alertServer = AlertServer.getInstance();
alertServer.start();

17
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/DingTalkManager.java
Unescape View File

@ -14,40 +14,41 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.alert.manager;
import org.apache.dolphinscheduler.alert.utils.Constants;
import org.apache.dolphinscheduler.alert.utils.DingTalkUtils;
import org.apache.dolphinscheduler.plugin.model.AlertInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Ding Talk Manager
*/
public class DingTalkManager {
private static final Logger logger = LoggerFactory.getLogger(EnterpriseWeChatManager.class);
private static final Logger logger = LoggerFactory.getLogger(DingTalkManager.class);
public Map<String,Object> send(AlertInfo alert) {
Map<String,Object> retMap = new HashMap<>();
public Map<String, Object> send(AlertInfo alert) {
Map<String, Object> retMap = new HashMap<>();
retMap.put(Constants.STATUS, false);
logger.info("send message {}", alert.getAlertData().getTitle());
try {
String msg = buildMessage(alert);
DingTalkUtils.sendDingTalkMsg(msg, Constants.UTF_8);
} catch (IOException e) {
logger.error(e.getMessage(),e);
logger.error(e.getMessage(), e);
}
retMap.put(Constants.STATUS, true);
return retMap;
}
private String buildMessage(AlertInfo alert) {
String msg = alert.getAlertData().getContent();
return msg;
return alert.getAlertData().getContent();
}
}

18
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java
Unescape View File

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.alert.plugin;
import org.apache.dolphinscheduler.alert.manager.DingTalkManager;
@ -28,14 +29,19 @@ import org.apache.dolphinscheduler.plugin.api.AlertPlugin;
import org.apache.dolphinscheduler.plugin.model.AlertData;
import org.apache.dolphinscheduler.plugin.model.AlertInfo;
import org.apache.dolphinscheduler.plugin.model.PluginName;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
/**
* EmailAlertPlugin
*
* <p>
* This plugin is a default plugin, and mix up email and enterprise wechat, because adapt with former alert behavior
*/
public class EmailAlertPlugin implements AlertPlugin {
@ -125,10 +131,10 @@ public class EmailAlertPlugin implements AlertPlugin {
}
}
if (DingTalkUtils.isEnableDingTalk) {
if (DingTalkUtils.IS_ENABLE_DING_TALK) {
logger.info("Ding Talk is enable.");
dingTalkManager.send(info);
}
dingTalkManager.send(info);
}
} else {
retMaps.put(Constants.MESSAGE, "alert send error.");

9
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactory.java
Unescape View File

@ -14,23 +14,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.alert.template;
import org.apache.dolphinscheduler.alert.template.impl.DefaultHTMLTemplate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* the alert template factory
*/
public class AlertTemplateFactory {
private static final Logger logger = LoggerFactory.getLogger(AlertTemplateFactory.class);
private AlertTemplateFactory(){}
private AlertTemplateFactory() {
}
/**
* get a template from alert.properties conf file
*
* @return a template, default is DefaultHTMLTemplate
*/
public static AlertTemplate getMessageTemplate() {

73
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java
Unescape View File

@ -14,21 +14,29 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.alert.template.impl;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import static org.apache.dolphinscheduler.common.utils.Preconditions.checkNotNull;
import org.apache.dolphinscheduler.alert.template.AlertTemplate;
import org.apache.dolphinscheduler.alert.utils.Constants;
import org.apache.dolphinscheduler.common.enums.ShowType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.dolphinscheduler.common.utils.*;
import java.util.*;
import static org.apache.dolphinscheduler.common.utils.Preconditions.*;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
/**
* the default html alert message template
@ -37,33 +45,33 @@ public class DefaultHTMLTemplate implements AlertTemplate {
public static final Logger logger = LoggerFactory.getLogger(DefaultHTMLTemplate.class);
@Override
public String getMessageFromTemplate(String content, ShowType showType,boolean showAll) {
public String getMessageFromTemplate(String content, ShowType showType, boolean showAll) {
switch (showType){
switch (showType) {
case TABLE:
return getTableTypeMessage(content,showAll);
return getTableTypeMessage(content, showAll);
case TEXT:
return getTextTypeMessage(content,showAll);
return getTextTypeMessage(content);
default:
throw new IllegalArgumentException(String.format("not support showType: %s in DefaultHTMLTemplate",showType));
throw new IllegalArgumentException(String.format("not support showType: %s in DefaultHTMLTemplate", showType));
}
}
/**
* get alert message which type is TABLE
*
* @param content message content
* @param showAll weather to show all
* @return alert message
*/
private String getTableTypeMessage(String content,boolean showAll){
private String getTableTypeMessage(String content, boolean showAll) {
if (StringUtils.isNotEmpty(content)){
if (StringUtils.isNotEmpty(content)) {
List<LinkedHashMap> mapItemsList = JSONUtils.toList(content, LinkedHashMap.class);
if(!showAll && mapItemsList.size() > Constants.NUMBER_1000){
mapItemsList = mapItemsList.subList(0,Constants.NUMBER_1000);
if (!showAll && mapItemsList.size() > Constants.NUMBER_1000) {
mapItemsList = mapItemsList.subList(0, Constants.NUMBER_1000);
}
StringBuilder contents = new StringBuilder(200);
@ -71,15 +79,15 @@ public class DefaultHTMLTemplate implements AlertTemplate {
boolean flag = true;
String title = "";
for (LinkedHashMap mapItems : mapItemsList){
for (LinkedHashMap mapItems : mapItemsList) {
Set<Map.Entry<String, Object>> entries = mapItems.entrySet();
Set<Entry<String, Object>> entries = mapItems.entrySet();
Iterator<Map.Entry<String, Object>> iterator = entries.iterator();
Iterator<Entry<String, Object>> iterator = entries.iterator();
StringBuilder t = new StringBuilder(Constants.TR);
StringBuilder cs = new StringBuilder(Constants.TR);
while (iterator.hasNext()){
while (iterator.hasNext()) {
Map.Entry<String, Object> entry = iterator.next();
t.append(Constants.TH).append(entry.getKey()).append(Constants.TH_END);
@ -88,14 +96,14 @@ public class DefaultHTMLTemplate implements AlertTemplate {
}
t.append(Constants.TR_END);
cs.append(Constants.TR_END);
if (flag){
if (flag) {
title = t.toString();
}
flag = false;
contents.append(cs);
}
return getMessageFromHtmlTemplate(title,contents.toString());
return getMessageFromHtmlTemplate(title, contents.toString());
}
return content;
@ -103,22 +111,22 @@ public class DefaultHTMLTemplate implements AlertTemplate {
/**
* get alert message which type is TEXT
*
* @param content message content
* @param showAll weather to show all
* @return alert message
*/
private String getTextTypeMessage(String content,boolean showAll){
private String getTextTypeMessage(String content) {
if (StringUtils.isNotEmpty(content)){
if (StringUtils.isNotEmpty(content)) {
ArrayNode list = JSONUtils.parseArray(content);
StringBuilder contents = new StringBuilder(100);
for (JsonNode jsonNode : list){
for (JsonNode jsonNode : list) {
contents.append(Constants.TR);
contents.append(Constants.TD).append(jsonNode.toString()).append(Constants.TD_END);
contents.append(Constants.TR_END);
}
return getMessageFromHtmlTemplate(null,contents.toString());
return getMessageFromHtmlTemplate(null, contents.toString());
}
@ -127,16 +135,17 @@ public class DefaultHTMLTemplate implements AlertTemplate {
/**
* get alert message from a html template
* @param title message title
* @param content message content
*
* @param title message title
* @param content message content
* @return alert message which use html template
*/
private String getMessageFromHtmlTemplate(String title,String content){
private String getMessageFromHtmlTemplate(String title, String content) {
checkNotNull(content);
String htmlTableThead = StringUtils.isEmpty(title) ? "" : String.format("<thead>%s</thead>\n",title);
String htmlTableThead = StringUtils.isEmpty(title) ? "" : String.format("<thead>%s</thead>%n", title);
return Constants.HTML_HEADER_PREFIX +htmlTableThead + content + Constants.TABLE_BODY_HTML_TAIL;
return Constants.HTML_HEADER_PREFIX + htmlTableThead + content + Constants.TABLE_BODY_HTML_TAIL;
}
}

14
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java
Unescape View File

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.alert.utils;
/**
@ -23,6 +24,7 @@ public class Constants {
private Constants() {
throw new IllegalStateException("Constants class");
}
/**
* alert properties path
*/
@ -67,7 +69,7 @@ public class Constants {
public static final String MAIL_SMTP_SSL_ENABLE = "mail.smtp.ssl.enable";
public static final String MAIL_SMTP_SSL_TRUST="mail.smtp.ssl.trust";
public static final String MAIL_SMTP_SSL_TRUST = "mail.smtp.ssl.trust";
public static final String TEXT_HTML_CHARSET_UTF_8 = "text/html;charset=utf-8";
@ -156,7 +158,7 @@ public class Constants {
public static final String ENTERPRISE_WECHAT_AGENT_ID = "enterprise.wechat.agent.id";
public static final String ENTERPRISE_WECHAT_USERS = "enterprise.wechat.users";
public static final String DINGTALK_WEBHOOK = "dingtalk.webhook";
@ -174,7 +176,13 @@ public class Constants {
public static final String DINGTALK_ENABLE = "dingtalk.isEnable";
public static final String HTML_HEADER_PREFIX = "<!DOCTYPE HTML PUBLIC '-//W3C//DTD HTML 4.01 Transitional//EN' 'http://www.w3.org/TR/html4/loose.dtd'><html><head><title>dolphinscheduler</title><meta name='Keywords' content=''><meta name='Description' content=''><style type=\"text/css\">table {margin-top:0px;padding-top:0px;border:1px solid;font-size: 14px;color: #333333;border-width: 1px;border-color: #666666;border-collapse: collapse;}table th {border-width: 1px;padding: 8px;border-style: solid;border-color: #666666;background-color: #dedede;text-align: left;}table td {border-width: 1px;padding: 8px;border-style: solid;border-color: #666666;background-color: #ffffff;text-align: left;}</style></head><body style=\"margin:0;padding:0\"><table border=\"1px\" cellpadding=\"5px\" cellspacing=\"-10px\"> ";
public static final String HTML_HEADER_PREFIX = "<!DOCTYPE HTML PUBLIC '-//W3C//DTD HTML 4.01 Transitional//EN' 'http://www.w3.org/TR/html4/loose.dtd'>"
+ "<html><head><title>dolphinscheduler</title><meta name='Keywords' content=''>"
+ "<meta name='Description' content=''>"
+ "<style type=\"text/css\">table {margin-top:0px;padding-top:0px;border:1px solid;font-size: 14px;color: #333333;border-width: 1px;border-color: #666666;border-collapse: collapse;}"
+ "table th {border-width: 1px;padding: 8px;border-style: solid;border-color: #666666;background-color: #dedede;text-align: left;}"
+ "table td {border-width: 1px;padding: 8px;border-style: solid;border-color: #666666;background-color: #ffffff;text-align: left;}</style>"
+ "/head><body style=\"margin:0;padding:0\"><table border=\"1px\" cellpadding=\"5px\" cellspacing=\"-10px\"> ";
public static final String TABLE_BODY_HTML_TAIL = "</table></body></html>";

57
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtils.java
Unescape View File

@ -14,10 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.alert.utils;
package org.apache.dolphinscheduler.alert.utils;
import org.apache.dolphinscheduler.common.utils.*;
import org.apache.commons.codec.binary.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
@ -32,44 +33,50 @@ import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* DingTalkUtils utils
* support send msg to ding talk by robot message push function.
* support proxy setting
* support PROXY setting
*/
public class DingTalkUtils {
public static final Logger logger = LoggerFactory.getLogger(DingTalkUtils.class);
public static final boolean isEnableDingTalk = PropertyUtils.getBoolean(Constants.DINGTALK_ENABLE);
private static final String dingTaskUrl = PropertyUtils.getString(Constants.DINGTALK_WEBHOOK);
private static final String keyword = PropertyUtils.getString(Constants.DINGTALK_KEYWORD);
private static final Boolean isEnableProxy = PropertyUtils.getBoolean(Constants.DINGTALK_PROXY_ENABLE);
private static final String proxy = PropertyUtils.getString(Constants.DINGTALK_PROXY);
private static final String user = PropertyUtils.getString(Constants.DINGTALK_USER);
private static final String passwd = PropertyUtils.getString(Constants.DINGTALK_PASSWORD);
private static final Integer port = PropertyUtils.getInt(Constants.DINGTALK_PORT);
public static final boolean IS_ENABLE_DING_TALK = PropertyUtils.getBoolean(Constants.DINGTALK_ENABLE);
private static final String DING_TASK_URL = PropertyUtils.getString(Constants.DINGTALK_WEBHOOK);
private static final String KEYWORD = PropertyUtils.getString(Constants.DINGTALK_KEYWORD);
private static final Boolean IS_ENABLE_PROXY = PropertyUtils.getBoolean(Constants.DINGTALK_PROXY_ENABLE);
private static final String PROXY = PropertyUtils.getString(Constants.DINGTALK_PROXY);
private static final String USER = PropertyUtils.getString(Constants.DINGTALK_USER);
private static final String PASSWD = PropertyUtils.getString(Constants.DINGTALK_PASSWORD);
private static final Integer PORT = PropertyUtils.getInt(Constants.DINGTALK_PORT);
private DingTalkUtils() {
throw new IllegalStateException(DingTalkUtils.class.getName());
}
/**
* send message interface
* only support text message format now.
*
* @param msg message context to send
* @param charset charset type
* @return result of sending msg
* @return result of sending msg
* @throws IOException the IOException
*/
public static String sendDingTalkMsg(String msg, String charset) throws IOException {
String msgToJson = textToJsonString(msg + "#" + keyword);
String msgToJson = textToJsonString(msg + "#" + KEYWORD);
HttpPost httpPost = constructHttpPost(msgToJson, charset);
CloseableHttpClient httpClient;
if (isEnableProxy) {
if (Boolean.TRUE.equals(IS_ENABLE_PROXY)) {
httpClient = getProxyClient();
RequestConfig rcf = getProxyConfig();
httpPost.setConfig(rcf);
@ -87,28 +94,26 @@ public class DingTalkUtils {
} finally {
response.close();
}
logger.info("Ding Talk send [{}], resp:{%s}", msg, resp);
logger.info("Ding Talk send [{}], resp:{%s}", msg);
return resp;
} finally {
} finally {
httpClient.close();
}
}
public static HttpPost constructHttpPost(String msg, String charset) {
HttpPost post = new HttpPost(dingTaskUrl);
HttpPost post = new HttpPost(DING_TASK_URL);
StringEntity entity = new StringEntity(msg, charset);
post.setEntity(entity);
post.addHeader("Content-Type", "application/json; charset=utf-8");
return post;
}
public static CloseableHttpClient getProxyClient() {
HttpHost httpProxy = new HttpHost(proxy, port);
HttpHost httpProxy = new HttpHost(PROXY, PORT);
CredentialsProvider provider = new BasicCredentialsProvider();
provider.setCredentials(new AuthScope(httpProxy), new UsernamePasswordCredentials(user, passwd));
CloseableHttpClient httpClient = HttpClients.custom().setDefaultCredentialsProvider(provider).build();
return httpClient;
provider.setCredentials(new AuthScope(httpProxy), new UsernamePasswordCredentials(USER, PASSWD));
return HttpClients.custom().setDefaultCredentialsProvider(provider).build();
}
public static CloseableHttpClient getDefaultClient() {
@ -116,14 +121,14 @@ public class DingTalkUtils {
}
public static RequestConfig getProxyConfig() {
HttpHost httpProxy = new HttpHost(proxy, port);
HttpHost httpProxy = new HttpHost(PROXY, PORT);
return RequestConfig.custom().setProxy(httpProxy).build();
}
public static String textToJsonString(String text) {
Map<String, Object> items = new HashMap<String, Object>();
Map<String, Object> items = new HashMap<>();
items.put("msgtype", "text");
Map<String, String> textContent = new HashMap<String, String>();
Map<String, String> textContent = new HashMap<>();
byte[] byt = StringUtils.getBytesUtf8(text);
String txt = StringUtils.newStringUtf8(byt);
textContent.put("content", txt);

30
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java
Unescape View File

@ -60,9 +60,13 @@ public class EnterpriseWeChatUtils {
private static final String ENTERPRISE_WE_CHAT_TEAM_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TEAM_SEND_MSG);
private static final String ENTERPRISE_WE_CHAT_USER_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USER_SEND_MSG);
private static final String agentIdRegExp = "\\{agentId}";
private static final String msgRegExp = "\\{msg}";
private static final String userRegExp = "\\{toUser}";
private static final String AGENT_ID_REG_EXP = "\\{agentId}";
private static final String MSG_REG_EXP = "\\{msg}";
private static final String USER_REG_EXP = "\\{toUser}";
private EnterpriseWeChatUtils() {
throw new IllegalStateException(EnterpriseWeChatUtils.class.getName());
}
/**
* get Enterprise WeChat is enable
@ -120,8 +124,8 @@ public class EnterpriseWeChatUtils {
*/
public static String makeTeamSendMsg(String toParty, String agentId, String msg) {
return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty}", toParty)
.replaceAll(agentIdRegExp, agentId)
.replaceAll(msgRegExp, msg);
.replaceAll(AGENT_ID_REG_EXP, agentId)
.replaceAll(MSG_REG_EXP, msg);
}
/**
@ -135,8 +139,8 @@ public class EnterpriseWeChatUtils {
public static String makeTeamSendMsg(Collection<String> toParty, String agentId, String msg) {
String listParty = FuncUtils.mkString(toParty, "|");
return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty}", listParty)
.replaceAll(agentIdRegExp, agentId)
.replaceAll(msgRegExp, msg);
.replaceAll(AGENT_ID_REG_EXP, agentId)
.replaceAll(MSG_REG_EXP, msg);
}
/**
@ -148,9 +152,9 @@ public class EnterpriseWeChatUtils {
* @return Enterprise WeChat send message
*/
public static String makeUserSendMsg(String toUser, String agentId, String msg) {
return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\{toUser}", toUser)
.replaceAll(agentIdRegExp, agentId)
.replaceAll(msgRegExp, msg);
return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll(USER_REG_EXP, toUser)
.replaceAll(AGENT_ID_REG_EXP, agentId)
.replaceAll(MSG_REG_EXP, msg);
}
/**
@ -163,9 +167,9 @@ public class EnterpriseWeChatUtils {
*/
public static String makeUserSendMsg(Collection<String> toUser, String agentId, String msg) {
String listUser = FuncUtils.mkString(toUser, "|");
return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll(userRegExp, listUser)
.replaceAll(agentIdRegExp, agentId)
.replaceAll(msgRegExp, msg);
return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll(USER_REG_EXP, listUser)
.replaceAll(AGENT_ID_REG_EXP, agentId)
.replaceAll(MSG_REG_EXP, msg);
}
/**

163
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java
Unescape View File

@ -14,23 +14,29 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.alert.utils;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.ss.usermodel.HorizontalAlignment;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.*;
import org.apache.dolphinscheduler.common.utils.*;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* excel utils
@ -38,19 +44,25 @@ import org.apache.dolphinscheduler.common.utils.*;
public class ExcelUtils {
private static final Logger logger = LoggerFactory.getLogger(ExcelUtils.class);
private ExcelUtils() {
throw new IllegalStateException(ExcelUtils.class.getName());
}
/**
* generate excel file
*
* @param content the content
* @param title the title
* @param xlsFilePath the xls path
*/
public static void genExcelFile(String content,String title,String xlsFilePath){
public static void genExcelFile(String content, String title, String xlsFilePath) {
List<LinkedHashMap> itemsList;
//The JSONUtils.toList has been try catch ex
itemsList = JSONUtils.toList(content, LinkedHashMap.class);
if (CollectionUtils.isEmpty(itemsList)){
if (CollectionUtils.isEmpty(itemsList)) {
logger.error("itemsList is null");
throw new RuntimeException("itemsList is null");
}
@ -59,82 +71,79 @@ public class ExcelUtils {
List<String> headerList = new ArrayList<>();
Iterator<Map.Entry<String, Object>> iter = headerMap.entrySet().iterator();
while (iter.hasNext()){
Map.Entry<String, Object> en = iter.next();
for (Map.Entry<String, Object> en : headerMap.entrySet()) {
headerList.add(en.getKey());
}
HSSFWorkbook wb = null;
FileOutputStream fos = null;
try {
// declare a workbook
wb = new HSSFWorkbook();
// generate a table
HSSFSheet sheet = wb.createSheet();
HSSFRow row = sheet.createRow(0);
//set the height of the first line
row.setHeight((short)500);
//set Horizontal right
CellStyle cellStyle = wb.createCellStyle();
cellStyle.setAlignment(HorizontalAlignment.RIGHT);
//setting excel headers
for (int i = 0; i < headerList.size(); i++) {
HSSFCell cell = row.createCell(i);
cell.setCellStyle(cellStyle);
cell.setCellValue(headerList.get(i));
}
//setting excel body
int rowIndex = 1;
for (LinkedHashMap<String, Object> itemsMap : itemsList){
Object[] values = itemsMap.values().toArray();
row = sheet.createRow(rowIndex);
//setting excel body height
row.setHeight((short)500);
rowIndex++;
for (int j = 0 ; j < values.length ; j++){
HSSFCell cell1 = row.createCell(j);
cell1.setCellStyle(cellStyle);
cell1.setCellValue(String.valueOf(values[j]));
}
}
for (int i = 0; i < headerList.size(); i++) {
sheet.setColumnWidth(i, headerList.get(i).length() * 800);
}
File file = new File(xlsFilePath);
if (!file.exists()) {
file.mkdirs();
}
//setting file output
fos = new FileOutputStream(xlsFilePath + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS);
wb.write(fos);
}catch (Exception e){
logger.error("generate excel error",e);
throw new RuntimeException("generate excel error",e);
}finally {
if (wb != null){
try {
wb.close();
} catch (IOException e) {
logger.error(e.getMessage(),e);
}
}
if (fos != null){
try {
fos.close();
} catch (IOException e) {
logger.error(e.getMessage(),e);
}
}
}
try {
// declare a workbook
wb = new HSSFWorkbook();
// generate a table
HSSFSheet sheet = wb.createSheet();
HSSFRow row = sheet.createRow(0);
//set the height of the first line
row.setHeight((short) 500);
//set Horizontal right
CellStyle cellStyle = wb.createCellStyle();
cellStyle.setAlignment(HorizontalAlignment.RIGHT);
//setting excel headers
for (int i = 0; i < headerList.size(); i++) {
HSSFCell cell = row.createCell(i);
cell.setCellStyle(cellStyle);
cell.setCellValue(headerList.get(i));
}
//setting excel body
int rowIndex = 1;
for (LinkedHashMap<String, Object> itemsMap : itemsList) {
Object[] values = itemsMap.values().toArray();
row = sheet.createRow(rowIndex);
//setting excel body height
row.setHeight((short) 500);
rowIndex++;
for (int j = 0; j < values.length; j++) {
HSSFCell cell1 = row.createCell(j);
cell1.setCellStyle(cellStyle);
cell1.setCellValue(String.valueOf(values[j]));
}
}
for (int i = 0; i < headerList.size(); i++) {
sheet.setColumnWidth(i, headerList.get(i).length() * 800);
}
File file = new File(xlsFilePath);
if (!file.exists()) {
file.mkdirs();
}
//setting file output
fos = new FileOutputStream(xlsFilePath + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS);
wb.write(fos);
} catch (Exception e) {
throw new RuntimeException("generate excel error", e);
} finally {
if (wb != null) {
try {
wb.close();
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
}
if (fos != null) {
try {
fos.close();
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
}
}
}
}

7
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java
Unescape View File

@ -14,15 +14,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.alert.utils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
public class FuncUtils {
private FuncUtils() {
throw new IllegalStateException(FuncUtils.class.getName());
}
public static String mkString(Iterable<String> list, String split) {
if (null == list || StringUtils.isEmpty(split)){
if (null == list || StringUtils.isEmpty(split)) {
return null;
}

125
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java
Unescape View File

@ -14,23 +14,39 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.alert.utils;
import org.apache.dolphinscheduler.alert.template.AlertTemplate;
import org.apache.dolphinscheduler.alert.template.AlertTemplateFactory;
import org.apache.dolphinscheduler.common.enums.ShowType;
import org.apache.commons.mail.EmailException;
import org.apache.commons.mail.HtmlEmail;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.mail.*;
import javax.mail.internet.*;
import java.io.*;
import java.util.*;
import org.apache.commons.mail.EmailException;
import org.apache.commons.mail.HtmlEmail;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import javax.mail.Authenticator;
import javax.mail.Message;
import javax.mail.MessagingException;
import javax.mail.PasswordAuthentication;
import javax.mail.Session;
import javax.mail.Transport;
import javax.mail.internet.InternetAddress;
import javax.mail.internet.MimeBodyPart;
import javax.mail.internet.MimeMessage;
import javax.mail.internet.MimeMultipart;
import javax.mail.internet.MimeUtility;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* mail utils
@ -55,7 +71,7 @@ public class MailUtils {
public static final Boolean MAIL_USE_SSL = PropertyUtils.getBoolean(Constants.MAIL_SMTP_SSL_ENABLE);
public static final String xlsFilePath = PropertyUtils.getString(Constants.XLS_FILE_PATH,"/tmp/xls");
public static final String XLS_FILE_PATH = PropertyUtils.getString(Constants.XLS_FILE_PATH, "/tmp/xls");
public static final String STARTTLS_ENABLE = PropertyUtils.getString(Constants.MAIL_SMTP_STARTTLS_ENABLE);
@ -67,23 +83,29 @@ public class MailUtils {
//Solve the problem of messy Chinese name in excel attachment
static {
System.setProperty("mail.mime.splitlongparameters","false");
System.setProperty("mail.mime.splitlongparameters", "false");
}
private MailUtils() {
throw new IllegalStateException(MailUtils.class.getName());
}
/**
* send mail to receivers
*
* @param receivers the receiver list
* @param title the title
* @param content the content
* @param showType the show type
* @return the result map
*/
public static Map<String,Object> sendMails(Collection<String> receivers, String title, String content,String showType) {
public static Map<String, Object> sendMails(Collection<String> receivers, String title, String content, String showType) {
return sendMails(receivers, null, title, content, showType);
}
/**
* send mail
*
* @param receivers the receiver list
* @param receiversCc cc list
* @param title the title
@ -91,8 +113,8 @@ public class MailUtils {
* @param showType the show type
* @return the send result
*/
public static Map<String,Object> sendMails(Collection<String> receivers, Collection<String> receiversCc, String title, String content, String showType) {
Map<String,Object> retMap = new HashMap<>();
public static Map<String, Object> sendMails(Collection<String> receivers, Collection<String> receiversCc, String title, String content, String showType) {
Map<String, Object> retMap = new HashMap<>();
retMap.put(Constants.STATUS, false);
// if there is no receivers && no receiversCc, no need to process
@ -111,14 +133,14 @@ public class MailUtils {
email.setMailSession(session);
email.setFrom(MAIL_SENDER);
email.setCharset(Constants.UTF_8);
if (CollectionUtils.isNotEmpty(receivers)){
if (CollectionUtils.isNotEmpty(receivers)) {
// receivers mail
for (String receiver : receivers) {
email.addTo(receiver);
}
}
if (CollectionUtils.isNotEmpty(receiversCc)){
if (CollectionUtils.isNotEmpty(receiversCc)) {
//cc
for (String receiverCc : receiversCc) {
email.addCc(receiverCc);
@ -129,16 +151,16 @@ public class MailUtils {
} catch (Exception e) {
handleException(receivers, retMap, e);
}
}else if (showType.equals(ShowType.ATTACHMENT.getDescp()) || showType.equals(ShowType.TABLEATTACHMENT.getDescp())) {
} else if (showType.equals(ShowType.ATTACHMENT.getDescp()) || showType.equals(ShowType.TABLEATTACHMENT.getDescp())) {
try {
String partContent = (showType.equals(ShowType.ATTACHMENT.getDescp()) ? "Please see the attachment " + title + Constants.EXCEL_SUFFIX_XLS : htmlTable(content,false));
String partContent = (showType.equals(ShowType.ATTACHMENT.getDescp()) ? "Please see the attachment " + title + Constants.EXCEL_SUFFIX_XLS : htmlTable(content, false));
attachment(receivers,receiversCc,title,content,partContent);
attachment(receivers, receiversCc, title, content, partContent);
retMap.put(Constants.STATUS, true);
return retMap;
}catch (Exception e){
} catch (Exception e) {
handleException(receivers, retMap, e);
return retMap;
}
@ -149,49 +171,52 @@ public class MailUtils {
/**
* html table content
*
* @param content the content
* @param showAll if show the whole content
* @return the html table form
*/
private static String htmlTable(String content, boolean showAll){
return alertTemplate.getMessageFromTemplate(content,ShowType.TABLE,showAll);
private static String htmlTable(String content, boolean showAll) {
return alertTemplate.getMessageFromTemplate(content, ShowType.TABLE, showAll);
}
/**
* html table content
*
* @param content the content
* @return the html table form
*/
private static String htmlTable(String content){
return htmlTable(content,true);
private static String htmlTable(String content) {
return htmlTable(content, true);
}
/**
* html text content
*
* @param content the content
* @return text in html form
*/
private static String htmlText(String content){
return alertTemplate.getMessageFromTemplate(content,ShowType.TEXT);
private static String htmlText(String content) {
return alertTemplate.getMessageFromTemplate(content, ShowType.TEXT);
}
/**
* send mail as Excel attachment
*
* @param receivers the receiver list
* @param title the title
* @throws Exception
*/
private static void attachment(Collection<String> receivers,Collection<String> receiversCc,String title,String content,String partContent)throws Exception{
private static void attachment(Collection<String> receivers, Collection<String> receiversCc, String title, String content, String partContent) throws Exception {
MimeMessage msg = getMimeMessage(receivers);
attachContent(receiversCc, title, content,partContent, msg);
attachContent(receiversCc, title, content, partContent, msg);
}
/**
* get MimeMessage
*
* @param receivers receivers
* @return the MimeMessage
* @throws MessagingException
*/
private static MimeMessage getMimeMessage(Collection<String> receivers) throws MessagingException {
@ -223,7 +248,7 @@ public class MailUtils {
props.setProperty(Constants.MAIL_SMTP_AUTH, Constants.STRING_TRUE);
props.setProperty(Constants.MAIL_TRANSPORT_PROTOCOL, MAIL_PROTOCOL);
props.setProperty(Constants.MAIL_SMTP_STARTTLS_ENABLE, STARTTLS_ENABLE);
if (SSL_ENABLE) {
if (Boolean.TRUE.equals(SSL_ENABLE)) {
props.setProperty(Constants.MAIL_SMTP_SSL_ENABLE, "true");
props.setProperty(Constants.MAIL_SMTP_SSL_TRUST, SSL_TRUST);
}
@ -241,20 +266,19 @@ public class MailUtils {
/**
* attach content
*
* @param receiversCc the cc list
* @param title the title
* @param content the content
* @param partContent the partContent
* @param msg the message
* @throws MessagingException
* @throws IOException
*/
private static void attachContent(Collection<String> receiversCc, String title, String content, String partContent,MimeMessage msg) throws MessagingException, IOException {
/**
private static void attachContent(Collection<String> receiversCc, String title, String content, String partContent, MimeMessage msg) throws MessagingException, IOException {
/*
* set receiverCc
*/
if(CollectionUtils.isNotEmpty(receiversCc)){
for (String receiverCc : receiversCc){
if (CollectionUtils.isNotEmpty(receiversCc)) {
for (String receiverCc : receiversCc) {
msg.addRecipients(Message.RecipientType.CC, InternetAddress.parse(receiverCc));
}
}
@ -267,16 +291,16 @@ public class MailUtils {
part1.setContent(partContent, Constants.TEXT_HTML_CHARSET_UTF_8);
// set attach file
MimeBodyPart part2 = new MimeBodyPart();
File file = new File(xlsFilePath + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS);
File file = new File(XLS_FILE_PATH + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS);
if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs();
}
// make excel file
ExcelUtils.genExcelFile(content,title,xlsFilePath);
ExcelUtils.genExcelFile(content, title, XLS_FILE_PATH);
part2.attachFile(file);
part2.setFileName(MimeUtility.encodeText(title + Constants.EXCEL_SUFFIX_XLS,Constants.UTF_8,"B"));
part2.setFileName(MimeUtility.encodeText(title + Constants.EXCEL_SUFFIX_XLS, Constants.UTF_8, "B"));
// add components to collection
partList.addBodyPart(part1);
partList.addBodyPart(part2);
@ -289,21 +313,21 @@ public class MailUtils {
/**
* the string object map
*
* @param title the title
* @param content the content
* @param showType the showType
* @param retMap the result map
* @param email the email
* @return the result map
* @throws EmailException
*/
private static Map<String, Object> getStringObjectMap(String title, String content, String showType, Map<String, Object> retMap, HtmlEmail email) throws EmailException {
/**
/*
* the subject of the message to be sent
*/
email.setSubject(title);
/**
/*
* to send information, you can use HTML tags in mail content because of the use of HtmlEmail
*/
if (showType.equals(ShowType.TABLE.getDescp())) {
@ -322,23 +346,24 @@ public class MailUtils {
/**
* file delete
*
* @param file the file to delete
*/
public static void deleteFile(File file){
if(file.exists()){
if(file.delete()){
logger.info("delete success: {}",file.getAbsolutePath() + file.getName());
}else{
public static void deleteFile(File file) {
if (file.exists()) {
if (file.delete()) {
logger.info("delete success: {}", file.getAbsolutePath() + file.getName());
} else {
logger.info("delete fail: {}", file.getAbsolutePath() + file.getName());
}
}else{
} else {
logger.info("file not exists: {}", file.getAbsolutePath() + file.getName());
}
}
/**
* handle exception
*
* @param receivers the receiver list
* @param retMap the result map
* @param e the exception
@ -347,6 +372,4 @@ public class MailUtils {
logger.error("Send email to {} failed", receivers, e);
retMap.put(Constants.MESSAGE, "Send email to {" + String.join(",", receivers) + "} failed，" + e.toString());
}
}

55
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/PropertyUtils.java
Unescape View File

@ -14,19 +14,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.alert.utils;
import static org.apache.dolphinscheduler.alert.utils.Constants.ALERT_PROPERTIES_PATH;
import org.apache.dolphinscheduler.common.utils.IOUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import java.util.regex.PatternSyntaxException;
import static org.apache.dolphinscheduler.alert.utils.Constants.ALERT_PROPERTIES_PATH;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* property utils
@ -41,13 +43,11 @@ public class PropertyUtils {
private static final Properties properties = new Properties();
private static final PropertyUtils propertyUtils = new PropertyUtils();
private PropertyUtils(){
private PropertyUtils() {
init();
}
private void init(){
private void init() {
String[] propertyFiles = new String[]{ALERT_PROPERTIES_PATH};
for (String fileName : propertyFiles) {
InputStream fis = null;
@ -69,6 +69,7 @@ public class PropertyUtils {
/**
* get property value
*
* @param key property name
* @return the value
*/
@ -95,7 +96,7 @@ public class PropertyUtils {
* get property value
*
* @param key property name
* @return get property int value , if key == null, then return -1
* @return get property int value , if key == null, then return -1
*/
public static int getInt(String key) {
@ -104,6 +105,7 @@ public class PropertyUtils {
/**
* get int value
*
* @param key the key
* @param defaultValue the default value
* @return the value related the key or the default value if the key not existed
@ -117,15 +119,16 @@ public class PropertyUtils {
try {
return Integer.parseInt(value);
} catch (NumberFormatException e) {
logger.info(e.getMessage(),e);
logger.info(e.getMessage(), e);
}
return defaultValue;
}
/**
* get property value
*
* @param key property name
* @return the boolean result value
* @return the boolean result value
*/
public static Boolean getBoolean(String key) {
@ -134,7 +137,7 @@ public class PropertyUtils {
}
String value = properties.getProperty(key.trim());
if(null != value){
if (null != value) {
return Boolean.parseBoolean(value);
}
@ -143,15 +146,17 @@ public class PropertyUtils {
/**
* get long value
*
* @param key the key
* @return if the value not existed, return -1, or will return the related value
*/
public static long getLong(String key) {
return getLong(key,-1);
return getLong(key, -1);
}
/**
* get long value
*
* @param key the key
* @param defaultVal the default value
* @return the value related the key or the default value if the key not existed
@ -166,7 +171,7 @@ public class PropertyUtils {
try {
return Long.parseLong(val);
} catch (NumberFormatException e) {
logger.info(e.getMessage(),e);
logger.info(e.getMessage(), e);
}
return defaultVal;
@ -174,16 +179,17 @@ public class PropertyUtils {
/**
* get double value
*
* @param key the key
* @return if the value not existed, return -1.0, or will return the related value
*/
public static double getDouble(String key) {
String val = getString(key);
return getDouble(key,-1.0);
return getDouble(key, -1.0);
}
/**
* get double value
*
* @param key the key
* @param defaultVal the default value
* @return the value related the key or the default value if the key not existed
@ -198,17 +204,17 @@ public class PropertyUtils {
try {
return Double.parseDouble(val);
} catch (NumberFormatException e) {
logger.info(e.getMessage(),e);
logger.info(e.getMessage(), e);
}
return defaultVal;
}
/**
* get array
* @param key property name
* @param splitStr separator
* get array
*
* @param key property name
* @param splitStr separator
* @return the result array
*/
public static String[] getArray(String key, String splitStr) {
@ -219,21 +225,22 @@ public class PropertyUtils {
try {
return value.split(splitStr);
} catch (PatternSyntaxException e) {
logger.info(e.getMessage(),e);
logger.info(e.getMessage(), e);
}
return null;
}
/**
* get enum
*
* @param key the key
* @param type the class type
* @param defaultValue the default value
* @param <T> the generic class type
* @return get enum value
* @return get enum value
*/
public static <T extends Enum<T>> T getEnum(String key, Class<T> type,
T defaultValue) {
T defaultValue) {
String val = getString(key);
if (val == null) {
return defaultValue;
@ -242,7 +249,7 @@ public class PropertyUtils {
try {
return Enum.valueOf(type, val);
} catch (IllegalArgumentException e) {
logger.info(e.getMessage(),e);
logger.info(e.getMessage(), e);
}
return defaultValue;

8
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java
Unescape View File

@ -32,8 +32,6 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import static org.junit.Assert.*;
@PrepareForTest(PropertyUtils.class)
@RunWith(PowerMockRunner.class)
@PowerMockIgnore("javax.net.ssl.*")
@ -97,8 +95,8 @@ public class DingTalkUtilsTest {
@Test
public void testProxyConfig() {
RequestConfig rc = DingTalkUtils.getProxyConfig();
Assert.assertEquals(rc.getProxy().getPort(), 80);
Assert.assertEquals(rc.getProxy().getHostName(), "proxy.com.cn");
Assert.assertEquals(80, rc.getProxy().getPort());
Assert.assertEquals("proxy.com.cn", rc.getProxy().getHostName());
}
@Test
@ -114,7 +112,7 @@ public class DingTalkUtilsTest {
String msg = DingTalkUtils.textToJsonString("this is test:中文");
logger.info("test support utf8, actual:" + msg);
logger.info("test support utf8, actual:" + DingTalkUtils.isEnableDingTalk);
logger.info("test support utf8, actual:" + DingTalkUtils.IS_ENABLE_DING_TALK);
String expect = "{\"text\":{\"content\":\"this is test:中文\"},\"msgtype\":\"text\"}";
Assert.assertEquals(expect, msg);
}

10
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/ExcelUtilsTest.java
Unescape View File

@ -17,6 +17,10 @@
package org.apache.dolphinscheduler.alert.utils;
import static org.junit.Assert.assertTrue;
import java.io.File;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
@ -25,8 +29,6 @@ import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import static org.junit.Assert.assertTrue;
public class ExcelUtilsTest {
@ -61,7 +63,7 @@ public class ExcelUtilsTest {
//Define dest file path
String xlsFilePath = rootPath + System.getProperty("file.separator");
logger.info("xlsFilePath: "+xlsFilePath);
logger.info("XLS_FILE_PATH: " + xlsFilePath);
//Define correctContent
String correctContent = "[{\"name\":\"ds name\",\"value\":\"ds value\"}]";
@ -99,4 +101,4 @@ public class ExcelUtilsTest {
File file = new File("/tmp/xls" + Constants.SINGLE_SLASH + "t" + Constants.EXCEL_SUFFIX_XLS);
file.delete();
}
}
}

50
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/PropertyUtilsTest.java
Unescape View File

@ -41,11 +41,11 @@ public class PropertyUtilsTest {
//Expected "EMAIL"
String result = PropertyUtils.getString("alert.type");
logger.info(result);
assertEquals(result, "EMAIL");
assertEquals("EMAIL", result);
//Expected "xxx.xxx.test"
result = PropertyUtils.getString("mail.server.host");
assertEquals(result, "xxx.xxx.test");
assertEquals("xxx.xxx.test", result);
//If key is undefine in alert.properties, then return null
result = PropertyUtils.getString("abc");
@ -88,23 +88,23 @@ public class PropertyUtilsTest {
//Expected 25
long result = PropertyUtils.getLong("mail.server.port");
assertSame(result, 25L);
assertSame(25L, result);
//If key is null, then return -1
result = PropertyUtils.getLong(null);
assertSame(result, -1L);
assertSame(-1L, result);
//If key is undefine in alert.properties, then return -1
result = PropertyUtils.getLong("abc");
assertSame(result, -1L);
assertSame(-1L, result);
//If key is undefine in alert.properties, and there is a defaultval, then return defaultval
result = PropertyUtils.getLong("abc", 200);
assertEquals(result, 200L);
assertEquals(200L, result);
//If the value can not parse to long ,it will log the error and return -1L
result = PropertyUtils.getLong("test.server.testnumber");
assertSame(result, -1L);
assertSame(-1L, result);
}
/**
@ -115,23 +115,23 @@ public class PropertyUtilsTest {
//Expected 3.0
double result = PropertyUtils.getDouble("test.server.factor");
assertEquals(result, 3.0, 0);
assertEquals(3.0, result, 0);
//If key is null, then return -1.0
result = PropertyUtils.getDouble(null);
assertEquals(result, -1.0, 0);
assertEquals(-1.0, result, 0);
//If key is undefine in alert.properties, then return -1
result = PropertyUtils.getDouble("abc");
assertEquals(result, -1.0, 0);
assertEquals(-1.0, result, 0);
//If key is undefine in alert.properties, and there is a defaultval, then return defaultval
result = PropertyUtils.getDouble("abc", 5.0);
assertEquals(result, 5.0, 0);
assertEquals(5.0, result, 0);
//If the value can not parse to double ,it will log the error and return -1.0
result = PropertyUtils.getDouble("test.server.testnumber");
assertEquals(result, -1.0, 0);
assertEquals(-1.0, result, 0);
}
/**
@ -145,9 +145,9 @@ public class PropertyUtilsTest {
assertEquals(result.length, 3);
//Equal array values
assertEquals(result[0], "xxx.xxx.test1");
assertEquals(result[1], "xxx.xxx.test2");
assertEquals(result[2], "xxx.xxx.test3");
assertEquals("xxx.xxx.test1", result[0]);
assertEquals("xxx.xxx.test2", result[1]);
assertEquals("xxx.xxx.test3", result[2]);
//If key is null, then return -1
result = PropertyUtils.getArray(null, ",");
@ -170,23 +170,23 @@ public class PropertyUtilsTest {
//Expected 25
int result = PropertyUtils.getInt("mail.server.port");
assertSame(result, 25);
assertSame(25, result);
//If key is null, then return -1
result = PropertyUtils.getInt(null);
assertSame(result, -1);
assertSame(-1, result);
//If key is undefine in alert.properties, then return -1
result = PropertyUtils.getInt("abc");
assertSame(result, -1);
assertSame(-1, result);
//If key is undefine in alert.properties, and there is a defaultval, then return defaultval
result = PropertyUtils.getInt("abc", 300);
assertEquals(result, 300);
assertEquals(300, result);
//If the value can not parse to int ,it will log the error and return -1
result = PropertyUtils.getInt("test.server.testnumber");
assertSame(result, -1);
assertSame(-1, result);
}
/**
@ -197,19 +197,19 @@ public class PropertyUtilsTest {
//Expected MASTER
ZKNodeType zkNodeType = PropertyUtils.getEnum("test.server.enum1", ZKNodeType.class,ZKNodeType.WORKER);
assertEquals(zkNodeType, ZKNodeType.MASTER);
assertEquals(ZKNodeType.MASTER, zkNodeType);
//Expected DEAD_SERVER
zkNodeType = PropertyUtils.getEnum("test.server.enum2", ZKNodeType.class,ZKNodeType.WORKER);
assertEquals(zkNodeType, ZKNodeType.DEAD_SERVER);
assertEquals(ZKNodeType.DEAD_SERVER, zkNodeType);
//If key is null, then return defaultval
zkNodeType = PropertyUtils.getEnum(null, ZKNodeType.class,ZKNodeType.WORKER);
assertEquals(zkNodeType, ZKNodeType.WORKER);
assertEquals(ZKNodeType.WORKER, zkNodeType);
//If the value doesn't define in enum ,it will log the error and return -1
zkNodeType = PropertyUtils.getEnum("test.server.enum3", ZKNodeType.class,ZKNodeType.WORKER);
assertEquals(zkNodeType, ZKNodeType.WORKER);
assertEquals(ZKNodeType.WORKER, zkNodeType);
}
}
}

63
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java
Unescape View File

@ -14,12 +14,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.controller;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import static org.apache.dolphinscheduler.api.enums.Status.AUTHORIZED_DATA_SOURCE;
import static org.apache.dolphinscheduler.api.enums.Status.CONNECTION_TEST_FAILURE;
import static org.apache.dolphinscheduler.api.enums.Status.CONNECT_DATASOURCE_FAILURE;
import static org.apache.dolphinscheduler.api.enums.Status.CREATE_DATASOURCE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.DELETE_DATA_SOURCE_FAILURE;
import static org.apache.dolphinscheduler.api.enums.Status.KERBEROS_STARTUP_STATE;
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_DATASOURCE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.UNAUTHORIZED_DATASOURCE;
import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_DATASOURCE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_DATASOURCE_NAME_FAILURE;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ApiException;
import org.apache.dolphinscheduler.api.service.DataSourceService;
@ -30,16 +38,26 @@ import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.dao.entity.User;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.*;
import springfox.documentation.annotations.ApiIgnore;
import java.util.Map;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestAttribute;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestController;
import static org.apache.dolphinscheduler.api.enums.Status.*;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import springfox.documentation.annotations.ApiIgnore;
/**
* data source controller
@ -101,8 +119,7 @@ public class DataSourceController extends BaseController {
logger.info("login user {} create datasource name: {}, note: {}, type: {}, host: {}, port: {}, database : {}, principal: {}, userName : {}, connectType: {}, other: {}",
loginUser.getUserName(), name, note, type, host, port, database, principal, userName, connectType, other);
String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other);
Map<String, Object> result = dataSourceService.createDataSource(loginUser, name, note, type, parameter);
return returnDataList(result);
return dataSourceService.createDataSource(loginUser, name, note, type, parameter);
}
@ -156,8 +173,7 @@ public class DataSourceController extends BaseController {
logger.info("login user {} updateProcessInstance datasource name: {}, note: {}, type: {}, connectType: {}, other: {}",
loginUser.getUserName(), name, note, type, connectType, other);
String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other);
Map<String, Object> dataSource = dataSourceService.updateDataSource(id, loginUser, name, note, type, parameter);
return returnDataList(dataSource);
return dataSourceService.updateDataSource(id, loginUser, name, note, type, parameter);
}
/**
@ -281,15 +297,7 @@ public class DataSourceController extends BaseController {
logger.info("login user {}, connect datasource: {}, note: {}, type: {}, connectType: {}, other: {}",
loginUser.getUserName(), name, note, type, connectType, other);
String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other);
Boolean isConnection = dataSourceService.checkConnection(type, parameter);
Result result = new Result();
if (isConnection) {
putMsg(result, SUCCESS);
} else {
putMsg(result, CONNECT_DATASOURCE_FAILURE);
}
return result;
return dataSourceService.checkConnection(type, parameter);
}
/**
@ -309,16 +317,7 @@ public class DataSourceController extends BaseController {
public Result connectionTest(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam("id") int id) {
logger.info("connection test, login user:{}, id:{}", loginUser.getUserName(), id);
Boolean isConnection = dataSourceService.connectionTest(id);
Result result = new Result();
if (isConnection) {
putMsg(result, SUCCESS);
} else {
putMsg(result, CONNECTION_TEST_FAILURE);
}
return result;
return dataSourceService.connectionTest(id);
}
/**

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
Unescape View File

@ -198,7 +198,7 @@ public enum Status {
CHECK_OS_TENANT_CODE_ERROR(10164, "Please enter the English os tenant code", "请输入英文操作系统租户"),
FORCE_TASK_SUCCESS_ERROR(10165, "force task success error", "强制成功任务实例错误"),
TASK_INSTANCE_STATE_OPERATION_ERROR(10166, "the status of task instance {0} is {1},Cannot perform force success operation", "任务实例[{0}]的状态是[{1}]，无法执行强制成功操作"),
DATASOURCE_TYPE_NOT_EXIST(10167, "data source type not exist", "数据源类型不存在"),
UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found", "UDF函数不存在"),
UDF_FUNCTION_EXISTS(20002, "UDF function already exists", "UDF函数已存在"),

192
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java
Unescape View File

@ -14,12 +14,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
@ -27,27 +24,36 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbConnectType;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.*;
import org.apache.dolphinscheduler.dao.datasource.*;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.datasource.BaseDataSource;
import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory;
import org.apache.dolphinscheduler.dao.datasource.OracleDataSource;
import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import java.sql.Connection;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.*;
import static org.apache.dolphinscheduler.common.utils.PropertyUtils.getString;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.fasterxml.jackson.databind.node.ObjectNode;
/**
* datasource service
@ -67,11 +73,9 @@ public class DataSourceService extends BaseService {
public static final String USER_NAME = "userName";
public static final String OTHER = "other";
@Autowired
private DataSourceMapper dataSourceMapper;
@Autowired
private DataSourceUserMapper datasourceUserMapper;
@ -85,24 +89,16 @@ public class DataSourceService extends BaseService {
* @param parameter datasource parameters
* @return create result code
*/
public Map<String, Object> createDataSource(User loginUser, String name, String desc, DbType type, String parameter) {
public Result<Object> createDataSource(User loginUser, String name, String desc, DbType type, String parameter) {
Map<String, Object> result = new HashMap<>();
Result<Object> result = new Result<>();
// check name can use or not
if (checkName(name)) {
putMsg(result, Status.DATASOURCE_EXIST);
return result;
}
Boolean isConnection = checkConnection(type, parameter);
if (!isConnection) {
logger.info("connect failed, type:{}, parameter:{}", type, parameter);
putMsg(result, Status.DATASOURCE_CONNECT_FAILED);
return result;
}
BaseDataSource datasource = DataSourceFactory.getDatasource(type, parameter);
if (datasource == null) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, parameter);
Result<Object> isConnection = checkConnection(type, parameter);
if (Status.SUCCESS.getCode() != isConnection.getCode()) {
return result;
}
@ -125,7 +121,6 @@ public class DataSourceService extends BaseService {
return result;
}
/**
* updateProcessInstance datasource
*
@ -137,9 +132,9 @@ public class DataSourceService extends BaseService {
* @param id data source id
* @return update result code
*/
public Map<String, Object> updateDataSource(int id, User loginUser, String name, String desc, DbType type, String parameter) {
public Result<Object> updateDataSource(int id, User loginUser, String name, String desc, DbType type, String parameter) {
Map<String, Object> result = new HashMap<>();
Result<Object> result = new Result<>();
// determine whether the data source exists
DataSource dataSource = dataSourceMapper.selectById(id);
if (dataSource == null) {
@ -168,12 +163,11 @@ public class DataSourceService extends BaseService {
// connectionParams json
String connectionParams = paramObject.toString();
Boolean isConnection = checkConnection(type, connectionParams);
if (!isConnection) {
logger.info("connect failed, type:{}, parameter:{}", type, parameter);
putMsg(result, Status.DATASOURCE_CONNECT_FAILED);
Result<Object> isConnection = checkConnection(type, parameter);
if (Status.SUCCESS.getCode() != isConnection.getCode()) {
return result;
}
Date now = new Date();
dataSource.setName(name.trim());
@ -192,7 +186,6 @@ public class DataSourceService extends BaseService {
return queryDataSource != null && queryDataSource.size() > 0;
}
/**
* updateProcessInstance datasource
*
@ -360,8 +353,8 @@ public class DataSourceService extends BaseService {
* @param name datasource name
* @return true if data datasource not exists, otherwise return false
*/
public Result verifyDataSourceName(String name) {
Result result = new Result();
public Result<Object> verifyDataSourceName(String name) {
Result<Object> result = new Result<>();
List<DataSource> dataSourceList = dataSourceMapper.queryDataSourceByName(name);
if (dataSourceList != null && dataSourceList.size() > 0) {
logger.error("datasource name:{} has exist, can't create again.", name);
@ -373,77 +366,6 @@ public class DataSourceService extends BaseService {
return result;
}
/**
* get connection
*
* @param dbType datasource type
* @param parameter parameter
* @return connection for datasource
*/
private Connection getConnection(DbType dbType, String parameter) {
Connection connection = null;
BaseDataSource datasource = null;
try {
switch (dbType) {
case POSTGRESQL:
datasource = JSONUtils.parseObject(parameter, PostgreDataSource.class);
Class.forName(Constants.ORG_POSTGRESQL_DRIVER);
break;
case MYSQL:
datasource = JSONUtils.parseObject(parameter, MySQLDataSource.class);
Class.forName(Constants.COM_MYSQL_JDBC_DRIVER);
break;
case HIVE:
case SPARK:
if (CommonUtils.getKerberosStartupState()) {
System.setProperty(org.apache.dolphinscheduler.common.Constants.JAVA_SECURITY_KRB5_CONF,
getString(org.apache.dolphinscheduler.common.Constants.JAVA_SECURITY_KRB5_CONF_PATH));
Configuration configuration = new Configuration();
configuration.set(org.apache.dolphinscheduler.common.Constants.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
UserGroupInformation.setConfiguration(configuration);
UserGroupInformation.loginUserFromKeytab(getString(org.apache.dolphinscheduler.common.Constants.LOGIN_USER_KEY_TAB_USERNAME),
getString(org.apache.dolphinscheduler.common.Constants.LOGIN_USER_KEY_TAB_PATH));
}
if (dbType == DbType.HIVE) {
datasource = JSONUtils.parseObject(parameter, HiveDataSource.class);
} else if (dbType == DbType.SPARK) {
datasource = JSONUtils.parseObject(parameter, SparkDataSource.class);
}
Class.forName(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER);
break;
case CLICKHOUSE:
datasource = JSONUtils.parseObject(parameter, ClickHouseDataSource.class);
Class.forName(Constants.COM_CLICKHOUSE_JDBC_DRIVER);
break;
case ORACLE:
datasource = JSONUtils.parseObject(parameter, OracleDataSource.class);
Class.forName(Constants.COM_ORACLE_JDBC_DRIVER);
break;
case SQLSERVER:
datasource = JSONUtils.parseObject(parameter, SQLServerDataSource.class);
Class.forName(Constants.COM_SQLSERVER_JDBC_DRIVER);
break;
case DB2:
datasource = JSONUtils.parseObject(parameter, DB2ServerDataSource.class);
Class.forName(Constants.COM_DB2_JDBC_DRIVER);
break;
case PRESTO:
datasource = JSONUtils.parseObject(parameter, PrestoDataSource.class);
Class.forName(Constants.COM_PRESTO_JDBC_DRIVER);
break;
default:
break;
}
if (datasource != null) {
connection = DriverManager.getConnection(datasource.getJdbcUrl(), datasource.getUser(), datasource.getPassword());
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
return connection;
}
/**
* check connection
*
@ -451,18 +373,24 @@ public class DataSourceService extends BaseService {
* @param parameter data source parameters
* @return true if connect successfully, otherwise false
*/
public boolean checkConnection(DbType type, String parameter) {
Boolean isConnection = false;
Connection con = getConnection(type, parameter);
if (con != null) {
isConnection = true;
try {
con.close();
} catch (SQLException e) {
logger.error("close connection fail at DataSourceService::checkConnection()", e);
public Result<Object> checkConnection(DbType type, String parameter) {
Result<Object> result = new Result<>();
BaseDataSource datasource = DataSourceFactory.getDatasource(type, parameter);
if (datasource == null) {
putMsg(result, Status.DATASOURCE_TYPE_NOT_EXIST, type);
return result;
}
try (Connection connection = datasource.getConnection()) {
if (connection == null) {
putMsg(result, Status.CONNECTION_TEST_FAILURE);
return result;
}
putMsg(result, Status.SUCCESS);
return result;
} catch (Exception e) {
logger.error("datasource test connection error, dbType:{}, jdbcUrl:{}, message:{}.", type, datasource.getJdbcUrl(), e.getMessage());
return new Result<>(Status.CONNECTION_TEST_FAILURE.getCode(),e.getMessage());
}
return isConnection;
}
/**
@ -471,13 +399,14 @@ public class DataSourceService extends BaseService {
* @param id datasource id
* @return connect result code
*/
public boolean connectionTest(int id) {
public Result<Object> connectionTest(int id) {
DataSource dataSource = dataSourceMapper.selectById(id);
if (dataSource != null) {
return checkConnection(dataSource.getType(), dataSource.getConnectionParams());
} else {
return false;
if (dataSource == null) {
Result<Object> result = new Result<>();
putMsg(result, Status.RESOURCE_NOT_EXIST);
return result;
}
return checkConnection(dataSource.getType(), dataSource.getConnectionParams());
}
/**
@ -510,8 +439,8 @@ public class DataSourceService extends BaseService {
parameterMap.put(Constants.ORACLE_DB_CONNECT_TYPE, connectType);
}
if (CommonUtils.getKerberosStartupState() &&
(type == DbType.HIVE || type == DbType.SPARK)) {
if (CommonUtils.getKerberosStartupState()
&& (type == DbType.HIVE || type == DbType.SPARK)) {
jdbcUrl += ";principal=" + principal;
}
@ -535,8 +464,8 @@ public class DataSourceService extends BaseService {
parameterMap.put(Constants.JDBC_URL, jdbcUrl);
parameterMap.put(Constants.USER, userName);
parameterMap.put(Constants.PASSWORD, CommonUtils.encodePassword(password));
if (CommonUtils.getKerberosStartupState() &&
(type == DbType.HIVE || type == DbType.SPARK)) {
if (CommonUtils.getKerberosStartupState()
&& (type == DbType.HIVE || type == DbType.SPARK)) {
parameterMap.put(Constants.PRINCIPAL, principal);
}
@ -557,7 +486,6 @@ public class DataSourceService extends BaseService {
}
return JSONUtils.toJsonString(parameterMap);
}
private String buildAddress(DbType type, String host, String port, DbConnectType connectType) {
@ -609,8 +537,8 @@ public class DataSourceService extends BaseService {
* @return delete result code
*/
@Transactional(rollbackFor = RuntimeException.class)
public Result delete(User loginUser, int datasourceId) {
Result result = new Result();
public Result<Object> delete(User loginUser, int datasourceId) {
Result<Object> result = new Result<>();
try {
//query datasource by id
DataSource dataSource = dataSourceMapper.selectById(datasourceId);
@ -673,7 +601,6 @@ public class DataSourceService extends BaseService {
return result;
}
/**
* authorized datasource
*
@ -695,7 +622,6 @@ public class DataSourceService extends BaseService {
return result;
}
/**
* get host and port by address
*

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java
Unescape View File

@ -72,6 +72,8 @@ public class BaseServiceTest {
}
@Test
public void testPutMsg(){

108
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java
Unescape View File

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.enums.Status;
@ -24,12 +25,19 @@ import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.dao.datasource.BaseDataSource;
import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory;
import org.apache.dolphinscheduler.dao.datasource.MySQLDataSource;
import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper;
import java.sql.Connection;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -38,16 +46,15 @@ import org.mockito.Mock;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@RunWith(PowerMockRunner.class)
@PowerMockIgnore({"sun.security.*", "javax.net.*"})
@PrepareForTest({DataSourceFactory.class})
public class DataSourceServiceTest {
@InjectMocks
private DataSourceService dataSourceService;
@Mock
@ -69,28 +76,31 @@ public class DataSourceServiceTest {
dataSource.setName(dataSourceName);
dataSourceList.add(dataSource);
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(dataSourceList);
Map<String, Object> dataSourceExitsResult = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.DATASOURCE_EXIST, dataSourceExitsResult.get(Constants.STATUS));
Result dataSourceExitsResult = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.DATASOURCE_EXIST.getCode(), dataSourceExitsResult.getCode().intValue());
// data source exits
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null);
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(false);
Map<String, Object> connectFailedResult = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.DATASOURCE_CONNECT_FAILED, connectFailedResult.get(Constants.STATUS));
Result connectionResult = new Result(Status.DATASOURCE_CONNECT_FAILED.getCode(),Status.DATASOURCE_CONNECT_FAILED.getMsg());
//PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult);
PowerMockito.doReturn(connectionResult).when(dataSourceService).checkConnection(dataSourceType, parameter);
Result connectFailedResult = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.DATASOURCE_CONNECT_FAILED.getCode(), connectFailedResult.getCode().intValue());
// data source exits
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null);
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(true);
connectionResult = new Result(Status.SUCCESS.getCode(),Status.SUCCESS.getMsg());
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult);
PowerMockito.when(DataSourceFactory.getDatasource(dataSourceType, parameter)).thenReturn(null);
Map<String, Object> notValidError = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, notValidError.get(Constants.STATUS));
Result notValidError = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR.getCode(), notValidError.getCode().intValue());
// success
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null);
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(true);
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult);
PowerMockito.when(DataSourceFactory.getDatasource(dataSourceType, parameter)).thenReturn(JSONUtils.parseObject(parameter, MySQLDataSource.class));
Map<String, Object> success = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.SUCCESS, success.get(Constants.STATUS));
Result success = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.SUCCESS.getCode(), success.getCode().intValue());
}
public void updateDataSourceTest() {
@ -104,14 +114,14 @@ public class DataSourceServiceTest {
// data source not exits
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(null);
Map<String, Object> resourceNotExits = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.RESOURCE_NOT_EXIST, resourceNotExits.get(Constants.STATUS));
Result resourceNotExits = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getCode(), resourceNotExits.getCode().intValue());
// user no operation perm
DataSource dataSource = new DataSource();
dataSource.setUserId(0);
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource);
Map<String, Object> userNoOperationPerm = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, userNoOperationPerm.get(Constants.STATUS));
Result userNoOperationPerm = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.USER_NO_OPERATION_PERM.getCode(), userNoOperationPerm.getCode().intValue());
// data source name exits
dataSource.setUserId(-1);
@ -119,22 +129,24 @@ public class DataSourceServiceTest {
dataSourceList.add(dataSource);
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource);
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(dataSourceList);
Map<String, Object> dataSourceNameExist = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.DATASOURCE_EXIST, dataSourceNameExist.get(Constants.STATUS));
Result dataSourceNameExist = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.DATASOURCE_EXIST.getCode(), dataSourceNameExist.getCode().intValue());
// data source connect failed
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource);
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(null);
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(true);
Map<String, Object> connectFailed = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.DATASOURCE_CONNECT_FAILED, connectFailed.get(Constants.STATUS));
Result connectionResult = new Result(Status.SUCCESS.getCode(),Status.SUCCESS.getMsg());
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult);
Result connectFailed = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.DATASOURCE_CONNECT_FAILED.getCode(), connectFailed.getCode().intValue());
//success
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource);
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(null);
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(false);
Map<String, Object> success = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.SUCCESS, connectFailed.get(Constants.STATUS));
connectionResult = new Result(Status.DATASOURCE_CONNECT_FAILED.getCode(),Status.DATASOURCE_CONNECT_FAILED.getMsg());
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult);
Result success = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Assert.assertEquals(Status.SUCCESS.getCode(), success.getCode().intValue());
}
@ -152,7 +164,8 @@ public class DataSourceServiceTest {
public void connectionTest() {
int dataSourceId = -1;
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(null);
Assert.assertFalse(dataSourceService.connectionTest(dataSourceId));
Result result = dataSourceService.connectionTest(dataSourceId);
Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getCode(),result.getCode().intValue());
}
@Test
@ -252,7 +265,8 @@ public class DataSourceServiceTest {
dataSource.setName("test");
dataSource.setNote("Note");
dataSource.setType(DbType.ORACLE);
dataSource.setConnectionParams("{\"connectType\":\"ORACLE_SID\",\"address\":\"jdbc:oracle:thin:@192.168.xx.xx:49161\",\"database\":\"XE\",\"jdbcUrl\":\"jdbc:oracle:thin:@192.168.xx.xx:49161/XE\",\"user\":\"system\",\"password\":\"oracle\"}");
dataSource.setConnectionParams("{\"connectType\":\"ORACLE_SID\",\"address\":\"jdbc:oracle:thin:@192.168.xx.xx:49161\",\"database\":\"XE\","
+ "\"jdbcUrl\":\"jdbc:oracle:thin:@192.168.xx.xx:49161/XE\",\"user\":\"system\",\"password\":\"oracle\"}");
return dataSource;
}
@ -261,7 +275,8 @@ public class DataSourceServiceTest {
public void buildParameter() {
String param = dataSourceService.buildParameter(DbType.ORACLE, "192.168.9.1", "1521", "im"
, "", "test", "test", DbConnectType.ORACLE_SERVICE_NAME, "");
String expected = "{\"connectType\":\"ORACLE_SERVICE_NAME\",\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}";
String expected = "{\"connectType\":\"ORACLE_SERVICE_NAME\",\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\","
+ "\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}";
Assert.assertEquals(expected, param);
}
@ -270,10 +285,10 @@ public class DataSourceServiceTest {
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "true");
String param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im"
, "", "test", "123456", null, "");
String expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\"}";
String expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\","
+ "\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\"}";
Assert.assertEquals(expected, param);
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "false");
param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im"
, "", "test", "123456", null, "");
@ -294,4 +309,31 @@ public class DataSourceServiceTest {
return loginUser;
}
}
/**
* test check connection
* @throws Exception
*/
@Test
public void testCheckConnection() throws Exception {
DbType dataSourceType = DbType.POSTGRESQL;
String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null);
PowerMockito.mockStatic(DataSourceFactory.class);
PowerMockito.when(DataSourceFactory.getDatasource(Mockito.any(), Mockito.anyString())).thenReturn(null);
Result result = dataSourceService.checkConnection(dataSourceType, parameter);
Assert.assertEquals(Status.DATASOURCE_TYPE_NOT_EXIST.getCode(), result.getCode().intValue());
BaseDataSource dataSource = PowerMockito.mock(BaseDataSource.class);
PowerMockito.when(DataSourceFactory.getDatasource(Mockito.any(), Mockito.anyString())).thenReturn(dataSource);
PowerMockito.when(dataSource.getConnection()).thenReturn(null);
result = dataSourceService.checkConnection(dataSourceType, parameter);
Assert.assertEquals(Status.CONNECTION_TEST_FAILURE.getCode(), result.getCode().intValue());
Connection connection = PowerMockito.mock(Connection.class);
PowerMockito.when(dataSource.getConnection()).thenReturn(connection);
result = dataSourceService.checkConnection(dataSourceType, parameter);
Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
}
}

14
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java
Unescape View File

@ -33,11 +33,17 @@ public class StringUtils {
return !isEmpty(cs);
}
public static boolean isBlank(String s) {
if (isEmpty(s)) {
return true;
public static boolean isBlank(String str) {
int strLen;
if (str != null && (strLen = str.length()) != 0) {
for (int i = 0; i < strLen; ++i) {
if (!Character.isWhitespace(str.charAt(i))) {
return false;
}
}
}
return s.trim().length() == 0;
return true;
}
public static boolean isNotBlank(String s) {

351
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java
Unescape View File

@ -14,14 +14,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.datasource;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -30,194 +32,183 @@ import org.slf4j.LoggerFactory;
*/
public abstract class BaseDataSource {
private static final Logger logger = LoggerFactory.getLogger(BaseDataSource.class);
/**
* user name
*/
protected String user;
/**
* user password
*/
protected String password;
/**
* data source address
*/
private String address;
/**
* database name
*/
private String database;
/**
* other connection parameters for the data source
*/
private String other;
/**
* principal
*/
private String principal;
public String getPrincipal() {
return principal;
}
public void setPrincipal(String principal) {
this.principal = principal;
}
/**
* @return driver class
*/
public abstract String driverClassSelector();
/**
* @return db type
*/
public abstract DbType dbTypeSelector();
/**
* gets the JDBC url for the data source connection
* @return getJdbcUrl
*/
public String getJdbcUrl() {
StringBuilder jdbcUrl = new StringBuilder(getAddress());
appendDatabase(jdbcUrl);
appendPrincipal(jdbcUrl);
appendOther(jdbcUrl);
return jdbcUrl.toString();
}
/**
* append database
* @param jdbcUrl jdbc url
*/
protected void appendDatabase(StringBuilder jdbcUrl) {
if (dbTypeSelector() == DbType.SQLSERVER) {
jdbcUrl.append(";databaseName=").append(getDatabase());
} else {
if (getAddress().lastIndexOf('/') != (jdbcUrl.length() - 1)) {
jdbcUrl.append("/");
}
jdbcUrl.append(getDatabase());
}
}
/**
* append principal
* @param jdbcUrl jdbc url
*/
private void appendPrincipal(StringBuilder jdbcUrl) {
boolean tag = dbTypeSelector() == DbType.HIVE || dbTypeSelector() == DbType.SPARK;
if (tag && StringUtils.isNotEmpty(getPrincipal())) {
jdbcUrl.append(";principal=").append(getPrincipal());
}
}
/**
* append other
* @param jdbcUrl jdbc url
*/
private void appendOther(StringBuilder jdbcUrl) {
String otherParams = filterOther(getOther());
if (StringUtils.isNotEmpty(otherParams)) {
String separator = "";
switch (dbTypeSelector()) {
case CLICKHOUSE:
case MYSQL:
case ORACLE:
case POSTGRESQL:
case PRESTO:
separator = "?";
break;
case DB2:
separator = ":";
break;
case HIVE:
case SPARK:
case SQLSERVER:
separator = ";";
break;
default:
logger.error("Db type mismatch!");
}
jdbcUrl.append(separator).append(otherParams);
}
}
protected String filterOther(String otherParams){
return otherParams;
}
/**
* test whether the data source can be connected successfully
*/
public void isConnectable() {
Connection con = null;
try {
Class.forName(driverClassSelector());
con = DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword());
} catch (ClassNotFoundException | SQLException e) {
logger.error("Get connection error: {}", e.getMessage());
} finally {
if (con != null) {
try {
con.close();
} catch (SQLException e) {
logger.error(e.getMessage(), e);
private static final Logger logger = LoggerFactory.getLogger(BaseDataSource.class);
/**
* user name
*/
protected String user;
/**
* user password
*/
protected String password;
/**
* data source address
*/
private String address;
/**
* database name
*/
private String database;
/**
* other connection parameters for the data source
*/
private String other;
/**
* principal
*/
private String principal;
public String getPrincipal() {
return principal;
}
public void setPrincipal(String principal) {
this.principal = principal;
}
/**
* @return driver class
*/
public abstract String driverClassSelector();
/**
* @return db type
*/
public abstract DbType dbTypeSelector();
/**
* gets the JDBC url for the data source connection
* @return getJdbcUrl
*/
public String getJdbcUrl() {
StringBuilder jdbcUrl = new StringBuilder(getAddress());
appendDatabase(jdbcUrl);
appendPrincipal(jdbcUrl);
appendOther(jdbcUrl);
return jdbcUrl.toString();
}
/**
* append database
* @param jdbcUrl jdbc url
*/
protected void appendDatabase(StringBuilder jdbcUrl) {
if (dbTypeSelector() == DbType.SQLSERVER) {
jdbcUrl.append(";databaseName=").append(getDatabase());
} else {
if (getAddress().lastIndexOf('/') != (jdbcUrl.length() - 1)) {
jdbcUrl.append("/");
}
jdbcUrl.append(getDatabase());
}
}
/**
* append principal
* @param jdbcUrl jdbc url
*/
private void appendPrincipal(StringBuilder jdbcUrl) {
boolean tag = dbTypeSelector() == DbType.HIVE || dbTypeSelector() == DbType.SPARK;
if (tag && StringUtils.isNotEmpty(getPrincipal())) {
jdbcUrl.append(";principal=").append(getPrincipal());
}
}
}
}
public String getUser() {
return user;
}
/**
* append other
* @param jdbcUrl jdbc url
*/
private void appendOther(StringBuilder jdbcUrl) {
String otherParams = filterOther(getOther());
if (StringUtils.isNotEmpty(otherParams)) {
String separator = "";
switch (dbTypeSelector()) {
case CLICKHOUSE:
case MYSQL:
case ORACLE:
case POSTGRESQL:
case PRESTO:
separator = "?";
break;
case DB2:
separator = ":";
break;
case HIVE:
case SPARK:
case SQLSERVER:
separator = ";";
break;
default:
logger.error("Db type mismatch!");
}
jdbcUrl.append(separator).append(otherParams);
}
}
/**
* the data source test connection
* @return Connection Connection
* @throws Exception Exception
*/
public Connection getConnection() throws Exception {
Class.forName(driverClassSelector());
return DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword());
}
protected String filterOther(String otherParams) {
return otherParams;
}
public String getUser() {
return user;
}
public void setUser(String user) {
this.user = user;
}
public void setUser(String user) {
this.user = user;
}
/**
* password need decode
* @return
*/
public String getPassword() {
return CommonUtils.decodePassword(password);
}
/**
* password need decode
* @return
*/
public String getPassword() {
return CommonUtils.decodePassword(password);
}
public void setPassword(String password) {
this.password = password;
}
public void setPassword(String password) {
this.password = password;
}
public void setAddress(String address) {
this.address = address;
}
public void setAddress(String address) {
this.address = address;
}
public String getAddress() {
return address;
}
public String getAddress() {
return address;
}
public String getDatabase() {
return database;
}
public String getDatabase() {
return database;
}
public void setDatabase(String database) {
this.database = database;
}
public void setDatabase(String database) {
this.database = database;
}
public String getOther() {
return other;
}
public String getOther() {
return other;
}
public void setOther(String other) {
this.other = other;
}
public void setOther(String other) {
this.other = other;
}
}

124
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java
Unescape View File

@ -19,78 +19,92 @@ package org.apache.dolphinscheduler.dao.datasource;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import java.sql.Connection;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
/**
* data source of hive
*/
public class HiveDataSource extends BaseDataSource {
/**
* gets the JDBC url for the data source connection
* @return jdbc url
*/
@Override
public String driverClassSelector() {
return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER;
}
/**
* @return db type
*/
@Override
public DbType dbTypeSelector() {
return DbType.HIVE;
}
/**
* build hive jdbc params,append : ?hive_conf_list
*
* hive jdbc url template:
*
* jdbc:hive2://<host1>:<port1>,<host2>:<port2>/dbName;initFile=<file>;sess_var_list?hive_conf_list#hive_var_list
*
* @param otherParams otherParams
* @return filter otherParams
*/
@Override
protected String filterOther(String otherParams) {
if (StringUtils.isBlank(otherParams)) {
return "";
/**
* gets the JDBC url for the data source connection
* @return jdbc url
*/
@Override
public String driverClassSelector() {
return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER;
}
StringBuilder hiveConfListSb = new StringBuilder();
hiveConfListSb.append("?");
StringBuilder sessionVarListSb = new StringBuilder();
/**
* @return db type
*/
@Override
public DbType dbTypeSelector() {
return DbType.HIVE;
}
String[] otherArray = otherParams.split(";", -1);
/**
* build hive jdbc params,append : ?hive_conf_list
*
* hive jdbc url template:
*
* jdbc:hive2://<host1>:<port1>,<host2>:<port2>/dbName;initFile=<file>;sess_var_list?hive_conf_list#hive_var_list
*
* @param otherParams otherParams
* @return filter otherParams
*/
@Override
protected String filterOther(String otherParams) {
if (StringUtils.isBlank(otherParams)) {
return "";
}
// get the default hive conf var name
Set<String> hiveConfSet = Stream.of(ConfVars.values()).map(confVars -> confVars.varname)
.collect(Collectors.toSet());
StringBuilder hiveConfListSb = new StringBuilder();
hiveConfListSb.append("?");
StringBuilder sessionVarListSb = new StringBuilder();
for (String conf : otherArray) {
if (hiveConfSet.contains(conf.split("=")[0])) {
hiveConfListSb.append(conf).append(";");
} else {
sessionVarListSb.append(conf).append(";");
}
}
String[] otherArray = otherParams.split(";", -1);
// remove the last ";"
if (sessionVarListSb.length() > 0) {
sessionVarListSb.deleteCharAt(sessionVarListSb.length() - 1);
}
// get the default hive conf var name
Set<String> hiveConfSet = Stream.of(ConfVars.values()).map(confVars -> confVars.varname)
.collect(Collectors.toSet());
for (String conf : otherArray) {
if (hiveConfSet.contains(conf.split("=")[0])) {
hiveConfListSb.append(conf).append(";");
} else {
sessionVarListSb.append(conf).append(";");
}
}
// remove the last ";"
if (sessionVarListSb.length() > 0) {
sessionVarListSb.deleteCharAt(sessionVarListSb.length() - 1);
}
if (hiveConfListSb.length() > 0) {
hiveConfListSb.deleteCharAt(hiveConfListSb.length() - 1);
if (hiveConfListSb.length() > 0) {
hiveConfListSb.deleteCharAt(hiveConfListSb.length() - 1);
}
return sessionVarListSb.toString() + hiveConfListSb.toString();
}
return sessionVarListSb.toString() + hiveConfListSb.toString();
}
/**
* the data source test connection
* @return Connection Connection
* @throws Exception Exception
*/
@Override
public Connection getConnection() throws Exception {
CommonUtils.loadKerberosConf();
return super.getConnection();
}
}

52
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java
Unescape View File

@ -14,25 +14,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.datasource;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
/**
* data source of SQL Server
*/
public class SQLServerDataSource extends BaseDataSource {
private static final Logger logger = LoggerFactory.getLogger(SQLServerDataSource.class);
/**
* gets the JDBC url for the data source connection
* @return jdbc url
@ -50,39 +43,18 @@ public class SQLServerDataSource extends BaseDataSource {
}
/**
* test whether the data source can be connected successfully
*/
* @return driver class
*/
@Override
public void isConnectable() {
Connection con = null;
try {
Class.forName(Constants.COM_SQLSERVER_JDBC_DRIVER);
con = DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword());
} catch (Exception e) {
logger.error("error", e);
} finally {
if (con != null) {
try {
con.close();
} catch (SQLException e) {
logger.error("SQL Server datasource try conn close conn error", e);
}
}
}
public String driverClassSelector() {
return Constants.COM_SQLSERVER_JDBC_DRIVER;
}
/**
* @return driver class
*/
@Override
public String driverClassSelector() {
return Constants.COM_SQLSERVER_JDBC_DRIVER;
}
/**
* @return db type
*/
@Override
public DbType dbTypeSelector() {
return DbType.SQLSERVER;
}
/**
* @return db type
*/
@Override
public DbType dbTypeSelector() {
return DbType.SQLSERVER;
}
}

45
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java
Unescape View File

@ -14,30 +14,45 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.datasource;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import java.sql.Connection;
/**
* data source of spark
*/
public class SparkDataSource extends BaseDataSource {
/**
* gets the JDBC url for the data source connection
* @return jdbc url
*/
@Override
public String driverClassSelector() {
return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER;
}
/**
* gets the JDBC url for the data source connection
* @return jdbc url
*/
@Override
public String driverClassSelector() {
return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER;
}
/**
* @return db type
*/
@Override
public DbType dbTypeSelector() {
return DbType.SPARK;
}
/**
* @return db type
*/
@Override
public DbType dbTypeSelector() {
return DbType.SPARK;
}
/**
* the data source test connection
* @return Connection Connection
* @throws Exception Exception
*/
@Override
public Connection getConnection() throws Exception {
CommonUtils.loadKerberosConf();
return super.getConnection();
}
}

6
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
Unescape View File

@ -687,6 +687,12 @@ public class ProcessService {
processInstance = generateNewProcessInstance(processDefinition, command, cmdParam);
} else {
processInstance = this.findProcessInstanceDetailById(processInstanceId);
// Recalculate global parameters after rerun.
processInstance.setGlobalParams(ParameterUtils.curingGlobalParams(
processDefinition.getGlobalParamMap(),
processDefinition.getGlobalParamList(),
getCommandTypeIfComplement(processInstance, command),
processInstance.getScheduleTime()));
}
processDefinition = processDefineMapper.selectById(processInstance.getProcessDefinitionId());
processInstance.setProcessDefinition(processDefinition);

9
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue
Unescape View File

@ -74,6 +74,11 @@
</div>
<div class="bottom">
<div class="submit">
<template v-if="router.history.current.name === 'projects-definition-details'">
<div class="lint-pt">
<el-checkbox v-model="releaseState" size="small" :false-label="0" :true-label="1">{{$t('Whether to go online the process definition')}}</el-checkbox>
</div>
</template>
<template v-if="router.history.current.name === 'projects-instance-details'">
<div class="lint-pt">
<el-checkbox v-model="syncDefine" size="small">{{$t('Whether to update the process definition')}}</el-checkbox>
@ -106,6 +111,8 @@
udpList: [],
// Global custom parameters
udpListCache: [],
// Whether to go online the process definition
releaseState: 1,
// Whether to update the process definition
syncDefine: true,
// Timeout alarm
@ -141,6 +148,7 @@
this.store.commit('dag/setTenantId', _.cloneDeep(this.tenantId))
this.store.commit('dag/setDesc', _.cloneDeep(this.description))
this.store.commit('dag/setSyncDefine', this.syncDefine)
this.store.commit('dag/setReleaseState', this.releaseState)
},
/**
* submit
@ -201,6 +209,7 @@
this.originalName = dag.name
this.description = dag.description
this.syncDefine = dag.syncDefine
this.releaseState = dag.releaseState
this.timeout = dag.timeout || 0
this.checkedTimeout = this.timeout !== 0
this.$nextTick(() => {

3
dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js
Unescape View File

@ -318,7 +318,8 @@ export default {
connects: JSON.stringify(state.connects),
name: _.trim(state.name),
description: _.trim(state.description),
id: payload
id: payload,
releaseState: state.releaseState
}, res => {
resolve(res)
state.isEditDag = false

3
dolphinscheduler-ui/src/js/conf/home/store/dag/mutations.js
Unescape View File

@ -75,6 +75,9 @@ export default {
setDesc (state, payload) {
state.description = payload
},
setReleaseState (state, payload) {
state.releaseState = payload
},
/**
* Whether to update the process definition
*/

2
dolphinscheduler-ui/src/js/conf/home/store/dag/state.js
Unescape View File

@ -45,6 +45,8 @@ export default {
isEditDag: false,
// Current project
projectName: projectName || '',
// Whether to go online the process definition
releaseState: 1,
// Whether to update the process definition
syncDefine: true,
// tasks processList

1
dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js
Unescape View File

@ -64,6 +64,7 @@ export default {
'Set the DAG diagram name': 'Set the DAG diagram name',
'Please enter description(optional)': 'Please enter description(optional)',
'Set global': 'Set global',
'Whether to go online the process definition': 'Whether to go online the process definition',
'Whether to update the process definition': 'Whether to update the process definition',
Add: 'Add',
'DAG graph name cannot be empty': 'DAG graph name cannot be empty',

1
dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js
Unescape View File

@ -64,6 +64,7 @@ export default {
'Set the DAG diagram name': '设置DAG图名称',
'Please enter description(optional)': '请输入描述(选填)',
'Set global': '设置全局',
'Whether to go online the process definition': '是否上线流程定义',
'Whether to update the process definition': '是否更新流程定义',
Add: '添加',
'DAG graph name cannot be empty': 'DAG图名称不能为空',

258
mvnw vendored
Unescape View File

@ -34,128 +34,137 @@
# MAVEN_SKIP_RC - flag to disable loading of mavenrc files
# ----------------------------------------------------------------------------
if [ -z "$MAVEN_SKIP_RC" ] ; then
if [ -z "$MAVEN_SKIP_RC" ]; then
if [ -f /etc/mavenrc ] ; then
if [ -f /etc/mavenrc ]; then
. /etc/mavenrc
fi
if [ -f "$HOME/.mavenrc" ] ; then
if [ -f "$HOME/.mavenrc" ]; then
. "$HOME/.mavenrc"
fi
fi
# OS specific support. $var _must_ be set to either true or false.
cygwin=false;
darwin=false;
cygwin=false
darwin=false
mingw=false
case "`uname`" in
CYGWIN*) cygwin=true ;;
MINGW*) mingw=true;;
Darwin*) darwin=true
# Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
# See https://developer.apple.com/library/mac/qa/qa1170/_index.html
if [ -z "$JAVA_HOME" ]; then
if [ -x "/usr/libexec/java_home" ]; then
export JAVA_HOME="`/usr/libexec/java_home`"
else
export JAVA_HOME="/Library/Java/Home"
fi
case "$(uname)" in
CYGWIN*) cygwin=true ;;
MINGW*) mingw=true ;;
Darwin*)
darwin=true
# Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
# See https://developer.apple.com/library/mac/qa/qa1170/_index.html
if [ -z "$JAVA_HOME" ]; then
if [ -x "/usr/libexec/java_home" ]; then
JAVA_HOME="$(/usr/libexec/java_home)"
export JAVA_HOME
else
JAVA_HOME="/Library/Java/Home"
export JAVA_HOME
fi
;;
fi
;;
esac
if [ -z "$JAVA_HOME" ] ; then
if [ -r /etc/gentoo-release ] ; then
JAVA_HOME=`java-config --jre-home`
if [ -z "$JAVA_HOME" ]; then
if [ -r /etc/gentoo-release ]; then
JAVA_HOME=$(java-config --jre-home)
fi
fi
if [ -z "$M2_HOME" ] ; then
if [ -z "$M2_HOME" ]; then
## resolve links - $0 may be a link to maven's home
PRG="$0"
# need this for relative symlinks
while [ -h "$PRG" ] ; do
ls=`ls -ld "$PRG"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
while [ -h "$PRG" ]; do
ls=$(ls -ld "$PRG")
link=$(expr "$ls" : '.*-> \(.*\)$')
if expr "$link" : '/.*' >/dev/null; then
PRG="$link"
else
PRG="`dirname "$PRG"`/$link"
PRG="$(dirname "$PRG")/$link"
fi
done
saveddir=`pwd`
saveddir=$(pwd)
M2_HOME=`dirname "$PRG"`/..
M2_HOME=$(dirname "$PRG")/..
# make it fully qualified
M2_HOME=`cd "$M2_HOME" && pwd`
M2_HOME=$(cd "$M2_HOME" && pwd)
cd "$saveddir"
cd "$saveddir" || exit
# echo Using m2 at $M2_HOME
fi
# For Cygwin, ensure paths are in UNIX format before anything is touched
if $cygwin ; then
if $cygwin; then
[ -n "$M2_HOME" ] &&
M2_HOME=`cygpath --unix "$M2_HOME"`
M2_HOME=$(cygpath --unix "$M2_HOME")
[ -n "$JAVA_HOME" ] &&
JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
JAVA_HOME=$(cygpath --unix "$JAVA_HOME")
[ -n "$CLASSPATH" ] &&
CLASSPATH=`cygpath --path --unix "$CLASSPATH"`
CLASSPATH=$(cygpath --path --unix "$CLASSPATH")
fi
# For Mingw, ensure paths are in UNIX format before anything is touched
if $mingw ; then
if $mingw; then
[ -n "$M2_HOME" ] &&
M2_HOME="`(cd "$M2_HOME"; pwd)`"
M2_HOME="$( (
cd "$M2_HOME" || exit
pwd
))"
[ -n "$JAVA_HOME" ] &&
JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`"
JAVA_HOME="$( (
cd "$JAVA_HOME" || exit
pwd
))"
fi
if [ -z "$JAVA_HOME" ]; then
javaExecutable="`which javac`"
if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then
javaExecutable="$(command -v javac)"
if [ -n "$javaExecutable" ] && ! [ "$(expr "$javaExecutable" : '\([^ ]*\)')" = "no" ]; then
# readlink(1) is not available as standard on Solaris 10.
readLink=`which readlink`
if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then
if $darwin ; then
javaHome="`dirname \"$javaExecutable\"`"
javaExecutable="`cd \"$javaHome\" && pwd -P`/javac"
readLink=$(command -v readlink)
if [ ! "$(expr "$readLink" : '\([^ ]*\)')" = "no" ]; then
if $darwin; then
javaHome="$(dirname "$javaExecutable")"
javaExecutable="$(cd "$javaHome" && pwd -P)/javac"
else
javaExecutable="`readlink -f \"$javaExecutable\"`"
javaExecutable="$(readlink -f "$javaExecutable")"
fi
javaHome="`dirname \"$javaExecutable\"`"
javaHome=`expr "$javaHome" : '\(.*\)/bin'`
javaHome="$(dirname "$javaExecutable")"
javaHome=$(expr "$javaHome" : '\(.*\)/bin')
JAVA_HOME="$javaHome"
export JAVA_HOME
fi
fi
fi
if [ -z "$JAVACMD" ] ; then
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
if [ -z "$JAVACMD" ]; then
if [ -n "$JAVA_HOME" ]; then
if [ -x "$JAVA_HOME/jre/sh/java" ]; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
else
JAVACMD="$JAVA_HOME/bin/java"
fi
else
JAVACMD="`which java`"
JAVACMD="$(command -v java)"
fi
fi
if [ ! -x "$JAVACMD" ] ; then
if [ ! -x "$JAVACMD" ]; then
echo "Error: JAVA_HOME is not defined correctly." >&2
echo " We cannot execute $JAVACMD" >&2
exit 1
fi
if [ -z "$JAVA_HOME" ] ; then
if [ -z "$JAVA_HOME" ]; then
echo "Warning: JAVA_HOME environment variable is not set."
fi
@ -165,22 +174,24 @@ CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher
# first directory with .mvn subdirectory is considered project base directory
find_maven_basedir() {
if [ -z "$1" ]
then
if [ -z "$1" ]; then
echo "Path not specified to find_maven_basedir"
return 1
fi
basedir="$1"
wdir="$1"
while [ "$wdir" != '/' ] ; do
if [ -d "$wdir"/.mvn ] ; then
while [ "$wdir" != '/' ]; do
if [ -d "$wdir"/.mvn ]; then
basedir=$wdir
break
fi
# workaround for JBEAP-8937 (on Solaris 10/Sparc)
if [ -d "${wdir}" ]; then
wdir=`cd "$wdir/.."; pwd`
wdir=$(
cd "$wdir/.." || exit
pwd
)
fi
# end of workaround
done
@ -190,13 +201,13 @@ find_maven_basedir() {
# concatenates all lines of a file
concat_lines() {
if [ -f "$1" ]; then
echo "$(tr -s '\n' ' ' < "$1")"
tr -s '\n' ' ' <"$1"
fi
}
BASE_DIR=`find_maven_basedir "$(pwd)"`
BASE_DIR=$(find_maven_basedir "$(pwd)")
if [ -z "$BASE_DIR" ]; then
exit 1;
exit 1
fi
##########################################################################################
@ -204,75 +215,78 @@ fi
# This allows using the maven wrapper in projects that prohibit checking in binary data.
##########################################################################################
if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then
if [ "$MVNW_VERBOSE" = true ]; then
echo "Found .mvn/wrapper/maven-wrapper.jar"
fi
else
if [ "$MVNW_VERBOSE" = true ]; then
echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..."
fi
if [ -n "$MVNW_REPOURL" ]; then
jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
else
jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
fi
while IFS="=" read -r key value; do
case "$key" in wrapperUrl)
jarUrl="$value"
break
;;
esac
done <"$BASE_DIR/.mvn/wrapper/maven-wrapper.properties"
if [ "$MVNW_VERBOSE" = true ]; then
echo "Downloading from: $jarUrl"
fi
wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar"
if $cygwin; then
wrapperJarPath=$(cygpath --path --windows "$wrapperJarPath")
fi
if command -v wget >/dev/null; then
if [ "$MVNW_VERBOSE" = true ]; then
echo "Found .mvn/wrapper/maven-wrapper.jar"
echo "Found wget ... using wget"
fi
else
if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
wget "$jarUrl" -O "$wrapperJarPath"
else
wget --http-user="$MVNW_USERNAME" --http-password="$MVNW_PASSWORD" "$jarUrl" -O "$wrapperJarPath"
fi
elif command -v curl >/dev/null; then
if [ "$MVNW_VERBOSE" = true ]; then
echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..."
echo "Found curl ... using curl"
fi
if [ -n "$MVNW_REPOURL" ]; then
jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
curl -o "$wrapperJarPath" "$jarUrl" -f
else
jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
curl --user "$MVNW_USERNAME":"$MVNW_PASSWORD" -o "$wrapperJarPath" "$jarUrl" -f
fi
while IFS="=" read key value; do
case "$key" in (wrapperUrl) jarUrl="$value"; break ;;
esac
done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties"
else
if [ "$MVNW_VERBOSE" = true ]; then
echo "Downloading from: $jarUrl"
echo "Falling back to using Java to download"
fi
wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar"
javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java"
# For Cygwin, switch paths to Windows format before running javac
if $cygwin; then
wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"`
javaClass=$(cygpath --path --windows "$javaClass")
fi
if command -v wget > /dev/null; then
if [ "$MVNW_VERBOSE" = true ]; then
echo "Found wget ... using wget"
fi
if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
wget "$jarUrl" -O "$wrapperJarPath"
else
wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath"
fi
elif command -v curl > /dev/null; then
if [ -e "$javaClass" ]; then
if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
if [ "$MVNW_VERBOSE" = true ]; then
echo "Found curl ... using curl"
fi
if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
curl -o "$wrapperJarPath" "$jarUrl" -f
else
curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f
echo " - Compiling MavenWrapperDownloader.java ..."
fi
else
# Compiling the Java class
("$JAVA_HOME/bin/javac" "$javaClass")
fi
if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
# Running the downloader
if [ "$MVNW_VERBOSE" = true ]; then
echo "Falling back to using Java to download"
fi
javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java"
# For Cygwin, switch paths to Windows format before running javac
if $cygwin; then
javaClass=`cygpath --path --windows "$javaClass"`
fi
if [ -e "$javaClass" ]; then
if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
if [ "$MVNW_VERBOSE" = true ]; then
echo " - Compiling MavenWrapperDownloader.java ..."
fi
# Compiling the Java class
("$JAVA_HOME/bin/javac" "$javaClass")
fi
if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
# Running the downloader
if [ "$MVNW_VERBOSE" = true ]; then
echo " - Running MavenWrapperDownloader.java ..."
fi
("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR")
fi
echo " - Running MavenWrapperDownloader.java ..."
fi
("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR")
fi
fi
fi
fi
##########################################################################################
# End of extension
@ -280,25 +294,25 @@ fi
export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}
if [ "$MVNW_VERBOSE" = true ]; then
echo $MAVEN_PROJECTBASEDIR
echo "$MAVEN_PROJECTBASEDIR"
fi
MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
# For Cygwin, switch paths to Windows format before running java
if $cygwin; then
[ -n "$M2_HOME" ] &&
M2_HOME=`cygpath --path --windows "$M2_HOME"`
M2_HOME=$(cygpath --path --windows "$M2_HOME")
[ -n "$JAVA_HOME" ] &&
JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"`
JAVA_HOME=$(cygpath --path --windows "$JAVA_HOME")
[ -n "$CLASSPATH" ] &&
CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
CLASSPATH=$(cygpath --path --windows "$CLASSPATH")
[ -n "$MAVEN_PROJECTBASEDIR" ] &&
MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"`
MAVEN_PROJECTBASEDIR=$(cygpath --path --windows "$MAVEN_PROJECTBASEDIR")
fi
# Provide a "standardized" way to retrieve the CLI args that will
# work with both Windows and non-Windows executions.
MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@"
MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $*"
export MAVEN_CMD_LINE_ARGS
WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain

Write Preview
Loading…
Cancel
Save