diff --git a/docker/build/hooks/build b/docker/build/hooks/build index 3276f167c4..6362fdd299 100644 --- a/docker/build/hooks/build +++ b/docker/build/hooks/build @@ -24,7 +24,8 @@ printenv if [ -z "${VERSION}" ] then echo "set default environment variable [VERSION]" - export VERSION=$(cat $(pwd)/pom.xml | grep '' -m 1 | awk '{print $1}' | sed 's///' | sed 's/<\/version>//') + VERSION=$(grep '' -m 1 "$(pwd)"/pom.xml | awk '{print $1}' | sed 's///' | sed 's/<\/version>//') + export VERSION fi if [ "${DOCKER_REPO}x" = "x" ] @@ -44,10 +45,10 @@ mvn -B clean compile package -Prelease -Dmaven.test.skip=true # mv dolphinscheduler-bin.tar.gz file to docker/build directory echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/\n" -mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/ +mv "$(pwd)"/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-"${VERSION}"-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/ # docker build echo -e "docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/\n" -sudo docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/ +sudo docker build --build-arg VERSION="${VERSION}" -t $DOCKER_REPO:"${VERSION}" "$(pwd)/docker/build/" echo "------ dolphinscheduler end - build -------" diff --git a/docker/build/hooks/push b/docker/build/hooks/push index 41a25c54fe..74ea06d5c5 100644 --- a/docker/build/hooks/push +++ b/docker/build/hooks/push @@ -19,6 +19,6 @@ echo "------ push start -------" printenv -docker push $DOCKER_REPO:${VERSION} +docker push "$DOCKER_REPO:${VERSION}" echo "------ push end -------" diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java index 347336cada..bf791ac9f0 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.alert; import org.apache.dolphinscheduler.alert.plugin.EmailAlertPlugin; @@ -25,11 +26,12 @@ import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.DaoFactory; import org.apache.dolphinscheduler.dao.entity.Alert; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * alert of start */ @@ -40,8 +42,6 @@ public class AlertServer { */ private AlertDao alertDao = DaoFactory.getDaoInstance(AlertDao.class); - private AlertSender alertSender; - private static AlertServer instance; private FilePluginManager alertPluginManager; @@ -61,7 +61,7 @@ public class AlertServer { alertPluginManager.addPlugin(new EmailAlertPlugin()); } - public synchronized static AlertServer getInstance() { + public static synchronized AlertServer getInstance() { if (null == instance) { instance = new AlertServer(); } @@ -78,12 +78,11 @@ public class AlertServer { Thread.currentThread().interrupt(); } List alerts = alertDao.listWaitExecutionAlert(); - alertSender = new AlertSender(alerts, alertDao, alertPluginManager); + AlertSender alertSender = new AlertSender(alerts, alertDao, alertPluginManager); alertSender.run(); } } - public static void main(String[] args) { AlertServer alertServer = AlertServer.getInstance(); alertServer.start(); diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/DingTalkManager.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/DingTalkManager.java index 6840794026..871ad958c4 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/DingTalkManager.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/DingTalkManager.java @@ -14,40 +14,41 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.alert.manager; import org.apache.dolphinscheduler.alert.utils.Constants; import org.apache.dolphinscheduler.alert.utils.DingTalkUtils; import org.apache.dolphinscheduler.plugin.model.AlertInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.HashMap; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Ding Talk Manager */ public class DingTalkManager { - private static final Logger logger = LoggerFactory.getLogger(EnterpriseWeChatManager.class); + private static final Logger logger = LoggerFactory.getLogger(DingTalkManager.class); - public Map send(AlertInfo alert) { - Map retMap = new HashMap<>(); + public Map send(AlertInfo alert) { + Map retMap = new HashMap<>(); retMap.put(Constants.STATUS, false); logger.info("send message {}", alert.getAlertData().getTitle()); try { String msg = buildMessage(alert); DingTalkUtils.sendDingTalkMsg(msg, Constants.UTF_8); } catch (IOException e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); } retMap.put(Constants.STATUS, true); return retMap; } private String buildMessage(AlertInfo alert) { - String msg = alert.getAlertData().getContent(); - return msg; + return alert.getAlertData().getContent(); } } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java index fbc600f39e..5bbc21930f 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.alert.plugin; import org.apache.dolphinscheduler.alert.manager.DingTalkManager; @@ -28,14 +29,19 @@ import org.apache.dolphinscheduler.plugin.api.AlertPlugin; import org.apache.dolphinscheduler.plugin.model.AlertData; import org.apache.dolphinscheduler.plugin.model.AlertInfo; import org.apache.dolphinscheduler.plugin.model.PluginName; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; - /** * EmailAlertPlugin - * + *

* This plugin is a default plugin, and mix up email and enterprise wechat, because adapt with former alert behavior */ public class EmailAlertPlugin implements AlertPlugin { @@ -125,10 +131,10 @@ public class EmailAlertPlugin implements AlertPlugin { } } - if (DingTalkUtils.isEnableDingTalk) { + if (DingTalkUtils.IS_ENABLE_DING_TALK) { logger.info("Ding Talk is enable."); - dingTalkManager.send(info); - } + dingTalkManager.send(info); + } } else { retMaps.put(Constants.MESSAGE, "alert send error."); diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactory.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactory.java index 965677e7e1..d38463123e 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactory.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactory.java @@ -14,23 +14,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.alert.template; import org.apache.dolphinscheduler.alert.template.impl.DefaultHTMLTemplate; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * the alert template factory */ public class AlertTemplateFactory { - private static final Logger logger = LoggerFactory.getLogger(AlertTemplateFactory.class); - - private AlertTemplateFactory(){} + private AlertTemplateFactory() { + } /** * get a template from alert.properties conf file + * * @return a template, default is DefaultHTMLTemplate */ public static AlertTemplate getMessageTemplate() { diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java index a01f301a24..f590849660 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java @@ -14,21 +14,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.alert.template.impl; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; +import static org.apache.dolphinscheduler.common.utils.Preconditions.checkNotNull; + import org.apache.dolphinscheduler.alert.template.AlertTemplate; import org.apache.dolphinscheduler.alert.utils.Constants; import org.apache.dolphinscheduler.common.enums.ShowType; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; + +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.dolphinscheduler.common.utils.*; -import java.util.*; - -import static org.apache.dolphinscheduler.common.utils.Preconditions.*; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; /** * the default html alert message template @@ -37,33 +45,33 @@ public class DefaultHTMLTemplate implements AlertTemplate { public static final Logger logger = LoggerFactory.getLogger(DefaultHTMLTemplate.class); - @Override - public String getMessageFromTemplate(String content, ShowType showType,boolean showAll) { + public String getMessageFromTemplate(String content, ShowType showType, boolean showAll) { - switch (showType){ + switch (showType) { case TABLE: - return getTableTypeMessage(content,showAll); + return getTableTypeMessage(content, showAll); case TEXT: - return getTextTypeMessage(content,showAll); + return getTextTypeMessage(content); default: - throw new IllegalArgumentException(String.format("not support showType: %s in DefaultHTMLTemplate",showType)); + throw new IllegalArgumentException(String.format("not support showType: %s in DefaultHTMLTemplate", showType)); } } /** * get alert message which type is TABLE + * * @param content message content * @param showAll weather to show all * @return alert message */ - private String getTableTypeMessage(String content,boolean showAll){ + private String getTableTypeMessage(String content, boolean showAll) { - if (StringUtils.isNotEmpty(content)){ + if (StringUtils.isNotEmpty(content)) { List mapItemsList = JSONUtils.toList(content, LinkedHashMap.class); - if(!showAll && mapItemsList.size() > Constants.NUMBER_1000){ - mapItemsList = mapItemsList.subList(0,Constants.NUMBER_1000); + if (!showAll && mapItemsList.size() > Constants.NUMBER_1000) { + mapItemsList = mapItemsList.subList(0, Constants.NUMBER_1000); } StringBuilder contents = new StringBuilder(200); @@ -71,15 +79,15 @@ public class DefaultHTMLTemplate implements AlertTemplate { boolean flag = true; String title = ""; - for (LinkedHashMap mapItems : mapItemsList){ + for (LinkedHashMap mapItems : mapItemsList) { - Set> entries = mapItems.entrySet(); + Set> entries = mapItems.entrySet(); - Iterator> iterator = entries.iterator(); + Iterator> iterator = entries.iterator(); StringBuilder t = new StringBuilder(Constants.TR); StringBuilder cs = new StringBuilder(Constants.TR); - while (iterator.hasNext()){ + while (iterator.hasNext()) { Map.Entry entry = iterator.next(); t.append(Constants.TH).append(entry.getKey()).append(Constants.TH_END); @@ -88,14 +96,14 @@ public class DefaultHTMLTemplate implements AlertTemplate { } t.append(Constants.TR_END); cs.append(Constants.TR_END); - if (flag){ + if (flag) { title = t.toString(); } flag = false; contents.append(cs); } - return getMessageFromHtmlTemplate(title,contents.toString()); + return getMessageFromHtmlTemplate(title, contents.toString()); } return content; @@ -103,22 +111,22 @@ public class DefaultHTMLTemplate implements AlertTemplate { /** * get alert message which type is TEXT + * * @param content message content - * @param showAll weather to show all * @return alert message */ - private String getTextTypeMessage(String content,boolean showAll){ + private String getTextTypeMessage(String content) { - if (StringUtils.isNotEmpty(content)){ + if (StringUtils.isNotEmpty(content)) { ArrayNode list = JSONUtils.parseArray(content); StringBuilder contents = new StringBuilder(100); - for (JsonNode jsonNode : list){ + for (JsonNode jsonNode : list) { contents.append(Constants.TR); contents.append(Constants.TD).append(jsonNode.toString()).append(Constants.TD_END); contents.append(Constants.TR_END); } - return getMessageFromHtmlTemplate(null,contents.toString()); + return getMessageFromHtmlTemplate(null, contents.toString()); } @@ -127,16 +135,17 @@ public class DefaultHTMLTemplate implements AlertTemplate { /** * get alert message from a html template - * @param title message title - * @param content message content + * + * @param title message title + * @param content message content * @return alert message which use html template */ - private String getMessageFromHtmlTemplate(String title,String content){ + private String getMessageFromHtmlTemplate(String title, String content) { checkNotNull(content); - String htmlTableThead = StringUtils.isEmpty(title) ? "" : String.format("%s\n",title); + String htmlTableThead = StringUtils.isEmpty(title) ? "" : String.format("%s%n", title); - return Constants.HTML_HEADER_PREFIX +htmlTableThead + content + Constants.TABLE_BODY_HTML_TAIL; + return Constants.HTML_HEADER_PREFIX + htmlTableThead + content + Constants.TABLE_BODY_HTML_TAIL; } } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java index 465d9bf895..c0f916a809 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.alert.utils; /** @@ -23,6 +24,7 @@ public class Constants { private Constants() { throw new IllegalStateException("Constants class"); } + /** * alert properties path */ @@ -67,7 +69,7 @@ public class Constants { public static final String MAIL_SMTP_SSL_ENABLE = "mail.smtp.ssl.enable"; - public static final String MAIL_SMTP_SSL_TRUST="mail.smtp.ssl.trust"; + public static final String MAIL_SMTP_SSL_TRUST = "mail.smtp.ssl.trust"; public static final String TEXT_HTML_CHARSET_UTF_8 = "text/html;charset=utf-8"; @@ -156,7 +158,7 @@ public class Constants { public static final String ENTERPRISE_WECHAT_AGENT_ID = "enterprise.wechat.agent.id"; public static final String ENTERPRISE_WECHAT_USERS = "enterprise.wechat.users"; - + public static final String DINGTALK_WEBHOOK = "dingtalk.webhook"; @@ -174,7 +176,13 @@ public class Constants { public static final String DINGTALK_ENABLE = "dingtalk.isEnable"; - public static final String HTML_HEADER_PREFIX = "dolphinscheduler "; + public static final String HTML_HEADER_PREFIX = "" + + "dolphinscheduler" + + "" + + "" + + "/head>
"; public static final String TABLE_BODY_HTML_TAIL = "
"; diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtils.java index 455d5de834..50a62e46fd 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtils.java @@ -14,10 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.alert.utils; +package org.apache.dolphinscheduler.alert.utils; import org.apache.dolphinscheduler.common.utils.*; + import org.apache.commons.codec.binary.StringUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; @@ -32,44 +33,50 @@ import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.HashMap; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * DingTalkUtils utils * support send msg to ding talk by robot message push function. - * support proxy setting + * support PROXY setting */ public class DingTalkUtils { public static final Logger logger = LoggerFactory.getLogger(DingTalkUtils.class); - public static final boolean isEnableDingTalk = PropertyUtils.getBoolean(Constants.DINGTALK_ENABLE); - private static final String dingTaskUrl = PropertyUtils.getString(Constants.DINGTALK_WEBHOOK); - private static final String keyword = PropertyUtils.getString(Constants.DINGTALK_KEYWORD); - private static final Boolean isEnableProxy = PropertyUtils.getBoolean(Constants.DINGTALK_PROXY_ENABLE); - private static final String proxy = PropertyUtils.getString(Constants.DINGTALK_PROXY); - private static final String user = PropertyUtils.getString(Constants.DINGTALK_USER); - private static final String passwd = PropertyUtils.getString(Constants.DINGTALK_PASSWORD); - private static final Integer port = PropertyUtils.getInt(Constants.DINGTALK_PORT); + public static final boolean IS_ENABLE_DING_TALK = PropertyUtils.getBoolean(Constants.DINGTALK_ENABLE); + private static final String DING_TASK_URL = PropertyUtils.getString(Constants.DINGTALK_WEBHOOK); + private static final String KEYWORD = PropertyUtils.getString(Constants.DINGTALK_KEYWORD); + private static final Boolean IS_ENABLE_PROXY = PropertyUtils.getBoolean(Constants.DINGTALK_PROXY_ENABLE); + private static final String PROXY = PropertyUtils.getString(Constants.DINGTALK_PROXY); + private static final String USER = PropertyUtils.getString(Constants.DINGTALK_USER); + private static final String PASSWD = PropertyUtils.getString(Constants.DINGTALK_PASSWORD); + private static final Integer PORT = PropertyUtils.getInt(Constants.DINGTALK_PORT); + + private DingTalkUtils() { + throw new IllegalStateException(DingTalkUtils.class.getName()); + } /** * send message interface * only support text message format now. + * * @param msg message context to send * @param charset charset type - * @return result of sending msg + * @return result of sending msg * @throws IOException the IOException */ public static String sendDingTalkMsg(String msg, String charset) throws IOException { - String msgToJson = textToJsonString(msg + "#" + keyword); + String msgToJson = textToJsonString(msg + "#" + KEYWORD); HttpPost httpPost = constructHttpPost(msgToJson, charset); CloseableHttpClient httpClient; - if (isEnableProxy) { + if (Boolean.TRUE.equals(IS_ENABLE_PROXY)) { httpClient = getProxyClient(); RequestConfig rcf = getProxyConfig(); httpPost.setConfig(rcf); @@ -87,28 +94,26 @@ public class DingTalkUtils { } finally { response.close(); } - logger.info("Ding Talk send [{}], resp:{%s}", msg, resp); + logger.info("Ding Talk send [{}], resp:{%s}", msg); return resp; - } finally { + } finally { httpClient.close(); } } public static HttpPost constructHttpPost(String msg, String charset) { - HttpPost post = new HttpPost(dingTaskUrl); + HttpPost post = new HttpPost(DING_TASK_URL); StringEntity entity = new StringEntity(msg, charset); post.setEntity(entity); post.addHeader("Content-Type", "application/json; charset=utf-8"); return post; } - public static CloseableHttpClient getProxyClient() { - HttpHost httpProxy = new HttpHost(proxy, port); + HttpHost httpProxy = new HttpHost(PROXY, PORT); CredentialsProvider provider = new BasicCredentialsProvider(); - provider.setCredentials(new AuthScope(httpProxy), new UsernamePasswordCredentials(user, passwd)); - CloseableHttpClient httpClient = HttpClients.custom().setDefaultCredentialsProvider(provider).build(); - return httpClient; + provider.setCredentials(new AuthScope(httpProxy), new UsernamePasswordCredentials(USER, PASSWD)); + return HttpClients.custom().setDefaultCredentialsProvider(provider).build(); } public static CloseableHttpClient getDefaultClient() { @@ -116,14 +121,14 @@ public class DingTalkUtils { } public static RequestConfig getProxyConfig() { - HttpHost httpProxy = new HttpHost(proxy, port); + HttpHost httpProxy = new HttpHost(PROXY, PORT); return RequestConfig.custom().setProxy(httpProxy).build(); } public static String textToJsonString(String text) { - Map items = new HashMap(); + Map items = new HashMap<>(); items.put("msgtype", "text"); - Map textContent = new HashMap(); + Map textContent = new HashMap<>(); byte[] byt = StringUtils.getBytesUtf8(text); String txt = StringUtils.newStringUtf8(byt); textContent.put("content", txt); diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java index b3cb5f7c4e..aeb6671791 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java @@ -60,9 +60,13 @@ public class EnterpriseWeChatUtils { private static final String ENTERPRISE_WE_CHAT_TEAM_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TEAM_SEND_MSG); private static final String ENTERPRISE_WE_CHAT_USER_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USER_SEND_MSG); - private static final String agentIdRegExp = "\\{agentId}"; - private static final String msgRegExp = "\\{msg}"; - private static final String userRegExp = "\\{toUser}"; + private static final String AGENT_ID_REG_EXP = "\\{agentId}"; + private static final String MSG_REG_EXP = "\\{msg}"; + private static final String USER_REG_EXP = "\\{toUser}"; + + private EnterpriseWeChatUtils() { + throw new IllegalStateException(EnterpriseWeChatUtils.class.getName()); + } /** * get Enterprise WeChat is enable @@ -120,8 +124,8 @@ public class EnterpriseWeChatUtils { */ public static String makeTeamSendMsg(String toParty, String agentId, String msg) { return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty}", toParty) - .replaceAll(agentIdRegExp, agentId) - .replaceAll(msgRegExp, msg); + .replaceAll(AGENT_ID_REG_EXP, agentId) + .replaceAll(MSG_REG_EXP, msg); } /** @@ -135,8 +139,8 @@ public class EnterpriseWeChatUtils { public static String makeTeamSendMsg(Collection toParty, String agentId, String msg) { String listParty = FuncUtils.mkString(toParty, "|"); return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty}", listParty) - .replaceAll(agentIdRegExp, agentId) - .replaceAll(msgRegExp, msg); + .replaceAll(AGENT_ID_REG_EXP, agentId) + .replaceAll(MSG_REG_EXP, msg); } /** @@ -148,9 +152,9 @@ public class EnterpriseWeChatUtils { * @return Enterprise WeChat send message */ public static String makeUserSendMsg(String toUser, String agentId, String msg) { - return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\{toUser}", toUser) - .replaceAll(agentIdRegExp, agentId) - .replaceAll(msgRegExp, msg); + return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll(USER_REG_EXP, toUser) + .replaceAll(AGENT_ID_REG_EXP, agentId) + .replaceAll(MSG_REG_EXP, msg); } /** @@ -163,9 +167,9 @@ public class EnterpriseWeChatUtils { */ public static String makeUserSendMsg(Collection toUser, String agentId, String msg) { String listUser = FuncUtils.mkString(toUser, "|"); - return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll(userRegExp, listUser) - .replaceAll(agentIdRegExp, agentId) - .replaceAll(msgRegExp, msg); + return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll(USER_REG_EXP, listUser) + .replaceAll(AGENT_ID_REG_EXP, agentId) + .replaceAll(MSG_REG_EXP, msg); } /** diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java index 08256860e2..76ce66ac2f 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/ExcelUtils.java @@ -14,23 +14,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.alert.utils; import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; + import org.apache.poi.hssf.usermodel.HSSFCell; import org.apache.poi.hssf.usermodel.HSSFRow; import org.apache.poi.hssf.usermodel.HSSFSheet; import org.apache.poi.hssf.usermodel.HSSFWorkbook; import org.apache.poi.ss.usermodel.CellStyle; import org.apache.poi.ss.usermodel.HorizontalAlignment; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.util.*; -import org.apache.dolphinscheduler.common.utils.*; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * excel utils @@ -38,19 +44,25 @@ import org.apache.dolphinscheduler.common.utils.*; public class ExcelUtils { private static final Logger logger = LoggerFactory.getLogger(ExcelUtils.class); + + private ExcelUtils() { + throw new IllegalStateException(ExcelUtils.class.getName()); + } + /** * generate excel file + * * @param content the content * @param title the title * @param xlsFilePath the xls path */ - public static void genExcelFile(String content,String title,String xlsFilePath){ + public static void genExcelFile(String content, String title, String xlsFilePath) { List itemsList; //The JSONUtils.toList has been try catch ex itemsList = JSONUtils.toList(content, LinkedHashMap.class); - if (CollectionUtils.isEmpty(itemsList)){ + if (CollectionUtils.isEmpty(itemsList)) { logger.error("itemsList is null"); throw new RuntimeException("itemsList is null"); } @@ -59,82 +71,79 @@ public class ExcelUtils { List headerList = new ArrayList<>(); - Iterator> iter = headerMap.entrySet().iterator(); - while (iter.hasNext()){ - Map.Entry en = iter.next(); + for (Map.Entry en : headerMap.entrySet()) { headerList.add(en.getKey()); } HSSFWorkbook wb = null; FileOutputStream fos = null; - try { - // declare a workbook - wb = new HSSFWorkbook(); - // generate a table - HSSFSheet sheet = wb.createSheet(); - HSSFRow row = sheet.createRow(0); - //set the height of the first line - row.setHeight((short)500); - - //set Horizontal right - CellStyle cellStyle = wb.createCellStyle(); - cellStyle.setAlignment(HorizontalAlignment.RIGHT); - - //setting excel headers - for (int i = 0; i < headerList.size(); i++) { - HSSFCell cell = row.createCell(i); - cell.setCellStyle(cellStyle); - cell.setCellValue(headerList.get(i)); - } - - //setting excel body - int rowIndex = 1; - for (LinkedHashMap itemsMap : itemsList){ - Object[] values = itemsMap.values().toArray(); - row = sheet.createRow(rowIndex); - //setting excel body height - row.setHeight((short)500); - rowIndex++; - for (int j = 0 ; j < values.length ; j++){ - HSSFCell cell1 = row.createCell(j); - cell1.setCellStyle(cellStyle); - cell1.setCellValue(String.valueOf(values[j])); - } - } - - for (int i = 0; i < headerList.size(); i++) { - sheet.setColumnWidth(i, headerList.get(i).length() * 800); - } - - File file = new File(xlsFilePath); - if (!file.exists()) { - file.mkdirs(); - } - - //setting file output - fos = new FileOutputStream(xlsFilePath + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS); - - wb.write(fos); - - }catch (Exception e){ - logger.error("generate excel error",e); - throw new RuntimeException("generate excel error",e); - }finally { - if (wb != null){ - try { - wb.close(); - } catch (IOException e) { - logger.error(e.getMessage(),e); - } - } - if (fos != null){ - try { - fos.close(); - } catch (IOException e) { - logger.error(e.getMessage(),e); - } - } - } + try { + // declare a workbook + wb = new HSSFWorkbook(); + // generate a table + HSSFSheet sheet = wb.createSheet(); + HSSFRow row = sheet.createRow(0); + //set the height of the first line + row.setHeight((short) 500); + + //set Horizontal right + CellStyle cellStyle = wb.createCellStyle(); + cellStyle.setAlignment(HorizontalAlignment.RIGHT); + + //setting excel headers + for (int i = 0; i < headerList.size(); i++) { + HSSFCell cell = row.createCell(i); + cell.setCellStyle(cellStyle); + cell.setCellValue(headerList.get(i)); + } + + //setting excel body + int rowIndex = 1; + for (LinkedHashMap itemsMap : itemsList) { + Object[] values = itemsMap.values().toArray(); + row = sheet.createRow(rowIndex); + //setting excel body height + row.setHeight((short) 500); + rowIndex++; + for (int j = 0; j < values.length; j++) { + HSSFCell cell1 = row.createCell(j); + cell1.setCellStyle(cellStyle); + cell1.setCellValue(String.valueOf(values[j])); + } + } + + for (int i = 0; i < headerList.size(); i++) { + sheet.setColumnWidth(i, headerList.get(i).length() * 800); + } + + File file = new File(xlsFilePath); + if (!file.exists()) { + file.mkdirs(); + } + + //setting file output + fos = new FileOutputStream(xlsFilePath + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS); + + wb.write(fos); + + } catch (Exception e) { + throw new RuntimeException("generate excel error", e); + } finally { + if (wb != null) { + try { + wb.close(); + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + } + if (fos != null) { + try { + fos.close(); + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + } + } } } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java index d68532a82b..e78b4ebec8 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java @@ -14,15 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.alert.utils; import org.apache.dolphinscheduler.common.utils.StringUtils; public class FuncUtils { + private FuncUtils() { + throw new IllegalStateException(FuncUtils.class.getName()); + } + public static String mkString(Iterable list, String split) { - if (null == list || StringUtils.isEmpty(split)){ + if (null == list || StringUtils.isEmpty(split)) { return null; } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java index 888c9dbb26..f57481bd0c 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java @@ -14,23 +14,39 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.alert.utils; import org.apache.dolphinscheduler.alert.template.AlertTemplate; import org.apache.dolphinscheduler.alert.template.AlertTemplateFactory; import org.apache.dolphinscheduler.common.enums.ShowType; -import org.apache.commons.mail.EmailException; -import org.apache.commons.mail.HtmlEmail; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import javax.mail.*; -import javax.mail.internet.*; -import java.io.*; -import java.util.*; +import org.apache.commons.mail.EmailException; +import org.apache.commons.mail.HtmlEmail; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import javax.mail.Authenticator; +import javax.mail.Message; +import javax.mail.MessagingException; +import javax.mail.PasswordAuthentication; +import javax.mail.Session; +import javax.mail.Transport; +import javax.mail.internet.InternetAddress; +import javax.mail.internet.MimeBodyPart; +import javax.mail.internet.MimeMessage; +import javax.mail.internet.MimeMultipart; +import javax.mail.internet.MimeUtility; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * mail utils @@ -55,7 +71,7 @@ public class MailUtils { public static final Boolean MAIL_USE_SSL = PropertyUtils.getBoolean(Constants.MAIL_SMTP_SSL_ENABLE); - public static final String xlsFilePath = PropertyUtils.getString(Constants.XLS_FILE_PATH,"/tmp/xls"); + public static final String XLS_FILE_PATH = PropertyUtils.getString(Constants.XLS_FILE_PATH, "/tmp/xls"); public static final String STARTTLS_ENABLE = PropertyUtils.getString(Constants.MAIL_SMTP_STARTTLS_ENABLE); @@ -67,23 +83,29 @@ public class MailUtils { //Solve the problem of messy Chinese name in excel attachment static { - System.setProperty("mail.mime.splitlongparameters","false"); + System.setProperty("mail.mime.splitlongparameters", "false"); + } + + private MailUtils() { + throw new IllegalStateException(MailUtils.class.getName()); } /** * send mail to receivers + * * @param receivers the receiver list * @param title the title * @param content the content * @param showType the show type * @return the result map */ - public static Map sendMails(Collection receivers, String title, String content,String showType) { + public static Map sendMails(Collection receivers, String title, String content, String showType) { return sendMails(receivers, null, title, content, showType); } /** * send mail + * * @param receivers the receiver list * @param receiversCc cc list * @param title the title @@ -91,8 +113,8 @@ public class MailUtils { * @param showType the show type * @return the send result */ - public static Map sendMails(Collection receivers, Collection receiversCc, String title, String content, String showType) { - Map retMap = new HashMap<>(); + public static Map sendMails(Collection receivers, Collection receiversCc, String title, String content, String showType) { + Map retMap = new HashMap<>(); retMap.put(Constants.STATUS, false); // if there is no receivers && no receiversCc, no need to process @@ -111,14 +133,14 @@ public class MailUtils { email.setMailSession(session); email.setFrom(MAIL_SENDER); email.setCharset(Constants.UTF_8); - if (CollectionUtils.isNotEmpty(receivers)){ + if (CollectionUtils.isNotEmpty(receivers)) { // receivers mail for (String receiver : receivers) { email.addTo(receiver); } } - if (CollectionUtils.isNotEmpty(receiversCc)){ + if (CollectionUtils.isNotEmpty(receiversCc)) { //cc for (String receiverCc : receiversCc) { email.addCc(receiverCc); @@ -129,16 +151,16 @@ public class MailUtils { } catch (Exception e) { handleException(receivers, retMap, e); } - }else if (showType.equals(ShowType.ATTACHMENT.getDescp()) || showType.equals(ShowType.TABLEATTACHMENT.getDescp())) { + } else if (showType.equals(ShowType.ATTACHMENT.getDescp()) || showType.equals(ShowType.TABLEATTACHMENT.getDescp())) { try { - String partContent = (showType.equals(ShowType.ATTACHMENT.getDescp()) ? "Please see the attachment " + title + Constants.EXCEL_SUFFIX_XLS : htmlTable(content,false)); + String partContent = (showType.equals(ShowType.ATTACHMENT.getDescp()) ? "Please see the attachment " + title + Constants.EXCEL_SUFFIX_XLS : htmlTable(content, false)); - attachment(receivers,receiversCc,title,content,partContent); + attachment(receivers, receiversCc, title, content, partContent); retMap.put(Constants.STATUS, true); return retMap; - }catch (Exception e){ + } catch (Exception e) { handleException(receivers, retMap, e); return retMap; } @@ -149,49 +171,52 @@ public class MailUtils { /** * html table content + * * @param content the content * @param showAll if show the whole content * @return the html table form */ - private static String htmlTable(String content, boolean showAll){ - return alertTemplate.getMessageFromTemplate(content,ShowType.TABLE,showAll); + private static String htmlTable(String content, boolean showAll) { + return alertTemplate.getMessageFromTemplate(content, ShowType.TABLE, showAll); } /** * html table content + * * @param content the content * @return the html table form */ - private static String htmlTable(String content){ - return htmlTable(content,true); + private static String htmlTable(String content) { + return htmlTable(content, true); } /** * html text content + * * @param content the content * @return text in html form */ - private static String htmlText(String content){ - return alertTemplate.getMessageFromTemplate(content,ShowType.TEXT); + private static String htmlText(String content) { + return alertTemplate.getMessageFromTemplate(content, ShowType.TEXT); } /** * send mail as Excel attachment + * * @param receivers the receiver list * @param title the title - * @throws Exception */ - private static void attachment(Collection receivers,Collection receiversCc,String title,String content,String partContent)throws Exception{ + private static void attachment(Collection receivers, Collection receiversCc, String title, String content, String partContent) throws Exception { MimeMessage msg = getMimeMessage(receivers); - attachContent(receiversCc, title, content,partContent, msg); + attachContent(receiversCc, title, content, partContent, msg); } /** * get MimeMessage + * * @param receivers receivers * @return the MimeMessage - * @throws MessagingException */ private static MimeMessage getMimeMessage(Collection receivers) throws MessagingException { @@ -223,7 +248,7 @@ public class MailUtils { props.setProperty(Constants.MAIL_SMTP_AUTH, Constants.STRING_TRUE); props.setProperty(Constants.MAIL_TRANSPORT_PROTOCOL, MAIL_PROTOCOL); props.setProperty(Constants.MAIL_SMTP_STARTTLS_ENABLE, STARTTLS_ENABLE); - if (SSL_ENABLE) { + if (Boolean.TRUE.equals(SSL_ENABLE)) { props.setProperty(Constants.MAIL_SMTP_SSL_ENABLE, "true"); props.setProperty(Constants.MAIL_SMTP_SSL_TRUST, SSL_TRUST); } @@ -241,20 +266,19 @@ public class MailUtils { /** * attach content + * * @param receiversCc the cc list * @param title the title * @param content the content * @param partContent the partContent * @param msg the message - * @throws MessagingException - * @throws IOException */ - private static void attachContent(Collection receiversCc, String title, String content, String partContent,MimeMessage msg) throws MessagingException, IOException { - /** + private static void attachContent(Collection receiversCc, String title, String content, String partContent, MimeMessage msg) throws MessagingException, IOException { + /* * set receiverCc */ - if(CollectionUtils.isNotEmpty(receiversCc)){ - for (String receiverCc : receiversCc){ + if (CollectionUtils.isNotEmpty(receiversCc)) { + for (String receiverCc : receiversCc) { msg.addRecipients(Message.RecipientType.CC, InternetAddress.parse(receiverCc)); } } @@ -267,16 +291,16 @@ public class MailUtils { part1.setContent(partContent, Constants.TEXT_HTML_CHARSET_UTF_8); // set attach file MimeBodyPart part2 = new MimeBodyPart(); - File file = new File(xlsFilePath + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS); + File file = new File(XLS_FILE_PATH + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS); if (!file.getParentFile().exists()) { file.getParentFile().mkdirs(); } // make excel file - ExcelUtils.genExcelFile(content,title,xlsFilePath); + ExcelUtils.genExcelFile(content, title, XLS_FILE_PATH); part2.attachFile(file); - part2.setFileName(MimeUtility.encodeText(title + Constants.EXCEL_SUFFIX_XLS,Constants.UTF_8,"B")); + part2.setFileName(MimeUtility.encodeText(title + Constants.EXCEL_SUFFIX_XLS, Constants.UTF_8, "B")); // add components to collection partList.addBodyPart(part1); partList.addBodyPart(part2); @@ -289,21 +313,21 @@ public class MailUtils { /** * the string object map + * * @param title the title * @param content the content * @param showType the showType * @param retMap the result map * @param email the email * @return the result map - * @throws EmailException */ private static Map getStringObjectMap(String title, String content, String showType, Map retMap, HtmlEmail email) throws EmailException { - /** + /* * the subject of the message to be sent */ email.setSubject(title); - /** + /* * to send information, you can use HTML tags in mail content because of the use of HtmlEmail */ if (showType.equals(ShowType.TABLE.getDescp())) { @@ -322,23 +346,24 @@ public class MailUtils { /** * file delete + * * @param file the file to delete */ - public static void deleteFile(File file){ - if(file.exists()){ - if(file.delete()){ - logger.info("delete success: {}",file.getAbsolutePath() + file.getName()); - }else{ + public static void deleteFile(File file) { + if (file.exists()) { + if (file.delete()) { + logger.info("delete success: {}", file.getAbsolutePath() + file.getName()); + } else { logger.info("delete fail: {}", file.getAbsolutePath() + file.getName()); } - }else{ + } else { logger.info("file not exists: {}", file.getAbsolutePath() + file.getName()); } } - /** * handle exception + * * @param receivers the receiver list * @param retMap the result map * @param e the exception @@ -347,6 +372,4 @@ public class MailUtils { logger.error("Send email to {} failed", receivers, e); retMap.put(Constants.MESSAGE, "Send email to {" + String.join(",", receivers) + "} failed，" + e.toString()); } - - } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/PropertyUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/PropertyUtils.java index 91f7261db2..0eb241366a 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/PropertyUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/PropertyUtils.java @@ -14,19 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.alert.utils; +import static org.apache.dolphinscheduler.alert.utils.Constants.ALERT_PROPERTIES_PATH; + import org.apache.dolphinscheduler.common.utils.IOUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.InputStream; import java.util.Properties; import java.util.regex.PatternSyntaxException; -import static org.apache.dolphinscheduler.alert.utils.Constants.ALERT_PROPERTIES_PATH; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * property utils @@ -41,13 +43,11 @@ public class PropertyUtils { private static final Properties properties = new Properties(); - private static final PropertyUtils propertyUtils = new PropertyUtils(); - - private PropertyUtils(){ + private PropertyUtils() { init(); } - private void init(){ + private void init() { String[] propertyFiles = new String[]{ALERT_PROPERTIES_PATH}; for (String fileName : propertyFiles) { InputStream fis = null; @@ -69,6 +69,7 @@ public class PropertyUtils { /** * get property value + * * @param key property name * @return the value */ @@ -95,7 +96,7 @@ public class PropertyUtils { * get property value * * @param key property name - * @return get property int value , if key == null, then return -1 + * @return get property int value , if key == null, then return -1 */ public static int getInt(String key) { @@ -104,6 +105,7 @@ public class PropertyUtils { /** * get int value + * * @param key the key * @param defaultValue the default value * @return the value related the key or the default value if the key not existed @@ -117,15 +119,16 @@ public class PropertyUtils { try { return Integer.parseInt(value); } catch (NumberFormatException e) { - logger.info(e.getMessage(),e); + logger.info(e.getMessage(), e); } return defaultValue; } /** * get property value + * * @param key property name - * @return the boolean result value + * @return the boolean result value */ public static Boolean getBoolean(String key) { @@ -134,7 +137,7 @@ public class PropertyUtils { } String value = properties.getProperty(key.trim()); - if(null != value){ + if (null != value) { return Boolean.parseBoolean(value); } @@ -143,15 +146,17 @@ public class PropertyUtils { /** * get long value + * * @param key the key * @return if the value not existed, return -1, or will return the related value */ public static long getLong(String key) { - return getLong(key,-1); + return getLong(key, -1); } /** * get long value + * * @param key the key * @param defaultVal the default value * @return the value related the key or the default value if the key not existed @@ -166,7 +171,7 @@ public class PropertyUtils { try { return Long.parseLong(val); } catch (NumberFormatException e) { - logger.info(e.getMessage(),e); + logger.info(e.getMessage(), e); } return defaultVal; @@ -174,16 +179,17 @@ public class PropertyUtils { /** * get double value + * * @param key the key * @return if the value not existed, return -1.0, or will return the related value */ public static double getDouble(String key) { - String val = getString(key); - return getDouble(key,-1.0); + return getDouble(key, -1.0); } /** * get double value + * * @param key the key * @param defaultVal the default value * @return the value related the key or the default value if the key not existed @@ -198,17 +204,17 @@ public class PropertyUtils { try { return Double.parseDouble(val); } catch (NumberFormatException e) { - logger.info(e.getMessage(),e); + logger.info(e.getMessage(), e); } return defaultVal; } - /** - * get array - * @param key property name - * @param splitStr separator + * get array + * + * @param key property name + * @param splitStr separator * @return the result array */ public static String[] getArray(String key, String splitStr) { @@ -219,21 +225,22 @@ public class PropertyUtils { try { return value.split(splitStr); } catch (PatternSyntaxException e) { - logger.info(e.getMessage(),e); + logger.info(e.getMessage(), e); } return null; } /** * get enum + * * @param key the key * @param type the class type * @param defaultValue the default value * @param the generic class type - * @return get enum value + * @return get enum value */ public static > T getEnum(String key, Class type, - T defaultValue) { + T defaultValue) { String val = getString(key); if (val == null) { return defaultValue; @@ -242,7 +249,7 @@ public class PropertyUtils { try { return Enum.valueOf(type, val); } catch (IllegalArgumentException e) { - logger.info(e.getMessage(),e); + logger.info(e.getMessage(), e); } return defaultValue; diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java index 049881c087..ac62c17ea2 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/DingTalkUtilsTest.java @@ -32,8 +32,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import static org.junit.Assert.*; - @PrepareForTest(PropertyUtils.class) @RunWith(PowerMockRunner.class) @PowerMockIgnore("javax.net.ssl.*") @@ -97,8 +95,8 @@ public class DingTalkUtilsTest { @Test public void testProxyConfig() { RequestConfig rc = DingTalkUtils.getProxyConfig(); - Assert.assertEquals(rc.getProxy().getPort(), 80); - Assert.assertEquals(rc.getProxy().getHostName(), "proxy.com.cn"); + Assert.assertEquals(80, rc.getProxy().getPort()); + Assert.assertEquals("proxy.com.cn", rc.getProxy().getHostName()); } @Test @@ -114,7 +112,7 @@ public class DingTalkUtilsTest { String msg = DingTalkUtils.textToJsonString("this is test:中文"); logger.info("test support utf8, actual:" + msg); - logger.info("test support utf8, actual:" + DingTalkUtils.isEnableDingTalk); + logger.info("test support utf8, actual:" + DingTalkUtils.IS_ENABLE_DING_TALK); String expect = "{\"text\":{\"content\":\"this is test:中文\"},\"msgtype\":\"text\"}"; Assert.assertEquals(expect, msg); } diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/ExcelUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/ExcelUtilsTest.java index 8ee62358dd..c4833252b2 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/ExcelUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/ExcelUtilsTest.java @@ -17,6 +17,10 @@ package org.apache.dolphinscheduler.alert.utils; +import static org.junit.Assert.assertTrue; + +import java.io.File; + import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -25,8 +29,6 @@ import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import static org.junit.Assert.assertTrue; public class ExcelUtilsTest { @@ -61,7 +63,7 @@ public class ExcelUtilsTest { //Define dest file path String xlsFilePath = rootPath + System.getProperty("file.separator"); - logger.info("xlsFilePath: "+xlsFilePath); + logger.info("XLS_FILE_PATH: " + xlsFilePath); //Define correctContent String correctContent = "[{\"name\":\"ds name\",\"value\":\"ds value\"}]"; @@ -99,4 +101,4 @@ public class ExcelUtilsTest { File file = new File("/tmp/xls" + Constants.SINGLE_SLASH + "t" + Constants.EXCEL_SUFFIX_XLS); file.delete(); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/PropertyUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/PropertyUtilsTest.java index 2a300c9d57..4e239b5c39 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/PropertyUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/PropertyUtilsTest.java @@ -41,11 +41,11 @@ public class PropertyUtilsTest { //Expected "EMAIL" String result = PropertyUtils.getString("alert.type"); logger.info(result); - assertEquals(result, "EMAIL"); + assertEquals("EMAIL", result); //Expected "xxx.xxx.test" result = PropertyUtils.getString("mail.server.host"); - assertEquals(result, "xxx.xxx.test"); + assertEquals("xxx.xxx.test", result); //If key is undefine in alert.properties, then return null result = PropertyUtils.getString("abc"); @@ -88,23 +88,23 @@ public class PropertyUtilsTest { //Expected 25 long result = PropertyUtils.getLong("mail.server.port"); - assertSame(result, 25L); + assertSame(25L, result); //If key is null, then return -1 result = PropertyUtils.getLong(null); - assertSame(result, -1L); + assertSame(-1L, result); //If key is undefine in alert.properties, then return -1 result = PropertyUtils.getLong("abc"); - assertSame(result, -1L); + assertSame(-1L, result); //If key is undefine in alert.properties, and there is a defaultval, then return defaultval result = PropertyUtils.getLong("abc", 200); - assertEquals(result, 200L); + assertEquals(200L, result); //If the value can not parse to long ,it will log the error and return -1L result = PropertyUtils.getLong("test.server.testnumber"); - assertSame(result, -1L); + assertSame(-1L, result); } /** @@ -115,23 +115,23 @@ public class PropertyUtilsTest { //Expected 3.0 double result = PropertyUtils.getDouble("test.server.factor"); - assertEquals(result, 3.0, 0); + assertEquals(3.0, result, 0); //If key is null, then return -1.0 result = PropertyUtils.getDouble(null); - assertEquals(result, -1.0, 0); + assertEquals(-1.0, result, 0); //If key is undefine in alert.properties, then return -1 result = PropertyUtils.getDouble("abc"); - assertEquals(result, -1.0, 0); + assertEquals(-1.0, result, 0); //If key is undefine in alert.properties, and there is a defaultval, then return defaultval result = PropertyUtils.getDouble("abc", 5.0); - assertEquals(result, 5.0, 0); + assertEquals(5.0, result, 0); //If the value can not parse to double ,it will log the error and return -1.0 result = PropertyUtils.getDouble("test.server.testnumber"); - assertEquals(result, -1.0, 0); + assertEquals(-1.0, result, 0); } /** @@ -145,9 +145,9 @@ public class PropertyUtilsTest { assertEquals(result.length, 3); //Equal array values - assertEquals(result[0], "xxx.xxx.test1"); - assertEquals(result[1], "xxx.xxx.test2"); - assertEquals(result[2], "xxx.xxx.test3"); + assertEquals("xxx.xxx.test1", result[0]); + assertEquals("xxx.xxx.test2", result[1]); + assertEquals("xxx.xxx.test3", result[2]); //If key is null, then return -1 result = PropertyUtils.getArray(null, ","); @@ -170,23 +170,23 @@ public class PropertyUtilsTest { //Expected 25 int result = PropertyUtils.getInt("mail.server.port"); - assertSame(result, 25); + assertSame(25, result); //If key is null, then return -1 result = PropertyUtils.getInt(null); - assertSame(result, -1); + assertSame(-1, result); //If key is undefine in alert.properties, then return -1 result = PropertyUtils.getInt("abc"); - assertSame(result, -1); + assertSame(-1, result); //If key is undefine in alert.properties, and there is a defaultval, then return defaultval result = PropertyUtils.getInt("abc", 300); - assertEquals(result, 300); + assertEquals(300, result); //If the value can not parse to int ,it will log the error and return -1 result = PropertyUtils.getInt("test.server.testnumber"); - assertSame(result, -1); + assertSame(-1, result); } /** @@ -197,19 +197,19 @@ public class PropertyUtilsTest { //Expected MASTER ZKNodeType zkNodeType = PropertyUtils.getEnum("test.server.enum1", ZKNodeType.class,ZKNodeType.WORKER); - assertEquals(zkNodeType, ZKNodeType.MASTER); + assertEquals(ZKNodeType.MASTER, zkNodeType); //Expected DEAD_SERVER zkNodeType = PropertyUtils.getEnum("test.server.enum2", ZKNodeType.class,ZKNodeType.WORKER); - assertEquals(zkNodeType, ZKNodeType.DEAD_SERVER); + assertEquals(ZKNodeType.DEAD_SERVER, zkNodeType); //If key is null, then return defaultval zkNodeType = PropertyUtils.getEnum(null, ZKNodeType.class,ZKNodeType.WORKER); - assertEquals(zkNodeType, ZKNodeType.WORKER); + assertEquals(ZKNodeType.WORKER, zkNodeType); //If the value doesn't define in enum ,it will log the error and return -1 zkNodeType = PropertyUtils.getEnum("test.server.enum3", ZKNodeType.class,ZKNodeType.WORKER); - assertEquals(zkNodeType, ZKNodeType.WORKER); + assertEquals(ZKNodeType.WORKER, zkNodeType); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java index 4bdaa365ee..a67ade2ba7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java @@ -14,12 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.api.controller; -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiImplicitParam; -import io.swagger.annotations.ApiImplicitParams; -import io.swagger.annotations.ApiOperation; +import static org.apache.dolphinscheduler.api.enums.Status.AUTHORIZED_DATA_SOURCE; +import static org.apache.dolphinscheduler.api.enums.Status.CONNECTION_TEST_FAILURE; +import static org.apache.dolphinscheduler.api.enums.Status.CONNECT_DATASOURCE_FAILURE; +import static org.apache.dolphinscheduler.api.enums.Status.CREATE_DATASOURCE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.DELETE_DATA_SOURCE_FAILURE; +import static org.apache.dolphinscheduler.api.enums.Status.KERBEROS_STARTUP_STATE; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_DATASOURCE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.UNAUTHORIZED_DATASOURCE; +import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_DATASOURCE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_DATASOURCE_NAME_FAILURE; + import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.DataSourceService; @@ -30,16 +38,26 @@ import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; + +import java.util.Map; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; -import org.springframework.web.bind.annotation.*; -import springfox.documentation.annotations.ApiIgnore; - -import java.util.Map; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestAttribute; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; -import static org.apache.dolphinscheduler.api.enums.Status.*; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; +import springfox.documentation.annotations.ApiIgnore; /** * data source controller @@ -101,8 +119,7 @@ public class DataSourceController extends BaseController { logger.info("login user {} create datasource name: {}, note: {}, type: {}, host: {}, port: {}, database : {}, principal: {}, userName : {}, connectType: {}, other: {}", loginUser.getUserName(), name, note, type, host, port, database, principal, userName, connectType, other); String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other); - Map result = dataSourceService.createDataSource(loginUser, name, note, type, parameter); - return returnDataList(result); + return dataSourceService.createDataSource(loginUser, name, note, type, parameter); } @@ -156,8 +173,7 @@ public class DataSourceController extends BaseController { logger.info("login user {} updateProcessInstance datasource name: {}, note: {}, type: {}, connectType: {}, other: {}", loginUser.getUserName(), name, note, type, connectType, other); String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other); - Map dataSource = dataSourceService.updateDataSource(id, loginUser, name, note, type, parameter); - return returnDataList(dataSource); + return dataSourceService.updateDataSource(id, loginUser, name, note, type, parameter); } /** @@ -281,15 +297,7 @@ public class DataSourceController extends BaseController { logger.info("login user {}, connect datasource: {}, note: {}, type: {}, connectType: {}, other: {}", loginUser.getUserName(), name, note, type, connectType, other); String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other); - Boolean isConnection = dataSourceService.checkConnection(type, parameter); - Result result = new Result(); - - if (isConnection) { - putMsg(result, SUCCESS); - } else { - putMsg(result, CONNECT_DATASOURCE_FAILURE); - } - return result; + return dataSourceService.checkConnection(type, parameter); } /** @@ -309,16 +317,7 @@ public class DataSourceController extends BaseController { public Result connectionTest(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("id") int id) { logger.info("connection test, login user:{}, id:{}", loginUser.getUserName(), id); - - Boolean isConnection = dataSourceService.connectionTest(id); - Result result = new Result(); - - if (isConnection) { - putMsg(result, SUCCESS); - } else { - putMsg(result, CONNECTION_TEST_FAILURE); - } - return result; + return dataSourceService.connectionTest(id); } /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java index 52d246dcb9..c56f7d0b95 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java @@ -198,7 +198,7 @@ public enum Status { CHECK_OS_TENANT_CODE_ERROR(10164, "Please enter the English os tenant code", "请输入英文操作系统租户"), FORCE_TASK_SUCCESS_ERROR(10165, "force task success error", "强制成功任务实例错误"), TASK_INSTANCE_STATE_OPERATION_ERROR(10166, "the status of task instance {0} is {1},Cannot perform force success operation", "任务实例[{0}]的状态是[{1}]，无法执行强制成功操作"), - + DATASOURCE_TYPE_NOT_EXIST(10167, "data source type not exist", "数据源类型不存在"), UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found", "UDF函数不存在"), UDF_FUNCTION_EXISTS(20002, "UDF function already exists", "UDF函数已存在"), diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java index 1fa0494d13..58bb657c6f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java @@ -14,12 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.api.service; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; @@ -27,27 +24,36 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DbConnectType; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.utils.CommonUtils; -import org.apache.dolphinscheduler.common.utils.*; -import org.apache.dolphinscheduler.dao.datasource.*; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; +import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; +import org.apache.dolphinscheduler.dao.datasource.OracleDataSource; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.UserGroupInformation; + +import java.sql.Connection; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.*; - -import static org.apache.dolphinscheduler.common.utils.PropertyUtils.getString; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.fasterxml.jackson.databind.node.ObjectNode; /** * datasource service @@ -67,11 +73,9 @@ public class DataSourceService extends BaseService { public static final String USER_NAME = "userName"; public static final String OTHER = "other"; - @Autowired private DataSourceMapper dataSourceMapper; - @Autowired private DataSourceUserMapper datasourceUserMapper; @@ -85,24 +89,16 @@ public class DataSourceService extends BaseService { * @param parameter datasource parameters * @return create result code */ - public Map createDataSource(User loginUser, String name, String desc, DbType type, String parameter) { + public Result createDataSource(User loginUser, String name, String desc, DbType type, String parameter) { - Map result = new HashMap<>(); + Result result = new Result<>(); // check name can use or not if (checkName(name)) { putMsg(result, Status.DATASOURCE_EXIST); return result; } - Boolean isConnection = checkConnection(type, parameter); - if (!isConnection) { - logger.info("connect failed, type:{}, parameter:{}", type, parameter); - putMsg(result, Status.DATASOURCE_CONNECT_FAILED); - return result; - } - - BaseDataSource datasource = DataSourceFactory.getDatasource(type, parameter); - if (datasource == null) { - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, parameter); + Result isConnection = checkConnection(type, parameter); + if (Status.SUCCESS.getCode() != isConnection.getCode()) { return result; } @@ -125,7 +121,6 @@ public class DataSourceService extends BaseService { return result; } - /** * updateProcessInstance datasource * @@ -137,9 +132,9 @@ public class DataSourceService extends BaseService { * @param id data source id * @return update result code */ - public Map updateDataSource(int id, User loginUser, String name, String desc, DbType type, String parameter) { + public Result updateDataSource(int id, User loginUser, String name, String desc, DbType type, String parameter) { - Map result = new HashMap<>(); + Result result = new Result<>(); // determine whether the data source exists DataSource dataSource = dataSourceMapper.selectById(id); if (dataSource == null) { @@ -168,12 +163,11 @@ public class DataSourceService extends BaseService { // connectionParams json String connectionParams = paramObject.toString(); - Boolean isConnection = checkConnection(type, connectionParams); - if (!isConnection) { - logger.info("connect failed, type:{}, parameter:{}", type, parameter); - putMsg(result, Status.DATASOURCE_CONNECT_FAILED); + Result isConnection = checkConnection(type, parameter); + if (Status.SUCCESS.getCode() != isConnection.getCode()) { return result; } + Date now = new Date(); dataSource.setName(name.trim()); @@ -192,7 +186,6 @@ public class DataSourceService extends BaseService { return queryDataSource != null && queryDataSource.size() > 0; } - /** * updateProcessInstance datasource * @@ -360,8 +353,8 @@ public class DataSourceService extends BaseService { * @param name datasource name * @return true if data datasource not exists, otherwise return false */ - public Result verifyDataSourceName(String name) { - Result result = new Result(); + public Result verifyDataSourceName(String name) { + Result result = new Result<>(); List dataSourceList = dataSourceMapper.queryDataSourceByName(name); if (dataSourceList != null && dataSourceList.size() > 0) { logger.error("datasource name:{} has exist, can't create again.", name); @@ -373,77 +366,6 @@ public class DataSourceService extends BaseService { return result; } - /** - * get connection - * - * @param dbType datasource type - * @param parameter parameter - * @return connection for datasource - */ - private Connection getConnection(DbType dbType, String parameter) { - Connection connection = null; - BaseDataSource datasource = null; - try { - switch (dbType) { - case POSTGRESQL: - datasource = JSONUtils.parseObject(parameter, PostgreDataSource.class); - Class.forName(Constants.ORG_POSTGRESQL_DRIVER); - break; - case MYSQL: - datasource = JSONUtils.parseObject(parameter, MySQLDataSource.class); - Class.forName(Constants.COM_MYSQL_JDBC_DRIVER); - break; - case HIVE: - case SPARK: - if (CommonUtils.getKerberosStartupState()) { - System.setProperty(org.apache.dolphinscheduler.common.Constants.JAVA_SECURITY_KRB5_CONF, - getString(org.apache.dolphinscheduler.common.Constants.JAVA_SECURITY_KRB5_CONF_PATH)); - Configuration configuration = new Configuration(); - configuration.set(org.apache.dolphinscheduler.common.Constants.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); - UserGroupInformation.setConfiguration(configuration); - UserGroupInformation.loginUserFromKeytab(getString(org.apache.dolphinscheduler.common.Constants.LOGIN_USER_KEY_TAB_USERNAME), - getString(org.apache.dolphinscheduler.common.Constants.LOGIN_USER_KEY_TAB_PATH)); - } - if (dbType == DbType.HIVE) { - datasource = JSONUtils.parseObject(parameter, HiveDataSource.class); - } else if (dbType == DbType.SPARK) { - datasource = JSONUtils.parseObject(parameter, SparkDataSource.class); - } - Class.forName(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER); - break; - case CLICKHOUSE: - datasource = JSONUtils.parseObject(parameter, ClickHouseDataSource.class); - Class.forName(Constants.COM_CLICKHOUSE_JDBC_DRIVER); - break; - case ORACLE: - datasource = JSONUtils.parseObject(parameter, OracleDataSource.class); - Class.forName(Constants.COM_ORACLE_JDBC_DRIVER); - break; - case SQLSERVER: - datasource = JSONUtils.parseObject(parameter, SQLServerDataSource.class); - Class.forName(Constants.COM_SQLSERVER_JDBC_DRIVER); - break; - case DB2: - datasource = JSONUtils.parseObject(parameter, DB2ServerDataSource.class); - Class.forName(Constants.COM_DB2_JDBC_DRIVER); - break; - case PRESTO: - datasource = JSONUtils.parseObject(parameter, PrestoDataSource.class); - Class.forName(Constants.COM_PRESTO_JDBC_DRIVER); - break; - default: - break; - } - - if (datasource != null) { - connection = DriverManager.getConnection(datasource.getJdbcUrl(), datasource.getUser(), datasource.getPassword()); - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - return connection; - } - /** * check connection * @@ -451,18 +373,24 @@ public class DataSourceService extends BaseService { * @param parameter data source parameters * @return true if connect successfully, otherwise false */ - public boolean checkConnection(DbType type, String parameter) { - Boolean isConnection = false; - Connection con = getConnection(type, parameter); - if (con != null) { - isConnection = true; - try { - con.close(); - } catch (SQLException e) { - logger.error("close connection fail at DataSourceService::checkConnection()", e); + public Result checkConnection(DbType type, String parameter) { + Result result = new Result<>(); + BaseDataSource datasource = DataSourceFactory.getDatasource(type, parameter); + if (datasource == null) { + putMsg(result, Status.DATASOURCE_TYPE_NOT_EXIST, type); + return result; + } + try (Connection connection = datasource.getConnection()) { + if (connection == null) { + putMsg(result, Status.CONNECTION_TEST_FAILURE); + return result; } + putMsg(result, Status.SUCCESS); + return result; + } catch (Exception e) { + logger.error("datasource test connection error, dbType:{}, jdbcUrl:{}, message:{}.", type, datasource.getJdbcUrl(), e.getMessage()); + return new Result<>(Status.CONNECTION_TEST_FAILURE.getCode(),e.getMessage()); } - return isConnection; } /** @@ -471,13 +399,14 @@ public class DataSourceService extends BaseService { * @param id datasource id * @return connect result code */ - public boolean connectionTest(int id) { + public Result connectionTest(int id) { DataSource dataSource = dataSourceMapper.selectById(id); - if (dataSource != null) { - return checkConnection(dataSource.getType(), dataSource.getConnectionParams()); - } else { - return false; + if (dataSource == null) { + Result result = new Result<>(); + putMsg(result, Status.RESOURCE_NOT_EXIST); + return result; } + return checkConnection(dataSource.getType(), dataSource.getConnectionParams()); } /** @@ -510,8 +439,8 @@ public class DataSourceService extends BaseService { parameterMap.put(Constants.ORACLE_DB_CONNECT_TYPE, connectType); } - if (CommonUtils.getKerberosStartupState() && - (type == DbType.HIVE || type == DbType.SPARK)) { + if (CommonUtils.getKerberosStartupState() + && (type == DbType.HIVE || type == DbType.SPARK)) { jdbcUrl += ";principal=" + principal; } @@ -535,8 +464,8 @@ public class DataSourceService extends BaseService { parameterMap.put(Constants.JDBC_URL, jdbcUrl); parameterMap.put(Constants.USER, userName); parameterMap.put(Constants.PASSWORD, CommonUtils.encodePassword(password)); - if (CommonUtils.getKerberosStartupState() && - (type == DbType.HIVE || type == DbType.SPARK)) { + if (CommonUtils.getKerberosStartupState() + && (type == DbType.HIVE || type == DbType.SPARK)) { parameterMap.put(Constants.PRINCIPAL, principal); } @@ -557,7 +486,6 @@ public class DataSourceService extends BaseService { } return JSONUtils.toJsonString(parameterMap); - } private String buildAddress(DbType type, String host, String port, DbConnectType connectType) { @@ -609,8 +537,8 @@ public class DataSourceService extends BaseService { * @return delete result code */ @Transactional(rollbackFor = RuntimeException.class) - public Result delete(User loginUser, int datasourceId) { - Result result = new Result(); + public Result delete(User loginUser, int datasourceId) { + Result result = new Result<>(); try { //query datasource by id DataSource dataSource = dataSourceMapper.selectById(datasourceId); @@ -673,7 +601,6 @@ public class DataSourceService extends BaseService { return result; } - /** * authorized datasource * @@ -695,7 +622,6 @@ public class DataSourceService extends BaseService { return result; } - /** * get host and port by address * diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java index 02086a8259..95083dd51b 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java @@ -72,6 +72,8 @@ public class BaseServiceTest { } + + @Test public void testPutMsg(){ diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java index 789e5f6cc5..84ccd2e0a3 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.api.service; import org.apache.dolphinscheduler.api.enums.Status; @@ -24,12 +25,19 @@ import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; import org.apache.dolphinscheduler.dao.datasource.MySQLDataSource; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper; + +import java.sql.Connection; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -38,16 +46,15 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - @RunWith(PowerMockRunner.class) @PowerMockIgnore({"sun.security.*", "javax.net.*"}) +@PrepareForTest({DataSourceFactory.class}) public class DataSourceServiceTest { + @InjectMocks private DataSourceService dataSourceService; @Mock @@ -69,28 +76,31 @@ public class DataSourceServiceTest { dataSource.setName(dataSourceName); dataSourceList.add(dataSource); PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(dataSourceList); - Map dataSourceExitsResult = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); - Assert.assertEquals(Status.DATASOURCE_EXIST, dataSourceExitsResult.get(Constants.STATUS)); + Result dataSourceExitsResult = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + Assert.assertEquals(Status.DATASOURCE_EXIST.getCode(), dataSourceExitsResult.getCode().intValue()); // data source exits PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null); - PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(false); - Map connectFailedResult = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); - Assert.assertEquals(Status.DATASOURCE_CONNECT_FAILED, connectFailedResult.get(Constants.STATUS)); + Result connectionResult = new Result(Status.DATASOURCE_CONNECT_FAILED.getCode(),Status.DATASOURCE_CONNECT_FAILED.getMsg()); + //PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult); + PowerMockito.doReturn(connectionResult).when(dataSourceService).checkConnection(dataSourceType, parameter); + Result connectFailedResult = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + Assert.assertEquals(Status.DATASOURCE_CONNECT_FAILED.getCode(), connectFailedResult.getCode().intValue()); // data source exits PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null); - PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(true); + connectionResult = new Result(Status.SUCCESS.getCode(),Status.SUCCESS.getMsg()); + PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult); PowerMockito.when(DataSourceFactory.getDatasource(dataSourceType, parameter)).thenReturn(null); - Map notValidError = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); - Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, notValidError.get(Constants.STATUS)); + Result notValidError = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR.getCode(), notValidError.getCode().intValue()); // success PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null); - PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(true); + PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult); PowerMockito.when(DataSourceFactory.getDatasource(dataSourceType, parameter)).thenReturn(JSONUtils.parseObject(parameter, MySQLDataSource.class)); - Map success = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); - Assert.assertEquals(Status.SUCCESS, success.get(Constants.STATUS)); + Result success = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + Assert.assertEquals(Status.SUCCESS.getCode(), success.getCode().intValue()); } public void updateDataSourceTest() { @@ -104,14 +114,14 @@ public class DataSourceServiceTest { // data source not exits PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(null); - Map resourceNotExits = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); - Assert.assertEquals(Status.RESOURCE_NOT_EXIST, resourceNotExits.get(Constants.STATUS)); + Result resourceNotExits = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getCode(), resourceNotExits.getCode().intValue()); // user no operation perm DataSource dataSource = new DataSource(); dataSource.setUserId(0); PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource); - Map userNoOperationPerm = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); - Assert.assertEquals(Status.USER_NO_OPERATION_PERM, userNoOperationPerm.get(Constants.STATUS)); + Result userNoOperationPerm = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM.getCode(), userNoOperationPerm.getCode().intValue()); // data source name exits dataSource.setUserId(-1); @@ -119,22 +129,24 @@ public class DataSourceServiceTest { dataSourceList.add(dataSource); PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource); PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(dataSourceList); - Map dataSourceNameExist = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); - Assert.assertEquals(Status.DATASOURCE_EXIST, dataSourceNameExist.get(Constants.STATUS)); + Result dataSourceNameExist = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + Assert.assertEquals(Status.DATASOURCE_EXIST.getCode(), dataSourceNameExist.getCode().intValue()); // data source connect failed PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource); PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(null); - PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(true); - Map connectFailed = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); - Assert.assertEquals(Status.DATASOURCE_CONNECT_FAILED, connectFailed.get(Constants.STATUS)); + Result connectionResult = new Result(Status.SUCCESS.getCode(),Status.SUCCESS.getMsg()); + PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult); + Result connectFailed = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + Assert.assertEquals(Status.DATASOURCE_CONNECT_FAILED.getCode(), connectFailed.getCode().intValue()); //success PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource); PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(null); - PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(false); - Map success = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); - Assert.assertEquals(Status.SUCCESS, connectFailed.get(Constants.STATUS)); + connectionResult = new Result(Status.DATASOURCE_CONNECT_FAILED.getCode(),Status.DATASOURCE_CONNECT_FAILED.getMsg()); + PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult); + Result success = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + Assert.assertEquals(Status.SUCCESS.getCode(), success.getCode().intValue()); } @@ -152,7 +164,8 @@ public class DataSourceServiceTest { public void connectionTest() { int dataSourceId = -1; PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(null); - Assert.assertFalse(dataSourceService.connectionTest(dataSourceId)); + Result result = dataSourceService.connectionTest(dataSourceId); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getCode(),result.getCode().intValue()); } @Test @@ -252,7 +265,8 @@ public class DataSourceServiceTest { dataSource.setName("test"); dataSource.setNote("Note"); dataSource.setType(DbType.ORACLE); - dataSource.setConnectionParams("{\"connectType\":\"ORACLE_SID\",\"address\":\"jdbc:oracle:thin:@192.168.xx.xx:49161\",\"database\":\"XE\",\"jdbcUrl\":\"jdbc:oracle:thin:@192.168.xx.xx:49161/XE\",\"user\":\"system\",\"password\":\"oracle\"}"); + dataSource.setConnectionParams("{\"connectType\":\"ORACLE_SID\",\"address\":\"jdbc:oracle:thin:@192.168.xx.xx:49161\",\"database\":\"XE\"," + + "\"jdbcUrl\":\"jdbc:oracle:thin:@192.168.xx.xx:49161/XE\",\"user\":\"system\",\"password\":\"oracle\"}"); return dataSource; } @@ -261,7 +275,8 @@ public class DataSourceServiceTest { public void buildParameter() { String param = dataSourceService.buildParameter(DbType.ORACLE, "192.168.9.1", "1521", "im" , "", "test", "test", DbConnectType.ORACLE_SERVICE_NAME, ""); - String expected = "{\"connectType\":\"ORACLE_SERVICE_NAME\",\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}"; + String expected = "{\"connectType\":\"ORACLE_SERVICE_NAME\",\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\"," + + "\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}"; Assert.assertEquals(expected, param); } @@ -270,10 +285,10 @@ public class DataSourceServiceTest { PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "true"); String param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im" , "", "test", "123456", null, ""); - String expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\"}"; + String expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\"," + + "\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\"}"; Assert.assertEquals(expected, param); - PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "false"); param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im" , "", "test", "123456", null, ""); @@ -294,4 +309,31 @@ public class DataSourceServiceTest { return loginUser; } -} \ No newline at end of file + /** + * test check connection + * @throws Exception + */ + @Test + public void testCheckConnection() throws Exception { + DbType dataSourceType = DbType.POSTGRESQL; + String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null); + + PowerMockito.mockStatic(DataSourceFactory.class); + PowerMockito.when(DataSourceFactory.getDatasource(Mockito.any(), Mockito.anyString())).thenReturn(null); + Result result = dataSourceService.checkConnection(dataSourceType, parameter); + Assert.assertEquals(Status.DATASOURCE_TYPE_NOT_EXIST.getCode(), result.getCode().intValue()); + + BaseDataSource dataSource = PowerMockito.mock(BaseDataSource.class); + PowerMockito.when(DataSourceFactory.getDatasource(Mockito.any(), Mockito.anyString())).thenReturn(dataSource); + PowerMockito.when(dataSource.getConnection()).thenReturn(null); + result = dataSourceService.checkConnection(dataSourceType, parameter); + Assert.assertEquals(Status.CONNECTION_TEST_FAILURE.getCode(), result.getCode().intValue()); + + Connection connection = PowerMockito.mock(Connection.class); + PowerMockito.when(dataSource.getConnection()).thenReturn(connection); + result = dataSourceService.checkConnection(dataSourceType, parameter); + Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); + + } + +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java index 256f19905c..6e32d12df3 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java @@ -33,11 +33,17 @@ public class StringUtils { return !isEmpty(cs); } - public static boolean isBlank(String s) { - if (isEmpty(s)) { - return true; + public static boolean isBlank(String str) { + int strLen; + if (str != null && (strLen = str.length()) != 0) { + for (int i = 0; i < strLen; ++i) { + if (!Character.isWhitespace(str.charAt(i))) { + return false; + } + } } - return s.trim().length() == 0; + return true; + } public static boolean isNotBlank(String s) { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java index 729a17f27b..bb3825fbf9 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java @@ -14,14 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.dao.datasource; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; + +import java.sql.Connection; +import java.sql.DriverManager; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,194 +32,183 @@ import org.slf4j.LoggerFactory; */ public abstract class BaseDataSource { - private static final Logger logger = LoggerFactory.getLogger(BaseDataSource.class); - - /** - * user name - */ - protected String user; - - /** - * user password - */ - protected String password; - - /** - * data source address - */ - private String address; - - /** - * database name - */ - private String database; - - /** - * other connection parameters for the data source - */ - private String other; - - /** - * principal - */ - private String principal; - - public String getPrincipal() { - return principal; - } - - public void setPrincipal(String principal) { - this.principal = principal; - } - - /** - * @return driver class - */ - public abstract String driverClassSelector(); - - /** - * @return db type - */ - public abstract DbType dbTypeSelector(); - - /** - * gets the JDBC url for the data source connection - * @return getJdbcUrl - */ - public String getJdbcUrl() { - StringBuilder jdbcUrl = new StringBuilder(getAddress()); - - appendDatabase(jdbcUrl); - appendPrincipal(jdbcUrl); - appendOther(jdbcUrl); - - return jdbcUrl.toString(); - } - - /** - * append database - * @param jdbcUrl jdbc url - */ - protected void appendDatabase(StringBuilder jdbcUrl) { - if (dbTypeSelector() == DbType.SQLSERVER) { - jdbcUrl.append(";databaseName=").append(getDatabase()); - } else { - if (getAddress().lastIndexOf('/') != (jdbcUrl.length() - 1)) { - jdbcUrl.append("/"); - } - jdbcUrl.append(getDatabase()); - } - } - - /** - * append principal - * @param jdbcUrl jdbc url - */ - private void appendPrincipal(StringBuilder jdbcUrl) { - boolean tag = dbTypeSelector() == DbType.HIVE || dbTypeSelector() == DbType.SPARK; - if (tag && StringUtils.isNotEmpty(getPrincipal())) { - jdbcUrl.append(";principal=").append(getPrincipal()); - } - } - - /** - * append other - * @param jdbcUrl jdbc url - */ - private void appendOther(StringBuilder jdbcUrl) { - String otherParams = filterOther(getOther()); - if (StringUtils.isNotEmpty(otherParams)) { - String separator = ""; - switch (dbTypeSelector()) { - case CLICKHOUSE: - case MYSQL: - case ORACLE: - case POSTGRESQL: - case PRESTO: - separator = "?"; - break; - case DB2: - separator = ":"; - break; - case HIVE: - case SPARK: - case SQLSERVER: - separator = ";"; - break; - default: - logger.error("Db type mismatch!"); - } - jdbcUrl.append(separator).append(otherParams); - } - } - - protected String filterOther(String otherParams){ - return otherParams; - } - - /** - * test whether the data source can be connected successfully - */ - public void isConnectable() { - Connection con = null; - try { - Class.forName(driverClassSelector()); - con = DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword()); - } catch (ClassNotFoundException | SQLException e) { - logger.error("Get connection error: {}", e.getMessage()); - } finally { - if (con != null) { - try { - con.close(); - } catch (SQLException e) { - logger.error(e.getMessage(), e); + private static final Logger logger = LoggerFactory.getLogger(BaseDataSource.class); + + /** + * user name + */ + protected String user; + + /** + * user password + */ + protected String password; + + /** + * data source address + */ + private String address; + + /** + * database name + */ + private String database; + + /** + * other connection parameters for the data source + */ + private String other; + + /** + * principal + */ + private String principal; + + public String getPrincipal() { + return principal; + } + + public void setPrincipal(String principal) { + this.principal = principal; + } + + /** + * @return driver class + */ + public abstract String driverClassSelector(); + + /** + * @return db type + */ + public abstract DbType dbTypeSelector(); + + /** + * gets the JDBC url for the data source connection + * @return getJdbcUrl + */ + public String getJdbcUrl() { + StringBuilder jdbcUrl = new StringBuilder(getAddress()); + + appendDatabase(jdbcUrl); + appendPrincipal(jdbcUrl); + appendOther(jdbcUrl); + + return jdbcUrl.toString(); + } + + /** + * append database + * @param jdbcUrl jdbc url + */ + protected void appendDatabase(StringBuilder jdbcUrl) { + if (dbTypeSelector() == DbType.SQLSERVER) { + jdbcUrl.append(";databaseName=").append(getDatabase()); + } else { + if (getAddress().lastIndexOf('/') != (jdbcUrl.length() - 1)) { + jdbcUrl.append("/"); + } + jdbcUrl.append(getDatabase()); + } + } + + /** + * append principal + * @param jdbcUrl jdbc url + */ + private void appendPrincipal(StringBuilder jdbcUrl) { + boolean tag = dbTypeSelector() == DbType.HIVE || dbTypeSelector() == DbType.SPARK; + if (tag && StringUtils.isNotEmpty(getPrincipal())) { + jdbcUrl.append(";principal=").append(getPrincipal()); } - } } - } - public String getUser() { - return user; - } + /** + * append other + * @param jdbcUrl jdbc url + */ + private void appendOther(StringBuilder jdbcUrl) { + String otherParams = filterOther(getOther()); + if (StringUtils.isNotEmpty(otherParams)) { + String separator = ""; + switch (dbTypeSelector()) { + case CLICKHOUSE: + case MYSQL: + case ORACLE: + case POSTGRESQL: + case PRESTO: + separator = "?"; + break; + case DB2: + separator = ":"; + break; + case HIVE: + case SPARK: + case SQLSERVER: + separator = ";"; + break; + default: + logger.error("Db type mismatch!"); + } + jdbcUrl.append(separator).append(otherParams); + } + } + + /** + * the data source test connection + * @return Connection Connection + * @throws Exception Exception + */ + public Connection getConnection() throws Exception { + Class.forName(driverClassSelector()); + return DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword()); + } + + protected String filterOther(String otherParams) { + return otherParams; + } + + public String getUser() { + return user; + } - public void setUser(String user) { - this.user = user; - } + public void setUser(String user) { + this.user = user; + } - /** - * password need decode - * @return - */ - public String getPassword() { - return CommonUtils.decodePassword(password); - } + /** + * password need decode + * @return + */ + public String getPassword() { + return CommonUtils.decodePassword(password); + } - public void setPassword(String password) { - this.password = password; - } + public void setPassword(String password) { + this.password = password; + } - public void setAddress(String address) { - this.address = address; - } + public void setAddress(String address) { + this.address = address; + } - public String getAddress() { - return address; - } + public String getAddress() { + return address; + } - public String getDatabase() { - return database; - } + public String getDatabase() { + return database; + } - public void setDatabase(String database) { - this.database = database; - } + public void setDatabase(String database) { + this.database = database; + } - public String getOther() { - return other; - } + public String getOther() { + return other; + } - public void setOther(String other) { - this.other = other; - } + public void setOther(String other) { + this.other = other; + } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java index e64b0395ec..0f9bfc1648 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java @@ -19,78 +19,92 @@ package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; + +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; + +import java.sql.Connection; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; /** * data source of hive */ public class HiveDataSource extends BaseDataSource { - /** - * gets the JDBC url for the data source connection - * @return jdbc url - */ - @Override - public String driverClassSelector() { - return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; - } - - /** - * @return db type - */ - @Override - public DbType dbTypeSelector() { - return DbType.HIVE; - } - - /** - * build hive jdbc params,append : ?hive_conf_list - * - * hive jdbc url template: - * - * jdbc:hive2://:,:/dbName;initFile=;sess_var_list?hive_conf_list#hive_var_list - * - * @param otherParams otherParams - * @return filter otherParams - */ - @Override - protected String filterOther(String otherParams) { - if (StringUtils.isBlank(otherParams)) { - return ""; + /** + * gets the JDBC url for the data source connection + * @return jdbc url + */ + @Override + public String driverClassSelector() { + return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; } - StringBuilder hiveConfListSb = new StringBuilder(); - hiveConfListSb.append("?"); - StringBuilder sessionVarListSb = new StringBuilder(); + /** + * @return db type + */ + @Override + public DbType dbTypeSelector() { + return DbType.HIVE; + } - String[] otherArray = otherParams.split(";", -1); + /** + * build hive jdbc params,append : ?hive_conf_list + * + * hive jdbc url template: + * + * jdbc:hive2://:,:/dbName;initFile=;sess_var_list?hive_conf_list#hive_var_list + * + * @param otherParams otherParams + * @return filter otherParams + */ + @Override + protected String filterOther(String otherParams) { + if (StringUtils.isBlank(otherParams)) { + return ""; + } - // get the default hive conf var name - Set hiveConfSet = Stream.of(ConfVars.values()).map(confVars -> confVars.varname) - .collect(Collectors.toSet()); + StringBuilder hiveConfListSb = new StringBuilder(); + hiveConfListSb.append("?"); + StringBuilder sessionVarListSb = new StringBuilder(); - for (String conf : otherArray) { - if (hiveConfSet.contains(conf.split("=")[0])) { - hiveConfListSb.append(conf).append(";"); - } else { - sessionVarListSb.append(conf).append(";"); - } - } + String[] otherArray = otherParams.split(";", -1); - // remove the last ";" - if (sessionVarListSb.length() > 0) { - sessionVarListSb.deleteCharAt(sessionVarListSb.length() - 1); - } + // get the default hive conf var name + Set hiveConfSet = Stream.of(ConfVars.values()).map(confVars -> confVars.varname) + .collect(Collectors.toSet()); + + for (String conf : otherArray) { + if (hiveConfSet.contains(conf.split("=")[0])) { + hiveConfListSb.append(conf).append(";"); + } else { + sessionVarListSb.append(conf).append(";"); + } + } + + // remove the last ";" + if (sessionVarListSb.length() > 0) { + sessionVarListSb.deleteCharAt(sessionVarListSb.length() - 1); + } - if (hiveConfListSb.length() > 0) { - hiveConfListSb.deleteCharAt(hiveConfListSb.length() - 1); + if (hiveConfListSb.length() > 0) { + hiveConfListSb.deleteCharAt(hiveConfListSb.length() - 1); + } + + return sessionVarListSb.toString() + hiveConfListSb.toString(); } - return sessionVarListSb.toString() + hiveConfListSb.toString(); - } - + /** + * the data source test connection + * @return Connection Connection + * @throws Exception Exception + */ + @Override + public Connection getConnection() throws Exception { + CommonUtils.loadKerberosConf(); + return super.getConnection(); + } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java index e4b8f4bf13..78062357e2 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java @@ -14,25 +14,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; /** * data source of SQL Server */ public class SQLServerDataSource extends BaseDataSource { - private static final Logger logger = LoggerFactory.getLogger(SQLServerDataSource.class); - /** * gets the JDBC url for the data source connection * @return jdbc url @@ -50,39 +43,18 @@ public class SQLServerDataSource extends BaseDataSource { } /** - * test whether the data source can be connected successfully - */ + * @return driver class + */ @Override - public void isConnectable() { - Connection con = null; - try { - Class.forName(Constants.COM_SQLSERVER_JDBC_DRIVER); - con = DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword()); - } catch (Exception e) { - logger.error("error", e); - } finally { - if (con != null) { - try { - con.close(); - } catch (SQLException e) { - logger.error("SQL Server datasource try conn close conn error", e); - } - } - } + public String driverClassSelector() { + return Constants.COM_SQLSERVER_JDBC_DRIVER; } - /** - * @return driver class - */ - @Override - public String driverClassSelector() { - return Constants.COM_SQLSERVER_JDBC_DRIVER; - } - /** - * @return db type - */ - @Override - public DbType dbTypeSelector() { - return DbType.SQLSERVER; - } + /** + * @return db type + */ + @Override + public DbType dbTypeSelector() { + return DbType.SQLSERVER; + } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java index 0329ef8400..207ed43942 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java @@ -14,30 +14,45 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.CommonUtils; + +import java.sql.Connection; /** * data source of spark */ public class SparkDataSource extends BaseDataSource { - /** - * gets the JDBC url for the data source connection - * @return jdbc url - */ - @Override - public String driverClassSelector() { - return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; - } + /** + * gets the JDBC url for the data source connection + * @return jdbc url + */ + @Override + public String driverClassSelector() { + return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; + } + + /** + * @return db type + */ + @Override + public DbType dbTypeSelector() { + return DbType.SPARK; + } - /** - * @return db type - */ - @Override - public DbType dbTypeSelector() { - return DbType.SPARK; - } + /** + * the data source test connection + * @return Connection Connection + * @throws Exception Exception + */ + @Override + public Connection getConnection() throws Exception { + CommonUtils.loadKerberosConf(); + return super.getConnection(); + } } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index aa70ed225b..9501e7f371 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -687,6 +687,12 @@ public class ProcessService { processInstance = generateNewProcessInstance(processDefinition, command, cmdParam); } else { processInstance = this.findProcessInstanceDetailById(processInstanceId); + // Recalculate global parameters after rerun. + processInstance.setGlobalParams(ParameterUtils.curingGlobalParams( + processDefinition.getGlobalParamMap(), + processDefinition.getGlobalParamList(), + getCommandTypeIfComplement(processInstance, command), + processInstance.getScheduleTime())); } processDefinition = processDefineMapper.selectById(processInstance.getProcessDefinitionId()); processInstance.setProcessDefinition(processDefinition); diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue index 74dbf3fc6b..a57f472977 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue @@ -74,6 +74,11 @@
+