diff --git a/.github/workflows/ci_backend.yml b/.github/workflows/ci_backend.yml index 0273251e99..c24a3ecf07 100644 --- a/.github/workflows/ci_backend.yml +++ b/.github/workflows/ci_backend.yml @@ -56,21 +56,9 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.8 + - name: Check license + run: ./mvnw -B apache-rat:check - name: Compile - run: mvn -B clean compile package -Prelease -Dmaven.test.skip=true - License-check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. - - name: checkout submodules - shell: bash - run: | - git submodule sync --recursive - git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - name: Check - run: mvn -B apache-rat:check + run: mvn -B clean compile install -Prelease -Dmaven.test.skip=true + - name: Check dependency license + run: tools/dependencies/check-LICENSE.sh diff --git a/.gitignore b/.gitignore index 6dd99201a9..17b0dc6610 100644 --- a/.gitignore +++ b/.gitignore @@ -4,27 +4,24 @@ .zip .gz .DS_Store -.idea .idea/ -.idea/* -.target -.target/ -**/**/target/** -target/* -*/target -*/target/* +dist/ +all-dependencies.txt +self-modules.txt +third-party-dependencies.txt +**/target/ .settings .nbproject .classpath .project -*.iml +**/*.iml *.ipr *.iws *.tgz .*.swp .vim .tmp -node_modules +**/node_modules npm-debug.log .vscode logs/* @@ -41,110 +38,10 @@ dolphinscheduler-alert/logs/ dolphinscheduler-alert/src/main/resources/alert.properties_bak dolphinscheduler-alert/src/main/resources/logback.xml dolphinscheduler-server/src/main/resources/logback.xml -dolphinscheduler-ui/dist +dolphinscheduler-ui/dist/ dolphinscheduler-ui/node -dolphinscheduler-ui/dist/css/common.16ac5d9.css -dolphinscheduler-ui/dist/css/home/index.b444b91.css -dolphinscheduler-ui/dist/css/login/index.5866c64.css -dolphinscheduler-ui/dist/js/0.ac94e5d.js -dolphinscheduler-ui/dist/js/0.ac94e5d.js.map -dolphinscheduler-ui/dist/js/1.0b043a3.js -dolphinscheduler-ui/dist/js/1.0b043a3.js.map -dolphinscheduler-ui/dist/js/10.1bce3dc.js -dolphinscheduler-ui/dist/js/10.1bce3dc.js.map -dolphinscheduler-ui/dist/js/11.79f04d8.js -dolphinscheduler-ui/dist/js/11.79f04d8.js.map -dolphinscheduler-ui/dist/js/12.420daa5.js -dolphinscheduler-ui/dist/js/12.420daa5.js.map -dolphinscheduler-ui/dist/js/13.e5bae1c.js -dolphinscheduler-ui/dist/js/13.e5bae1c.js.map -dolphinscheduler-ui/dist/js/14.f2a0dca.js -dolphinscheduler-ui/dist/js/14.f2a0dca.js.map -dolphinscheduler-ui/dist/js/15.45373e8.js -dolphinscheduler-ui/dist/js/15.45373e8.js.map -dolphinscheduler-ui/dist/js/16.fecb0fc.js -dolphinscheduler-ui/dist/js/16.fecb0fc.js.map -dolphinscheduler-ui/dist/js/17.84be279.js -dolphinscheduler-ui/dist/js/17.84be279.js.map -dolphinscheduler-ui/dist/js/18.307ea70.js -dolphinscheduler-ui/dist/js/18.307ea70.js.map -dolphinscheduler-ui/dist/js/19.144db9c.js -dolphinscheduler-ui/dist/js/19.144db9c.js.map -dolphinscheduler-ui/dist/js/2.8b4ef29.js -dolphinscheduler-ui/dist/js/2.8b4ef29.js.map -dolphinscheduler-ui/dist/js/20.4c527e9.js -dolphinscheduler-ui/dist/js/20.4c527e9.js.map -dolphinscheduler-ui/dist/js/21.831b2a2.js -dolphinscheduler-ui/dist/js/21.831b2a2.js.map -dolphinscheduler-ui/dist/js/22.2b4bb2a.js -dolphinscheduler-ui/dist/js/22.2b4bb2a.js.map -dolphinscheduler-ui/dist/js/23.81467ef.js -dolphinscheduler-ui/dist/js/23.81467ef.js.map -dolphinscheduler-ui/dist/js/24.54a00e4.js -dolphinscheduler-ui/dist/js/24.54a00e4.js.map -dolphinscheduler-ui/dist/js/25.8d7bd36.js -dolphinscheduler-ui/dist/js/25.8d7bd36.js.map -dolphinscheduler-ui/dist/js/26.2ec5e78.js -dolphinscheduler-ui/dist/js/26.2ec5e78.js.map -dolphinscheduler-ui/dist/js/27.3ab48c2.js -dolphinscheduler-ui/dist/js/27.3ab48c2.js.map -dolphinscheduler-ui/dist/js/28.363088a.js -dolphinscheduler-ui/dist/js/28.363088a.js.map -dolphinscheduler-ui/dist/js/29.6c5853a.js -dolphinscheduler-ui/dist/js/29.6c5853a.js.map -dolphinscheduler-ui/dist/js/3.a0edb5b.js -dolphinscheduler-ui/dist/js/3.a0edb5b.js.map -dolphinscheduler-ui/dist/js/30.940fdd3.js -dolphinscheduler-ui/dist/js/30.940fdd3.js.map -dolphinscheduler-ui/dist/js/31.168a460.js -dolphinscheduler-ui/dist/js/31.168a460.js.map -dolphinscheduler-ui/dist/js/32.8df6594.js -dolphinscheduler-ui/dist/js/32.8df6594.js.map -dolphinscheduler-ui/dist/js/33.4480bbe.js -dolphinscheduler-ui/dist/js/33.4480bbe.js.map -dolphinscheduler-ui/dist/js/34.b407fe1.js -dolphinscheduler-ui/dist/js/34.b407fe1.js.map -dolphinscheduler-ui/dist/js/35.f340b0a.js -dolphinscheduler-ui/dist/js/35.f340b0a.js.map -dolphinscheduler-ui/dist/js/36.8880c2d.js -dolphinscheduler-ui/dist/js/36.8880c2d.js.map -dolphinscheduler-ui/dist/js/37.ea2a25d.js -dolphinscheduler-ui/dist/js/37.ea2a25d.js.map -dolphinscheduler-ui/dist/js/38.98a59ee.js -dolphinscheduler-ui/dist/js/38.98a59ee.js.map -dolphinscheduler-ui/dist/js/39.a5e958a.js -dolphinscheduler-ui/dist/js/39.a5e958a.js.map -dolphinscheduler-ui/dist/js/4.4ca44db.js -dolphinscheduler-ui/dist/js/4.4ca44db.js.map -dolphinscheduler-ui/dist/js/40.e187b1e.js -dolphinscheduler-ui/dist/js/40.e187b1e.js.map -dolphinscheduler-ui/dist/js/41.0e89182.js -dolphinscheduler-ui/dist/js/41.0e89182.js.map -dolphinscheduler-ui/dist/js/42.341047c.js -dolphinscheduler-ui/dist/js/42.341047c.js.map -dolphinscheduler-ui/dist/js/43.27b8228.js -dolphinscheduler-ui/dist/js/43.27b8228.js.map -dolphinscheduler-ui/dist/js/44.e8869bc.js -dolphinscheduler-ui/dist/js/44.e8869bc.js.map -dolphinscheduler-ui/dist/js/45.8d54901.js -dolphinscheduler-ui/dist/js/45.8d54901.js.map -dolphinscheduler-ui/dist/js/5.e1ed7f3.js -dolphinscheduler-ui/dist/js/5.e1ed7f3.js.map -dolphinscheduler-ui/dist/js/6.241ba07.js -dolphinscheduler-ui/dist/js/6.241ba07.js.map -dolphinscheduler-ui/dist/js/7.ab2e297.js -dolphinscheduler-ui/dist/js/7.ab2e297.js.map -dolphinscheduler-ui/dist/js/8.83ff814.js -dolphinscheduler-ui/dist/js/8.83ff814.js.map -dolphinscheduler-ui/dist/js/9.39cb29f.js -dolphinscheduler-ui/dist/js/9.39cb29f.js.map -dolphinscheduler-ui/dist/js/common.733e342.js -dolphinscheduler-ui/dist/js/common.733e342.js.map -dolphinscheduler-ui/dist/js/home/index.78a5d12.js -dolphinscheduler-ui/dist/js/home/index.78a5d12.js.map -dolphinscheduler-ui/dist/js/login/index.291b8e3.js -dolphinscheduler-ui/dist/js/login/index.291b8e3.js.map -dolphinscheduler-ui/dist/lib/external/ -/dolphinscheduler-dao/src/main/resources/dao/data_source.properties +dolphinscheduler-dao/src/main/resources/dao/data_source.properties + +.mvn/wrapper/*.jar !/zookeeper_data/ diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java index 58a37c2f41..347336cada 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java @@ -16,8 +16,11 @@ */ package org.apache.dolphinscheduler.alert; +import org.apache.dolphinscheduler.alert.plugin.EmailAlertPlugin; import org.apache.dolphinscheduler.alert.runner.AlertSender; import org.apache.dolphinscheduler.alert.utils.Constants; +import org.apache.dolphinscheduler.alert.utils.PropertyUtils; +import org.apache.dolphinscheduler.common.plugin.FilePluginManager; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.DaoFactory; @@ -41,34 +44,47 @@ public class AlertServer { private static AlertServer instance; - public AlertServer() { + private FilePluginManager alertPluginManager; + + private static final String[] whitePrefixes = new String[]{"org.apache.dolphinscheduler.plugin.utils."}; + private static final String[] excludePrefixes = new String[]{ + "org.apache.dolphinscheduler.plugin.", + "ch.qos.logback.", + "org.slf4j." + }; + + public AlertServer() { + alertPluginManager = + new FilePluginManager(PropertyUtils.getString(Constants.PLUGIN_DIR), whitePrefixes, excludePrefixes); + // add default alert plugins + alertPluginManager.addPlugin(new EmailAlertPlugin()); } - public synchronized static AlertServer getInstance(){ + public synchronized static AlertServer getInstance() { if (null == instance) { instance = new AlertServer(); } return instance; } - public void start(){ + public void start() { logger.info("alert server ready start "); - while (Stopper.isRunning()){ + while (Stopper.isRunning()) { try { Thread.sleep(Constants.ALERT_SCAN_INTERVAL); } catch (InterruptedException e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); Thread.currentThread().interrupt(); } List alerts = alertDao.listWaitExecutionAlert(); - alertSender = new AlertSender(alerts, alertDao); + alertSender = new AlertSender(alerts, alertDao, alertPluginManager); alertSender.run(); } } - public static void main(String[] args){ + public static void main(String[] args) { AlertServer alertServer = AlertServer.getInstance(); alertServer.start(); } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EmailManager.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EmailManager.java index 047ee8bfed..96feb7f09e 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EmailManager.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EmailManager.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.alert.manager; import org.apache.dolphinscheduler.alert.utils.MailUtils; -import org.apache.dolphinscheduler.common.enums.ShowType; import java.util.List; import java.util.Map; @@ -35,7 +34,7 @@ public class EmailManager { * @param showType the showType * @return the send result */ - public Map send(List receviersList,List receviersCcList,String title,String content,ShowType showType){ + public Map send(List receviersList,List receviersCcList,String title,String content,String showType){ return MailUtils.sendMails(receviersList, receviersCcList, title, content, showType); } @@ -48,7 +47,7 @@ public class EmailManager { * @param showType the showType * @return the send result */ - public Map send(List receviersList,String title,String content,ShowType showType){ + public Map send(List receviersList,String title,String content,String showType){ return MailUtils.sendMails(receviersList,title, content, showType); } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java index bb06be6561..43649d6758 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.alert.manager; import org.apache.dolphinscheduler.alert.utils.Constants; import org.apache.dolphinscheduler.alert.utils.EnterpriseWeChatUtils; -import org.apache.dolphinscheduler.dao.entity.Alert; +import org.apache.dolphinscheduler.plugin.model.AlertInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,18 +35,18 @@ public class EnterpriseWeChatManager { private static final Logger logger = LoggerFactory.getLogger(EnterpriseWeChatManager.class); /** * Enterprise We Chat send - * @param alert the alert + * @param alertInfo the alert info * @param token the token * @return the send result */ - public Map send(Alert alert, String token){ + public Map send(AlertInfo alertInfo, String token){ Map retMap = new HashMap<>(); retMap.put(Constants.STATUS, false); String agentId = EnterpriseWeChatUtils.ENTERPRISE_WE_CHAT_AGENT_ID; String users = EnterpriseWeChatUtils.ENTERPRISE_WE_CHAT_USERS; List userList = Arrays.asList(users.split(",")); - logger.info("send message {}",alert); - String msg = EnterpriseWeChatUtils.makeUserSendMsg(userList, agentId,EnterpriseWeChatUtils.markdownByAlert(alert)); + logger.info("send message {}", alertInfo.getAlertData().getTitle()); + String msg = EnterpriseWeChatUtils.makeUserSendMsg(userList, agentId,EnterpriseWeChatUtils.markdownByAlert(alertInfo.getAlertData())); try { EnterpriseWeChatUtils.sendEnterpriseWeChat(Constants.UTF_8, msg, token); } catch (IOException e) { diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java new file mode 100644 index 0000000000..d20306b153 --- /dev/null +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.alert.plugin; + +import org.apache.dolphinscheduler.alert.manager.EmailManager; +import org.apache.dolphinscheduler.alert.manager.EnterpriseWeChatManager; +import org.apache.dolphinscheduler.alert.utils.Constants; +import org.apache.dolphinscheduler.alert.utils.EnterpriseWeChatUtils; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.plugin.api.AlertPlugin; +import org.apache.dolphinscheduler.plugin.model.AlertData; +import org.apache.dolphinscheduler.plugin.model.AlertInfo; +import org.apache.dolphinscheduler.plugin.model.PluginName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +/** + * EmailAlertPlugin + * + * This plugin is a default plugin, and mix up email and enterprise wechat, because adapt with former alert behavior + */ +public class EmailAlertPlugin implements AlertPlugin { + + private static final Logger logger = LoggerFactory.getLogger(EmailAlertPlugin.class); + + private PluginName pluginName; + + private static final EmailManager emailManager = new EmailManager(); + private static final EnterpriseWeChatManager weChatManager = new EnterpriseWeChatManager(); + + public EmailAlertPlugin() { + this.pluginName = new PluginName(); + this.pluginName.setEnglish(Constants.PLUGIN_DEFAULT_EMAIL_EN); + this.pluginName.setChinese(Constants.PLUGIN_DEFAULT_EMAIL_CH); + } + + @Override + public String getId() { + return Constants.PLUGIN_DEFAULT_EMAIL; + } + + @Override + public PluginName getName() { + return pluginName; + } + + @Override + @SuppressWarnings("unchecked") + public Map process(AlertInfo info) { + Map retMaps = new HashMap<>(); + + AlertData alert = info.getAlertData(); + + List receviersList = (List) info.getProp(Constants.PLUGIN_DEFAULT_EMAIL_RECEIVERS); + + // receiving group list + // custom receiver + String receivers = alert.getReceivers(); + if (StringUtils.isNotEmpty(receivers)) { + String[] splits = receivers.split(","); + receviersList.addAll(Arrays.asList(splits)); + } + + List receviersCcList = new ArrayList<>(); + // Custom Copier + String receiversCc = alert.getReceiversCc(); + if (StringUtils.isNotEmpty(receiversCc)) { + String[] splits = receiversCc.split(","); + receviersCcList.addAll(Arrays.asList(splits)); + } + + if (CollectionUtils.isEmpty(receviersList) && CollectionUtils.isEmpty(receviersCcList)) { + logger.warn("alert send error : At least one receiver address required"); + retMaps.put(Constants.STATUS, "false"); + retMaps.put(Constants.MESSAGE, "execution failure,At least one receiver address required."); + return retMaps; + } + + retMaps = emailManager.send(receviersList, receviersCcList, alert.getTitle(), alert.getContent(), + alert.getShowType()); + + //send flag + boolean flag = false; + + if (retMaps == null) { + retMaps = new HashMap<>(); + retMaps.put(Constants.MESSAGE, "alert send error."); + retMaps.put(Constants.STATUS, "false"); + logger.info("alert send error : {}", retMaps.get(Constants.MESSAGE)); + return retMaps; + } + + flag = Boolean.parseBoolean(String.valueOf(retMaps.get(Constants.STATUS))); + + if (flag) { + logger.info("alert send success"); + retMaps.put(Constants.MESSAGE, "email send success."); + if (EnterpriseWeChatUtils.isEnable()) { + logger.info("Enterprise WeChat is enable!"); + try { + String token = EnterpriseWeChatUtils.getToken(); + weChatManager.send(info, token); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + } else { + retMaps.put(Constants.MESSAGE, "alert send error."); + logger.info("alert send error : {}", retMaps.get(Constants.MESSAGE)); + } + + return retMaps; + } + +} diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java index 5feb36b60f..d3ac852b0f 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java @@ -16,139 +16,85 @@ */ package org.apache.dolphinscheduler.alert.runner; -import org.apache.dolphinscheduler.alert.manager.EmailManager; -import org.apache.dolphinscheduler.alert.manager.EnterpriseWeChatManager; import org.apache.dolphinscheduler.alert.utils.Constants; -import org.apache.dolphinscheduler.alert.utils.EnterpriseWeChatUtils; import org.apache.dolphinscheduler.common.enums.AlertStatus; -import org.apache.dolphinscheduler.common.enums.AlertType; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.plugin.PluginManager; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.entity.Alert; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.api.AlertPlugin; +import org.apache.dolphinscheduler.plugin.model.AlertData; +import org.apache.dolphinscheduler.plugin.model.AlertInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; /** * alert sender */ -public class AlertSender{ +public class AlertSender { private static final Logger logger = LoggerFactory.getLogger(AlertSender.class); - private static final EmailManager emailManager= new EmailManager(); - private static final EnterpriseWeChatManager weChatManager= new EnterpriseWeChatManager(); - - private List alertList; private AlertDao alertDao; + private PluginManager pluginManager; + + public AlertSender() { + } - public AlertSender(){} - public AlertSender(List alertList, AlertDao alertDao){ + public AlertSender(List alertList, AlertDao alertDao, PluginManager pluginManager) { super(); this.alertList = alertList; this.alertDao = alertDao; + this.pluginManager = pluginManager; } public void run() { - List users; - Map retMaps = null; - for(Alert alert:alertList){ + for (Alert alert : alertList) { users = alertDao.listUserByAlertgroupId(alert.getAlertGroupId()); // receiving group list List receviersList = new ArrayList<>(); - for(User user:users){ + for (User user : users) { receviersList.add(user.getEmail()); } - // custom receiver - String receivers = alert.getReceivers(); - if (StringUtils.isNotEmpty(receivers)){ - String[] splits = receivers.split(","); - receviersList.addAll(Arrays.asList(splits)); - } - // copy list - List receviersCcList = new ArrayList<>(); - - - // Custom Copier - String receiversCc = alert.getReceiversCc(); - - if (StringUtils.isNotEmpty(receiversCc)){ - String[] splits = receiversCc.split(","); - receviersCcList.addAll(Arrays.asList(splits)); - } - - if (CollectionUtils.isEmpty(receviersList) && CollectionUtils.isEmpty(receviersCcList)) { - logger.warn("alert send error : At least one receiver address required"); - alertDao.updateAlert(AlertStatus.EXECUTION_FAILURE, "execution failure,At least one receiver address required.", alert.getId()); - continue; - } - - if (alert.getAlertType() == AlertType.EMAIL){ - retMaps = emailManager.send(receviersList,receviersCcList, alert.getTitle(), alert.getContent(),alert.getShowType()); - - alert.setInfo(retMaps); - }else if (alert.getAlertType() == AlertType.SMS){ - retMaps = emailManager.send(getReciversForSMS(users), alert.getTitle(), alert.getContent(),alert.getShowType()); - alert.setInfo(retMaps); + AlertData alertData = new AlertData(); + alertData.setId(alert.getId()) + .setAlertGroupId(alert.getAlertGroupId()) + .setContent(alert.getContent()) + .setLog(alert.getLog()) + .setReceivers(alert.getReceivers()) + .setReceiversCc(alert.getReceiversCc()) + .setShowType(alert.getShowType().getDescp()) + .setTitle(alert.getTitle()); + + AlertInfo alertInfo = new AlertInfo(); + alertInfo.setAlertData(alertData); + + alertInfo.addProp("receivers", receviersList); + + AlertPlugin emailPlugin = pluginManager.findOne(Constants.PLUGIN_DEFAULT_EMAIL); + retMaps = emailPlugin.process(alertInfo); + + if (retMaps == null) { + alertDao.updateAlert(AlertStatus.EXECUTION_FAILURE, "alert send error", alert.getId()); + logger.info("alert send error : return value is null"); + } else if (!Boolean.parseBoolean(String.valueOf(retMaps.get(Constants.STATUS)))) { + alertDao.updateAlert(AlertStatus.EXECUTION_FAILURE, String.valueOf(retMaps.get(Constants.MESSAGE)), alert.getId()); + logger.info("alert send error : {}", retMaps.get(Constants.MESSAGE)); } else { - logger.error("AlertType is not defined. code: {}, descp: {}", - alert.getAlertType().getCode(), - alert.getAlertType().getDescp()); - return; - } - - //send flag - boolean flag = false; - - if (null != retMaps) { - flag = Boolean.parseBoolean(String.valueOf(retMaps.get(Constants.STATUS))); - } - - if (flag) { - alertDao.updateAlert(AlertStatus.EXECUTION_SUCCESS, "execution success", alert.getId()); + alertDao.updateAlert(AlertStatus.EXECUTION_SUCCESS, (String) retMaps.get(Constants.MESSAGE), alert.getId()); logger.info("alert send success"); - if (EnterpriseWeChatUtils.isEnable()) { - logger.info("Enterprise WeChat is enable!"); - try { - String token = EnterpriseWeChatUtils.getToken(); - weChatManager.send(alert, token); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - } else { - if (null != retMaps) { - alertDao.updateAlert(AlertStatus.EXECUTION_FAILURE, String.valueOf(retMaps.get(Constants.MESSAGE)), alert.getId()); - logger.info("alert send error : {}", retMaps.get(Constants.MESSAGE)); - } } } } - - /** - * get a list of SMS users - * @param users - * @return - */ - private List getReciversForSMS(List users){ - List list = new ArrayList<>(); - for (User user : users){ - list.add(user.getPhone()); - } - return list; - } } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java index 28be8aa195..8fa38c62fc 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java @@ -156,4 +156,21 @@ public class Constants { public static final String ENTERPRISE_WECHAT_AGENT_ID = "enterprise.wechat.agent.id"; public static final String ENTERPRISE_WECHAT_USERS = "enterprise.wechat.users"; + + /** + * plugin config + */ + public static final String PLUGIN_DIR = "plugin.dir"; + + public static final String PLUGIN_DEFAULT_EMAIL = "email"; + + public static final String PLUGIN_DEFAULT_EMAIL_CH = "邮件"; + + public static final String PLUGIN_DEFAULT_EMAIL_EN = "email"; + + public static final String PLUGIN_DEFAULT_EMAIL_RECEIVERS = "receivers"; + + public static final String PLUGIN_DEFAULT_EMAIL_RECEIVERCCS = "receiverCcs"; + + public static final String RETMAP_MSG = "msg"; } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java index 170c0dd37e..d199d154aa 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java @@ -18,10 +18,10 @@ package org.apache.dolphinscheduler.alert.utils; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.dao.entity.Alert; import com.alibaba.fastjson.JSON; import com.google.common.reflect.TypeToken; +import org.apache.dolphinscheduler.plugin.model.AlertData; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; @@ -66,15 +66,17 @@ public class EnterpriseWeChatUtils { * get Enterprise WeChat is enable * @return isEnable */ - public static Boolean isEnable(){ - Boolean isEnable = false; + public static boolean isEnable(){ + Boolean isEnable = null; try { isEnable = PropertyUtils.getBoolean(Constants.ENTERPRISE_WECHAT_ENABLE); } catch (Exception e) { logger.error(e.getMessage(),e); } + if (isEnable == null) { + return false; + } return isEnable; - } /** @@ -253,14 +255,13 @@ public class EnterpriseWeChatUtils { /** * Determine the mardown style based on the show type of the alert - * @param alert the alert * @return the markdown alert table/text */ - public static String markdownByAlert(Alert alert){ + public static String markdownByAlert(AlertData alert){ String result = ""; - if (alert.getShowType() == ShowType.TABLE) { + if (alert.getShowType().equals(ShowType.TABLE.getDescp())) { result = markdownTable(alert.getTitle(),alert.getContent()); - }else if(alert.getShowType() == ShowType.TEXT){ + }else if(alert.getShowType().equals(ShowType.TEXT.getDescp())){ result = markdownText(alert.getTitle(),alert.getContent()); } return result; diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java index ef364cb1c2..bb565f5133 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java @@ -74,7 +74,7 @@ public class MailUtils { * @param showType the show type * @return the result map */ - public static Map sendMails(Collection receivers, String title, String content,ShowType showType) { + public static Map sendMails(Collection receivers, String title, String content,String showType) { return sendMails(receivers, null, title, content, showType); } @@ -87,7 +87,7 @@ public class MailUtils { * @param showType the show type * @return the send result */ - public static Map sendMails(Collection receivers, Collection receiversCc, String title, String content, ShowType showType) { + public static Map sendMails(Collection receivers, Collection receiversCc, String title, String content, String showType) { Map retMap = new HashMap<>(); retMap.put(Constants.STATUS, false); @@ -98,7 +98,7 @@ public class MailUtils { receivers.removeIf(StringUtils::isEmpty); - if (showType == ShowType.TABLE || showType == ShowType.TEXT){ + if (showType.equals(ShowType.TABLE.getDescp()) || showType.equals(ShowType.TEXT.getDescp())) { // send email HtmlEmail email = new HtmlEmail(); @@ -125,10 +125,10 @@ public class MailUtils { } catch (Exception e) { handleException(receivers, retMap, e); } - }else if (showType == ShowType.ATTACHMENT || showType == ShowType.TABLEATTACHMENT){ + }else if (showType.equals(ShowType.ATTACHMENT.getDescp()) || showType.equals(ShowType.TABLEATTACHMENT.getDescp())) { try { - String partContent = (showType == ShowType.ATTACHMENT ? "Please see the attachment " + title + Constants.EXCEL_SUFFIX_XLS : htmlTable(content,false)); + String partContent = (showType.equals(ShowType.ATTACHMENT.getDescp()) ? "Please see the attachment " + title + Constants.EXCEL_SUFFIX_XLS : htmlTable(content,false)); attachment(receivers,receiversCc,title,content,partContent); @@ -290,7 +290,7 @@ public class MailUtils { * @return the result map * @throws EmailException */ - private static Map getStringObjectMap(String title, String content, ShowType showType, Map retMap, HtmlEmail email) throws EmailException { + private static Map getStringObjectMap(String title, String content, String showType, Map retMap, HtmlEmail email) throws EmailException { /** * the subject of the message to be sent @@ -299,9 +299,9 @@ public class MailUtils { /** * to send information, you can use HTML tags in mail content because of the use of HtmlEmail */ - if (showType == ShowType.TABLE) { + if (showType.equals(ShowType.TABLE.getDescp())) { email.setMsg(htmlTable(content)); - } else if (showType == ShowType.TEXT) { + } else if (showType.equals(ShowType.TEXT.getDescp())) { email.setMsg(htmlText(content)); } diff --git a/dolphinscheduler-alert/src/main/resources/alert.properties b/dolphinscheduler-alert/src/main/resources/alert.properties index 3e83c01235..19b55fec97 100644 --- a/dolphinscheduler-alert/src/main/resources/alert.properties +++ b/dolphinscheduler-alert/src/main/resources/alert.properties @@ -45,5 +45,5 @@ enterprise.wechat.enable=false #enterprise.wechat.team.send.msg={\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"} #enterprise.wechat.user.send.msg={\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}} - +plugin.dir=/Users/xx/your/path/to/plugin/dir diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java new file mode 100644 index 0000000000..52c2fc050e --- /dev/null +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.alert.plugin; + +import org.apache.dolphinscheduler.alert.utils.Constants; +import org.apache.dolphinscheduler.common.enums.ShowType; +import org.apache.dolphinscheduler.plugin.api.AlertPlugin; +import org.apache.dolphinscheduler.plugin.model.AlertData; +import org.apache.dolphinscheduler.plugin.model.AlertInfo; +import org.apache.dolphinscheduler.plugin.model.PluginName; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.*; + +public class EmailAlertPluginTest { + + private static final Logger logger = LoggerFactory.getLogger(EmailAlertPluginTest.class); + + private AlertPlugin plugin; + + @Before + public void before() { + plugin = new EmailAlertPlugin(); + } + + @Test + public void getId() { + String id = plugin.getId(); + assertEquals(Constants.PLUGIN_DEFAULT_EMAIL, id); + } + + @Test + public void getName() { + PluginName pluginName = plugin.getName(); + assertEquals(Constants.PLUGIN_DEFAULT_EMAIL_CH, pluginName.getChinese()); + assertEquals(Constants.PLUGIN_DEFAULT_EMAIL_EN, pluginName.getEnglish()); + } + + @Test + public void process() { + AlertInfo alertInfo = new AlertInfo(); + AlertData alertData = new AlertData(); + alertData.setId(1) + .setAlertGroupId(1) + .setContent("[\"alarm time：2018-02-05\", \"service name：MYSQL_ALTER\", \"alarm name：MYSQL_ALTER_DUMP\", " + + "\"get the alarm exception.！，interface error，exception information：timed out\", \"request address：http://blog.csdn.net/dreamInTheWorld/article/details/78539286\"]") + .setLog("test log") + .setReceivers("xx@xx.com") + .setReceiversCc("xx@xx.com") + .setShowType(ShowType.TEXT.getDescp()) + .setTitle("test title"); + + alertInfo.setAlertData(alertData); + List list = new ArrayList(){{ add("xx@xx.com"); }}; + alertInfo.addProp("receivers", list); + Map ret = plugin.process(alertInfo); + assertFalse(Boolean.parseBoolean(String.valueOf(ret.get(Constants.STATUS)))); + } +} \ No newline at end of file diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java index d0f3538c1b..2b405cc436 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java @@ -20,6 +20,7 @@ import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.dao.entity.Alert; +import org.apache.dolphinscheduler.plugin.model.AlertData; import org.junit.Assert; import org.junit.Before; import org.junit.Ignore; @@ -120,14 +121,22 @@ public class EnterpriseWeChatUtilsTest { @Test public void testMarkdownByAlertForText(){ Alert alertForText = createAlertForText(); - String result = EnterpriseWeChatUtils.markdownByAlert(alertForText); + AlertData alertData = new AlertData(); + alertData.setTitle(alertForText.getTitle()) + .setShowType(alertForText.getShowType().getDescp()) + .setContent(alertForText.getContent()); + String result = EnterpriseWeChatUtils.markdownByAlert(alertData); Assert.assertNotNull(result); } @Test public void testMarkdownByAlertForTable(){ Alert alertForText = createAlertForTable(); - String result = EnterpriseWeChatUtils.markdownByAlert(alertForText); + AlertData alertData = new AlertData(); + alertData.setTitle(alertForText.getTitle()) + .setShowType(alertForText.getShowType().getDescp()) + .setContent(alertForText.getContent()); + String result = EnterpriseWeChatUtils.markdownByAlert(alertData); Assert.assertNotNull(result); } diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java index 1820a1ef89..11322da0e3 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java @@ -58,7 +58,7 @@ public class MailUtilsTest { alert.setAlertType(AlertType.EMAIL); alert.setAlertGroupId(4); - MailUtils.sendMails(Arrays.asList(receivers),Arrays.asList(receiversCc),alert.getTitle(),alert.getContent(), ShowType.TEXT); + MailUtils.sendMails(Arrays.asList(receivers),Arrays.asList(receiversCc),alert.getTitle(),alert.getContent(), ShowType.TEXT.getDescp()); } @@ -70,7 +70,7 @@ public class MailUtilsTest { String[] mails = new String[]{"xx@xx.com"}; for(Alert alert : alerts){ - MailUtils.sendMails(Arrays.asList(mails),"gaojing", alert.getContent(), alert.getShowType()); + MailUtils.sendMails(Arrays.asList(mails),"gaojing", alert.getContent(), ShowType.TABLE.getDescp()); } } @@ -111,7 +111,7 @@ public class MailUtilsTest { alert.setContent(content); alert.setAlertType(AlertType.EMAIL); alert.setAlertGroupId(1); - MailUtils.sendMails(Arrays.asList(mails),"gaojing", alert.getContent(), ShowType.TABLE); + MailUtils.sendMails(Arrays.asList(mails),"gaojing", alert.getContent(), ShowType.TABLE.getDescp()); } /** @@ -170,7 +170,7 @@ public class MailUtilsTest { alert.setContent(content); alert.setAlertType(AlertType.EMAIL); alert.setAlertGroupId(1); - MailUtils.sendMails(Arrays.asList(mails),"gaojing",alert.getContent(),ShowType.ATTACHMENT); + MailUtils.sendMails(Arrays.asList(mails),"gaojing",alert.getContent(),ShowType.ATTACHMENT.getDescp()); } @Test @@ -183,7 +183,7 @@ public class MailUtilsTest { alert.setContent(content); alert.setAlertType(AlertType.EMAIL); alert.setAlertGroupId(1); - MailUtils.sendMails(Arrays.asList(mails),"gaojing",alert.getContent(),ShowType.TABLEATTACHMENT); + MailUtils.sendMails(Arrays.asList(mails),"gaojing",alert.getContent(),ShowType.TABLEATTACHMENT.getDescp()); } } diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index ca75a84a62..3954159dfa 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -32,12 +32,15 @@ 3.1.0 + + org.apache.dolphinscheduler + dolphinscheduler-plugin-api + com.alibaba fastjson compile - org.apache.httpcomponents httpclient diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 853ab95d1c..effa4f0f8e 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -954,4 +954,11 @@ public final class Constants { * authorize readable perm */ public static final int AUTHORIZE_READABLE_PERM=4; + + + /** + * plugin configurations + */ + public static final String PLUGIN_JAR_SUFFIX = ".jar"; + } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/FilePluginManager.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/FilePluginManager.java new file mode 100644 index 0000000000..d4dd4bf283 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/FilePluginManager.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.plugin; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.plugin.api.AlertPlugin; +import org.apache.dolphinscheduler.plugin.spi.AlertPluginProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Map; +import java.util.ServiceLoader; +import java.util.concurrent.ConcurrentHashMap; + +/** + * FilePluginManager + */ +public class FilePluginManager implements PluginManager { + + private static final Logger logger = LoggerFactory.getLogger(FilePluginManager.class); + + private Map pluginMap = new ConcurrentHashMap<>(); + + private Map> pluginLoaderMap = new ConcurrentHashMap<>(); + + private Map classLoaderMap = new ConcurrentHashMap<>(); + + private String[] whitePrefixes; + + private String[] excludePrefixes; + + public FilePluginManager(String dirPath, String[] whitePrefixes, String[] excludePrefixes) { + this.whitePrefixes = whitePrefixes; + this.excludePrefixes = excludePrefixes; + try { + load(dirPath); + } catch (MalformedURLException e) { + logger.error("load plugins failed.", e); + } + } + + private void load(String dirPath) throws MalformedURLException { + logger.info("start to load jar files in {}", dirPath); + if (dirPath == null) { + logger.error("not a valid path - {}", dirPath); + return; + } + File[] files = new File(dirPath).listFiles(); + if (files == null) { + logger.error("not a valid path - {}", dirPath); + return; + } + for (File file : files) { + if (file.isDirectory() && !file.getPath().endsWith(Constants.PLUGIN_JAR_SUFFIX)) { + continue; + } + String pluginName = file.getName() + .substring(0, file.getName().length() - Constants.PLUGIN_JAR_SUFFIX.length()); + URL[] urls = new URL[]{ file.toURI().toURL() }; + PluginClassLoader classLoader = + new PluginClassLoader(urls, Thread.currentThread().getContextClassLoader(), whitePrefixes, excludePrefixes); + classLoaderMap.put(pluginName, classLoader); + + ServiceLoader loader = ServiceLoader.load(AlertPluginProvider.class, classLoader); + pluginLoaderMap.put(pluginName, loader); + + loader.forEach(provider -> { + AlertPlugin plugin = provider.createPlugin(); + pluginMap.put(plugin.getId(), plugin); + logger.info("loaded plugin - {}", plugin.getId()); + }); + } + } + + @Override + public AlertPlugin findOne(String name) { + return pluginMap.get(name); + } + + @Override + public Map findAll() { + return pluginMap; + } + + @Override + public void addPlugin(AlertPlugin plugin) { + pluginMap.put(plugin.getId(), plugin); + } + +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/PluginClassLoader.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/PluginClassLoader.java new file mode 100644 index 0000000000..528e83a59c --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/PluginClassLoader.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.plugin; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Enumeration; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; + +/** + * Plugin Class Loader + */ +public class PluginClassLoader extends URLClassLoader { + + private static final Logger logger = LoggerFactory.getLogger(PluginClassLoader.class); + + private static final String JAVA_PACKAGE_PREFIX = "java."; + private static final String JAVAX_PACKAGE_PREFIX = "javax."; + + private final String[] whitePrefixes; + + private final String[] excludePrefixes; + + public PluginClassLoader(URL[] urls, ClassLoader parent, String[] whitePrefix, String[] excludePreifx) { + super(urls, parent); + this.whitePrefixes = whitePrefix; + this.excludePrefixes = excludePreifx; + } + + @Override + public Class loadClass(String name) throws ClassNotFoundException { + logger.trace("Received request to load class '{}'", name); + synchronized (getClassLoadingLock(name)) { + if (name.startsWith(JAVA_PACKAGE_PREFIX) || name.startsWith(JAVAX_PACKAGE_PREFIX)) { + return findSystemClass(name); + } + + boolean isWhitePrefixes = fromWhitePrefix(name); + boolean isExcludePrefixed = fromExcludePrefix(name); + + // if the class is part of the plugin engine use parent class loader + if (!isWhitePrefixes && isExcludePrefixed) { + return getParent().loadClass(name); + } + + // check whether it's already been loaded + Class loadedClass = findLoadedClass(name); + if (loadedClass != null) { + logger.debug("Found loaded class '{}'", name); + return loadedClass; + } + + // nope, try to load locally + try { + loadedClass = findClass(name); + logger.debug("Found class '{}' in plugin classpath", name); + return loadedClass; + } catch (ClassNotFoundException e) { + // try next step + } + + // use the standard ClassLoader (which follows normal parent delegation) + return super.loadClass(name); + } + } + + private boolean fromWhitePrefix(String name) { + if (this.whitePrefixes == null) { + return false; + } + for (String whitePrefix : this.whitePrefixes) { + if (name.startsWith(whitePrefix)) { + return true; + } + } + return false; + } + + private boolean fromExcludePrefix(String name) { + if (this.excludePrefixes == null) { + return false; + } + for (String excludePrefix : this.excludePrefixes) { + if (name.startsWith(excludePrefix)) { + return true; + } + } + return false; + } + + @Override + public Enumeration getResources(String name) throws IOException { + List allRes = new LinkedList<>(); + + Enumeration thisRes = findResources(name); + if (thisRes != null) { + while (thisRes.hasMoreElements()) { + allRes.add(thisRes.nextElement()); + } + } + + Enumeration parentRes = super.findResources(name); + if (parentRes != null) { + while (parentRes.hasMoreElements()) { + allRes.add(parentRes.nextElement()); + } + } + + return new Enumeration() { + Iterator it = allRes.iterator(); + + @Override + public boolean hasMoreElements() { + return it.hasNext(); + } + + @Override + public URL nextElement() { + return it.next(); + } + }; + } + + @Override + public URL getResource(String name) { + URL res = null; + + res = findResource(name); + if (res == null) { + res = super.getResource(name); + } + return res; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/PluginManager.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/PluginManager.java new file mode 100644 index 0000000000..f8078841e4 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/PluginManager.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.plugin; + +import org.apache.dolphinscheduler.plugin.api.AlertPlugin; + +import java.util.Map; + +/** + * PluginManager + */ +public interface PluginManager { + + AlertPlugin findOne(String name); + + Map findAll(); + + void addPlugin(AlertPlugin plugin); +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/plugin/FilePluginManagerTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/plugin/FilePluginManagerTest.java new file mode 100644 index 0000000000..1a57cb10fa --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/plugin/FilePluginManagerTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.plugin; + +import org.apache.dolphinscheduler.plugin.api.AlertPlugin; +import org.apache.dolphinscheduler.plugin.model.AlertInfo; +import org.apache.dolphinscheduler.plugin.model.PluginName; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.*; + +public class FilePluginManagerTest { + + private FilePluginManager filePluginManager; + private AlertPlugin alertPlugin; + + @Before + public void before() { + filePluginManager = new FilePluginManager(null, null, null); + alertPlugin = new AlertPlugin() { + @Override + public String getId() { + return "test"; + } + + @Override + public PluginName getName() { + return new PluginName().setChinese("ch").setEnglish("en"); + } + + @Override + public Map process(AlertInfo info) { + return new HashMap<>(); + } + }; + } + + @Test + public void findOne() { + filePluginManager.addPlugin(alertPlugin); + assertEquals(alertPlugin, filePluginManager.findOne(alertPlugin.getId())); + } + + @Test + public void findAll() { + assertNotNull(filePluginManager.findAll()); + } + + @Test + public void addPlugin() { + filePluginManager.addPlugin(alertPlugin); + assertNotNull(filePluginManager.findAll()); + } +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/plugin/PluginClassLoaderTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/plugin/PluginClassLoaderTest.java new file mode 100644 index 0000000000..8a6bfaee13 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/plugin/PluginClassLoaderTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.plugin; + +import org.junit.Before; +import org.junit.Test; + +import java.net.URL; + +import static org.junit.Assert.*; + +public class PluginClassLoaderTest { + + private PluginClassLoader pluginClassLoader; + private ClassLoader parent; + + @Before + public void setUp() { + parent = Thread.currentThread().getContextClassLoader(); + pluginClassLoader = new PluginClassLoader( + new URL[]{}, parent, + null, null); + } + + @Test + public void loadClassNull() { + Class clazz = null; + try { + clazz = pluginClassLoader.loadClass("java.lang.Object"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + assertEquals(null, clazz.getClassLoader()); + } + + @Test + public void loadClassApp() { + Class clazz = null; + try { + clazz = pluginClassLoader.loadClass("org.apache.dolphinscheduler.common.Constants"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + assertEquals(parent, clazz.getClassLoader()); + } + +} \ No newline at end of file diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index 82e641ec72..52563f5573 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -474,7 +474,6 @@ The following components are provided under a MIT 2.0 license. See project link The text of each license is also included at licenses/LICENSE-[project].txt. jul-to-slf4j 1.7.25: https://mvnrepository.com/artifact/org.slf4j/jul-to-slf4j/1.7.25, MIT - lombok 1.18.6: https://mvnrepository.com/artifact/org.projectlombok/lombok/1.18.6, MIT mssql-jdbc 6.1.0.jre8: https://mvnrepository.com/artifact/com.microsoft.sqlserver/mssql-jdbc/6.1.0.jre8, MIT slf4j-api 1.7.5: https://mvnrepository.com/artifact/org.slf4j/slf4j-api/1.7.5, MIT diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-lombok.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-lombok.txt deleted file mode 100644 index 5da7dbeff3..0000000000 --- a/dolphinscheduler-dist/release-docs/licenses/LICENSE-lombok.txt +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (C) 2009-2015 The Project Lombok Authors. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/dolphinscheduler-plugin-api/pom.xml b/dolphinscheduler-plugin-api/pom.xml new file mode 100644 index 0000000000..b958b21616 --- /dev/null +++ b/dolphinscheduler-plugin-api/pom.xml @@ -0,0 +1,52 @@ + + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler + 1.2.1-SNAPSHOT + + dolphinscheduler-plugin-api + ${project.artifactId} + jar + + + UTF-8 + + + + + org.slf4j + slf4j-api + + + junit + junit + test + + + commons-io + commons-io + + + + diff --git a/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/api/AlertPlugin.java b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/api/AlertPlugin.java new file mode 100644 index 0000000000..deb7ff6aa4 --- /dev/null +++ b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/api/AlertPlugin.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.api; + +import org.apache.dolphinscheduler.plugin.model.AlertInfo; +import org.apache.dolphinscheduler.plugin.model.PluginName; + +import java.util.Map; + +/** + * Plugin + */ +public interface AlertPlugin { + + /** + * Get alert plugin id + * + * @return alert plugin id, which should be unique + */ + String getId(); + + /** + * Get alert plugin name, which will show in front end portal + * + * @return plugin name + */ + PluginName getName(); + + Map process(AlertInfo info); + +} diff --git a/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/AlertData.java b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/AlertData.java new file mode 100644 index 0000000000..89ab5c4279 --- /dev/null +++ b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/AlertData.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.model; + +/** + * AlertData + */ +public class AlertData { + + /** + * alert primary key + */ + private int id; + /** + * title + */ + private String title; + /** + * content + */ + private String content; + /** + * log + */ + private String log; + /** + * alertgroup_id + */ + private int alertGroupId; + /** + * receivers + */ + private String receivers; + /** + * show_type + */ + private String showType; + /** + * receivers_cc + */ + private String receiversCc; + + public int getId() { + return id; + } + + public AlertData setId(int id) { + this.id = id; + return this; + } + + public String getTitle() { + return title; + } + + public AlertData setTitle(String title) { + this.title = title; + return this; + } + + public String getContent() { + return content; + } + + public AlertData setContent(String content) { + this.content = content; + return this; + } + + public String getLog() { + return log; + } + + public AlertData setLog(String log) { + this.log = log; + return this; + } + + public int getAlertGroupId() { + return alertGroupId; + } + + public AlertData setAlertGroupId(int alertGroupId) { + this.alertGroupId = alertGroupId; + return this; + } + + public String getReceivers() { + return receivers; + } + + public AlertData setReceivers(String receivers) { + this.receivers = receivers; + return this; + } + + public String getReceiversCc() { + return receiversCc; + } + + public AlertData setReceiversCc(String receiversCc) { + this.receiversCc = receiversCc; + return this; + } + + public String getShowType() { + return showType; + } + + public AlertData setShowType(String showType) { + this.showType = showType; + return this; + } + +} diff --git a/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/AlertInfo.java b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/AlertInfo.java new file mode 100644 index 0000000000..1d71ed7d8a --- /dev/null +++ b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/AlertInfo.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.model; + +import java.util.HashMap; +import java.util.Map; + +/** + * AlertInfo + */ +public class AlertInfo { + + private Map alertProps; + + private AlertData alertData; + + public AlertInfo() { + this.alertProps = new HashMap<>(); + } + + public Map getAlertProps() { + return alertProps; + } + + public AlertInfo setAlertProps(Map alertProps) { + this.alertProps = alertProps; + return this; + } + + public AlertInfo addProp(String key, Object value) { + this.alertProps.put(key, value); + return this; + } + + public Object getProp(String key) { + return this.alertProps.get(key); + } + + public AlertData getAlertData() { + return alertData; + } + + public AlertInfo setAlertData(AlertData alertData) { + this.alertData = alertData; + return this; + } +} diff --git a/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/PluginName.java b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/PluginName.java new file mode 100644 index 0000000000..8066e45f1d --- /dev/null +++ b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/PluginName.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.model; + +/** + * PluginName + */ +public class PluginName { + + private String chinese; + + private String english; + + public String getChinese() { + return chinese; + } + + public PluginName setChinese(String chinese) { + this.chinese = chinese; + return this; + } + + public String getEnglish() { + return english; + } + + public PluginName setEnglish(String english) { + this.english = english; + return this; + } +} diff --git a/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/spi/AlertPluginProvider.java b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/spi/AlertPluginProvider.java new file mode 100644 index 0000000000..594636f4eb --- /dev/null +++ b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/spi/AlertPluginProvider.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.spi; + +import org.apache.dolphinscheduler.plugin.api.AlertPlugin; + +/** + * PluginProvider + */ +public interface AlertPluginProvider { + + /** + * create an alert plugin + * + * @return an alert plugin + */ + AlertPlugin createPlugin(); + +} diff --git a/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/utils/PropertyUtils.java b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/utils/PropertyUtils.java new file mode 100644 index 0000000000..a244dd491e --- /dev/null +++ b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/utils/PropertyUtils.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.utils; + +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +/** + * property utils + * single instance + */ +public class PropertyUtils { + + private static final Logger logger = LoggerFactory.getLogger(PropertyUtils.class); + + private static final Properties properties = new Properties(); + + private PropertyUtils() { + throw new IllegalStateException("PropertyUtils class"); + } + + static { + String propertyFiles = "/plugin.properties"; + InputStream fis = null; + try { + fis = PropertyUtils.class.getResourceAsStream(propertyFiles); + properties.load(fis); + } catch (IOException e) { + logger.error(e.getMessage(), e); + if (fis != null) { + IOUtils.closeQuietly(fis); + } + } finally { + IOUtils.closeQuietly(fis); + } + } + + /** + * get property value + * + * @param key property name + * @param defaultVal default value + * @return property value + */ + public static String getString(String key, String defaultVal) { + String val = properties.getProperty(key.trim()); + return val == null ? defaultVal : val; + } + + /** + * get property value + * + * @param key property name + * @return property value + */ + public static String getString(String key) { + if (key == null) { + return null; + } + return properties.getProperty(key.trim()); + } + + /** + * get property value + * + * @param key property name + * @return get property int value , if key == null, then return -1 + */ + public static int getInt(String key) { + return getInt(key, -1); + } + + /** + * get int + * + * @param key key + * @param defaultValue default value + * @return property value + */ + public static int getInt(String key, int defaultValue) { + String value = properties.getProperty(key.trim()); + if (value == null) { + return defaultValue; + } + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + logger.info(e.getMessage(), e); + } + return defaultValue; + } + + /** + * get property value + * + * @param key property name + * @return property value + */ + public static boolean getBoolean(String key) { + String value = properties.getProperty(key.trim()); + if (value == null) { + return false; + } + + return Boolean.parseBoolean(value); + } + + /** + * get property value + * + * @param key property name + * @param defaultValue default value + * @return property value + */ + public static Boolean getBoolean(String key, boolean defaultValue) { + String value = properties.getProperty(key.trim()); + if (value == null) { + return defaultValue; + } + + return Boolean.parseBoolean(value); + } + + /** + * get property long value + * + * @param key key + * @param defaultVal default value + * @return property value + */ + public static long getLong(String key, long defaultVal) { + String val = getString(key); + return val == null ? defaultVal : Long.parseLong(val); + } + + /** + * get long + * + * @param key key + * @return property value + */ + public static long getLong(String key) { + return getLong(key, -1); + } + + /** + * get double + * + * @param key key + * @param defaultVal default value + * @return property value + */ + public static double getDouble(String key, double defaultVal) { + String val = properties.getProperty(key.trim()); + return val == null ? defaultVal : Double.parseDouble(val); + } + + /** + * @param key key + * @param type type + * @param defaultValue default value + * @param T + * @return get enum value + */ + public > T getEnum(String key, Class type, + T defaultValue) { + String val = properties.getProperty(key.trim()); + return val == null ? defaultValue : Enum.valueOf(type, val); + } + +} diff --git a/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/model/AlertDataTest.java b/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/model/AlertDataTest.java new file mode 100644 index 0000000000..c19b5bc29a --- /dev/null +++ b/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/model/AlertDataTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.model; + +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class AlertDataTest { + + private AlertData alertData; + + @Before + public void before() { + alertData = new AlertData(); + alertData.setId(1) + .setContent("content") + .setShowType("email") + .setTitle("title") + .setReceivers("receivers") + .setReceiversCc("cc") + .setLog("log") + .setAlertGroupId(1); + } + + @Test + public void getId() { + assertEquals(1, alertData.getId()); + } + + @Test + public void getTitle() { + assertEquals("title", alertData.getTitle()); + } + + @Test + public void getContent() { + assertEquals("content", alertData.getContent()); + } + + @Test + public void getLog() { + assertEquals("log", alertData.getLog()); + } + + @Test + public void getAlertGroupId() { + assertEquals(1, alertData.getAlertGroupId()); + } + + @Test + public void getReceivers() { + assertEquals("receivers", alertData.getReceivers()); + } + + @Test + public void getReceiversCc() { + assertEquals("cc", alertData.getReceiversCc()); + } + + @Test + public void getShowType() { + assertEquals("email", alertData.getShowType()); + } +} \ No newline at end of file diff --git a/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/model/AlertInfoTest.java b/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/model/AlertInfoTest.java new file mode 100644 index 0000000000..13eb595ac3 --- /dev/null +++ b/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/model/AlertInfoTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.model; + +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.*; + +public class AlertInfoTest { + + private AlertInfo alertInfo; + + @Before + public void before() { + alertInfo = new AlertInfo(); + } + + @Test + public void getAlertProps() { + Map map = new HashMap<>(); + alertInfo.setAlertProps(map); + assertNotNull(alertInfo.getAlertProps()); + } + + @Test + public void getProp() { + alertInfo.addProp("k", "v"); + assertEquals("v", alertInfo.getProp("k")); + } + + @Test + public void getAlertData() { + alertInfo.setAlertData(new AlertData()); + assertNotNull(alertInfo.getAlertData()); + } +} \ No newline at end of file diff --git a/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/utils/PropertyUtilsTest.java b/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/utils/PropertyUtilsTest.java new file mode 100644 index 0000000000..614a7c009c --- /dev/null +++ b/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/utils/PropertyUtilsTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.utils; + +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.junit.Assert.*; + +public class PropertyUtilsTest { + + private static final Logger logger = LoggerFactory.getLogger(PropertyUtilsTest.class); + + /** + * Test getString + */ + @Test + public void testGetString() { + + String result = PropertyUtils.getString("test.string"); + logger.info(result); + assertEquals("teststring", result); + + //If key is null, then return null + result = PropertyUtils.getString(null); + assertNull(result); + } + + + /** + * Test getBoolean + */ + @Test + public void testGetBoolean() { + + //Expected true + Boolean result = PropertyUtils.getBoolean("test.true"); + assertTrue(result); + + //Expected false + result = PropertyUtils.getBoolean("test.false"); + assertFalse(result); + } + + /** + * Test getLong + */ + @Test + public void testGetLong() { + long result = PropertyUtils.getLong("test.long"); + assertSame(100L, result); + } + + /** + * Test getDouble + */ + @Test + public void testGetDouble() { + + //If key is undefine in alert.properties, and there is a defaultval, then return defaultval + double result = PropertyUtils.getDouble("abc", 5.0); + assertEquals(5.0, result, 0); + + result = PropertyUtils.getDouble("cba", 5.0); + assertEquals(3.1, result, 0.01); + } + +} \ No newline at end of file diff --git a/dolphinscheduler-plugin-api/src/test/resources/plugin.properties b/dolphinscheduler-plugin-api/src/test/resources/plugin.properties new file mode 100644 index 0000000000..d2ea3831be --- /dev/null +++ b/dolphinscheduler-plugin-api/src/test/resources/plugin.properties @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +test.string=teststring +test.false=false +test.true=true +cba=3.1 +test.long=100 \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index 84e4e54a50..ccd4bb2214 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -457,7 +457,7 @@ public class SqlTask extends AbstractTask { String showTypeName = sqlParameters.getShowType().replace(COMMA,"").trim(); if(EnumUtils.isValidEnum(ShowType.class,showTypeName)){ Map mailResult = MailUtils.sendMails(receviersList, - receviersCcList, title, content, ShowType.valueOf(showTypeName)); + receviersCcList, title, content, ShowType.valueOf(showTypeName).getDescp()); if(!(boolean) mailResult.get(STATUS)){ throw new RuntimeException("send mail failed!"); } diff --git a/dolphinscheduler-ui/package.json b/dolphinscheduler-ui/package.json index b23969803b..5dc1cb44db 100644 --- a/dolphinscheduler-ui/package.json +++ b/dolphinscheduler-ui/package.json @@ -55,7 +55,7 @@ "html-loader": "^0.5.5", "html-webpack-plugin": "^3.2.0", "mini-css-extract-plugin": "^0.8.2", - "node-sass": "^4.13.1", + "node-sass": "^4.14.0", "postcss-loader": "^3.0.0", "progress-bar-webpack-plugin": "^1.12.1", "rimraf": "^2.6.2", diff --git a/pom.xml b/pom.xml index dad1e3696b..7e291d2c4e 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,6 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - 4.0.0 @@ -205,6 +204,11 @@ dolphinscheduler-server ${project.version} + + org.apache.dolphinscheduler + dolphinscheduler-plugin-api + ${project.version} + org.apache.dolphinscheduler dolphinscheduler-common @@ -690,6 +694,7 @@ **/alert/utils/FuncUtilsTest.java **/alert/utils/JSONUtilsTest.java **/alert/utils/MailUtilsTest.java + **/alert/plugin/EmailAlertPluginTest.java **/api/dto/resources/filter/ResourceFilterTest.java **/api/dto/resources/visitor/ResourceTreeVisitorTest.java **/api/enums/testGetEnum.java @@ -765,6 +770,8 @@ **/common/utils/HttpUtilsTest.java **/common/ConstantsTest.java **/common/utils/HadoopUtils.java + **/common/plugin/FilePluginManagerTest + **/common/plugin/PluginClassLoaderTest **/dao/mapper/AccessTokenMapperTest.java **/dao/mapper/AlertGroupMapperTest.java **/dao/mapper/CommandMapperTest.java @@ -831,6 +838,9 @@ **/dao/mapper/UserMapperTest.java **/dao/utils/DagHelperTest.java **/dao/AlertDaoTest.java + **/plugin/model/AlertDataTest.java + **/plugin/model/AlertInfoTest.java + **/plugin/utils/PropertyUtilsTest.java @@ -866,6 +876,22 @@ apache-rat-plugin ${apache.rat.version} + false + false + + + AL20 + Apache License, 2.0 + + Licensed to the Apache Software Foundation (ASF) + + + + + + Apache License, 2.0 + + **/node_modules/** **/node/** @@ -884,7 +910,7 @@ **/*.babelrc **/*.eslintrc **/.mvn/jvm.config - **/.mvn/wrapper/maven-wrapper.properties + **/.mvn/wrapper/** true @@ -974,6 +1000,6 @@ dolphinscheduler-dist dolphinscheduler-remote dolphinscheduler-service + dolphinscheduler-plugin-api - diff --git a/tools/dependencies/check-LICENSE.sh b/tools/dependencies/check-LICENSE.sh new file mode 100755 index 0000000000..d414bd40c6 --- /dev/null +++ b/tools/dependencies/check-LICENSE.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +mkdir dist || true + +tar -zxf dolphinscheduler-dist/target/apache-dolphinscheduler*-bin.tar.gz --strip=1 -C dist + +# List all modules(jars) that belong to the DolphinScheduler itself, these will be ignored when checking the dependency +# licenses +echo '=== Self modules: ' && ./mvnw --batch-mode --quiet -Dexec.executable='echo' -Dexec.args='${project.artifactId}-${project.version}.jar' exec:exec | tee self-modules.txt + +echo '=== Distributed dependencies: ' && ls dist/lib | tee all-dependencies.txt + +# Exclude all self modules(jars) to generate all third-party dependencies +echo '=== Third party dependencies: ' && grep -vf self-modules.txt all-dependencies.txt | tee third-party-dependencies.txt + +# 1. Compare the third-party dependencies with known dependencies, expect that all third-party dependencies are KNOWN +# and the exit code of the command is 0, otherwise we should add its license to LICENSE file and add the dependency to +# known-dependencies.txt. 2. Unify the `sort` behaviour: here we'll sort them again in case that the behaviour of `sort` +# command in target OS is different from what we used to sort the file `known-dependencies.txt`, i.e. "sort the two file +# using the same command (and default arguments)" + +diff -w -B -U0 <(sort < tools/dependencies/known-dependencies.txt) <(sort < third-party-dependencies.txt) diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt new file mode 100755 index 0000000000..6601cf9c34 --- /dev/null +++ b/tools/dependencies/known-dependencies.txt @@ -0,0 +1,211 @@ +HikariCP-3.2.0.jar +activation-1.1.jar +ant-1.6.5.jar +aopalliance-1.0.jar +apache-el-8.5.35.1.jar +apacheds-i18n-2.0.0-M15.jar +apacheds-kerberos-codec-2.0.0-M15.jar +api-asn1-api-1.0.0-M20.jar +api-util-1.0.0-M20.jar +asm-3.1.jar +aspectjweaver-1.9.2.jar +audience-annotations-0.5.0.jar +avro-1.7.4.jar +aws-java-sdk-1.7.4.jar +bonecp-0.8.0.RELEASE.jar +byte-buddy-1.9.10.jar +classmate-1.4.0.jar +clickhouse-jdbc-0.1.52.jar +commons-cli-1.2.jar +commons-codec-1.6.jar +commons-collections-3.2.2.jar +commons-collections4-4.1.jar +commons-compiler-3.0.12.jar +commons-compress-1.4.1.jar +commons-configuration-1.10.jar +commons-daemon-1.0.13.jar +commons-dbcp-1.4.jar +commons-email-1.5.jar +commons-httpclient-3.0.1.jar +commons-io-2.4.jar +commons-lang-2.6.jar +commons-logging-1.1.1.jar +commons-math3-3.1.1.jar +commons-net-3.1.jar +commons-pool-1.6.jar +core-3.1.1.jar +cron-utils-5.0.5.jar +curator-client-4.3.0.jar +curator-framework-4.3.0.jar +curator-recipes-4.3.0.jar +datanucleus-api-jdo-4.2.1.jar +datanucleus-core-4.1.6.jar +datanucleus-rdbms-4.1.7.jar +derby-10.14.2.0.jar +druid-1.1.14.jar +fastjson-1.2.61.jar +gson-2.8.5.jar +guava-20.0.jar +guice-3.0.jar +guice-servlet-3.0.jar +h2-1.4.200.jar +hadoop-annotations-2.7.3.jar +hadoop-auth-2.7.3.jar +hadoop-aws-2.7.3.jar +hadoop-client-2.7.3.jar +hadoop-common-2.7.3.jar +hadoop-hdfs-2.7.3.jar +hadoop-mapreduce-client-app-2.7.3.jar +hadoop-mapreduce-client-common-2.7.3.jar +hadoop-mapreduce-client-core-2.7.3.jar +hadoop-mapreduce-client-jobclient-2.7.3.jar +hadoop-mapreduce-client-shuffle-2.7.3.jar +hadoop-yarn-api-2.7.3.jar +hadoop-yarn-client-2.7.3.jar +hadoop-yarn-common-2.7.3.jar +hadoop-yarn-server-common-2.7.3.jar +hamcrest-core-1.3.jar +hibernate-validator-6.0.14.Final.jar +hive-common-2.1.0.jar +hive-jdbc-2.1.0.jar +hive-metastore-2.1.0.jar +hive-orc-2.1.0.jar +hive-serde-2.1.0.jar +hive-service-2.1.0.jar +hive-service-rpc-2.1.0.jar +hive-storage-api-2.1.0.jar +htrace-core-3.1.0-incubating.jar +httpclient-4.4.1.jar +httpcore-4.4.1.jar +httpmime-4.5.7.jar +jackson-annotations-2.9.8.jar +jackson-core-2.9.8.jar +jackson-core-asl-1.9.13.jar +jackson-databind-2.9.8.jar +jackson-datatype-jdk8-2.9.8.jar +jackson-datatype-jsr310-2.9.8.jar +jackson-jaxrs-1.9.13.jar +jackson-mapper-asl-1.9.13.jar +jackson-module-parameter-names-2.9.8.jar +jackson-xc-1.9.13.jar +jamon-runtime-2.3.1.jar +janino-3.0.12.jar +java-xmlbuilder-0.4.jar +javax.activation-api-1.2.0.jar +javax.annotation-api-1.3.2.jar +javax.inject-1.jar +javax.jdo-3.2.0-m3.jar +javax.mail-1.6.2.jar +javax.servlet-api-3.1.0.jar +javolution-5.5.1.jar +jaxb-api-2.3.1.jar +jaxb-impl-2.2.3-1.jar +jboss-logging-3.3.2.Final.jar +jdo-api-3.0.1.jar +jersey-client-1.9.jar +jersey-core-1.9.jar +jersey-guice-1.9.jar +jersey-json-1.9.jar +jersey-server-1.9.jar +jets3t-0.9.0.jar +jettison-1.1.jar +jetty-6.1.26.jar +jetty-continuation-9.4.14.v20181114.jar +jetty-http-9.4.14.v20181114.jar +jetty-io-9.4.14.v20181114.jar +jetty-security-9.4.14.v20181114.jar +jetty-server-9.4.14.v20181114.jar +jetty-servlet-9.4.14.v20181114.jar +jetty-servlets-9.4.14.v20181114.jar +jetty-util-6.1.26.jar +jetty-util-9.4.14.v20181114.jar +jetty-webapp-9.4.14.v20181114.jar +jetty-xml-9.4.14.v20181114.jar +jline-0.9.94.jar +jna-4.5.2.jar +jna-platform-4.5.2.jar +joda-time-2.10.1.jar +jpam-1.1.jar +jsch-0.1.42.jar +jsp-2.1-6.1.14.jar +jsp-api-2.1-6.1.14.jar +jsp-api-2.1.jar +jsqlparser-2.1.jar +jsr305-3.0.0.jar +jta-1.1.jar +jul-to-slf4j-1.7.25.jar +junit-4.12.jar +leveldbjni-all-1.8.jar +libfb303-0.9.3.jar +libthrift-0.9.3.jar +log4j-1.2-api-2.11.2.jar +log4j-1.2.17.jar +log4j-api-2.11.2.jar +log4j-core-2.11.2.jar +logback-classic-1.2.3.jar +logback-core-1.2.3.jar +lz4-1.3.0.jar +mapstruct-1.2.0.Final.jar +mssql-jdbc-6.1.0.jre8.jar +mybatis-3.5.2.jar +mybatis-plus-3.2.0.jar +mybatis-plus-annotation-3.2.0.jar +mybatis-plus-boot-starter-3.2.0.jar +mybatis-plus-core-3.2.0.jar +mybatis-plus-extension-3.2.0.jar +mybatis-spring-2.0.2.jar +netty-3.6.2.Final.jar +netty-all-4.1.33.Final.jar +opencsv-2.3.jar +oshi-core-3.5.0.jar +paranamer-2.3.jar +parquet-hadoop-bundle-1.8.1.jar +poi-3.17.jar +postgresql-42.1.4.jar +protobuf-java-2.5.0.jar +quartz-2.2.3.jar +quartz-jobs-2.2.3.jar +slf4j-api-1.7.5.jar +snakeyaml-1.23.jar +snappy-0.2.jar +snappy-java-1.0.4.1.jar +spring-aop-5.1.5.RELEASE.jar +spring-beans-5.1.5.RELEASE.jar +spring-boot-2.1.3.RELEASE.jar +spring-boot-autoconfigure-2.1.3.RELEASE.jar +spring-boot-starter-2.1.3.RELEASE.jar +spring-boot-starter-aop-2.1.3.RELEASE.jar +spring-boot-starter-jdbc-2.1.3.RELEASE.jar +spring-boot-starter-jetty-2.1.3.RELEASE.jar +spring-boot-starter-json-2.1.3.RELEASE.jar +spring-boot-starter-logging-2.1.3.RELEASE.jar +spring-boot-starter-web-2.1.3.RELEASE.jar +spring-context-5.1.5.RELEASE.jar +spring-core-5.1.5.RELEASE.jar +spring-expression-5.1.5.RELEASE.jar +spring-jcl-5.1.5.RELEASE.jar +spring-jdbc-5.1.5.RELEASE.jar +spring-plugin-core-1.2.0.RELEASE.jar +spring-plugin-metadata-1.2.0.RELEASE.jar +spring-tx-5.1.5.RELEASE.jar +spring-web-5.1.5.RELEASE.jar +spring-webmvc-5.1.5.RELEASE.jar +springfox-core-2.9.2.jar +springfox-schema-2.9.2.jar +springfox-spi-2.9.2.jar +springfox-spring-web-2.9.2.jar +springfox-swagger-common-2.9.2.jar +springfox-swagger-ui-2.9.2.jar +springfox-swagger2-2.9.2.jar +swagger-annotations-1.5.20.jar +swagger-bootstrap-ui-1.9.3.jar +swagger-models-1.5.20.jar +tephra-api-0.6.0.jar +threetenbp-1.3.6.jar +transaction-api-1.1.jar +validation-api-2.0.1.Final.jar +xercesImpl-2.9.1.jar +xml-apis-1.4.01.jar +xmlenc-0.52.jar +xz-1.0.jar +zookeeper-3.4.14.jar