diff --git a/.github/workflows/ci_backend.yml b/.github/workflows/ci_backend.yml index 0273251e99..c24a3ecf07 100644 --- a/.github/workflows/ci_backend.yml +++ b/.github/workflows/ci_backend.yml @@ -56,21 +56,9 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.8 + - name: Check license + run: ./mvnw -B apache-rat:check - name: Compile - run: mvn -B clean compile package -Prelease -Dmaven.test.skip=true - License-check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. - - name: checkout submodules - shell: bash - run: | - git submodule sync --recursive - git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - name: Check - run: mvn -B apache-rat:check + run: mvn -B clean compile install -Prelease -Dmaven.test.skip=true + - name: Check dependency license + run: tools/dependencies/check-LICENSE.sh diff --git a/.gitignore b/.gitignore index 6dd99201a9..17b0dc6610 100644 --- a/.gitignore +++ b/.gitignore @@ -4,27 +4,24 @@ .zip .gz .DS_Store -.idea .idea/ -.idea/* -.target -.target/ -**/**/target/** -target/* -*/target -*/target/* +dist/ +all-dependencies.txt +self-modules.txt +third-party-dependencies.txt +**/target/ .settings .nbproject .classpath .project -*.iml +**/*.iml *.ipr *.iws *.tgz .*.swp .vim .tmp -node_modules +**/node_modules npm-debug.log .vscode logs/* @@ -41,110 +38,10 @@ dolphinscheduler-alert/logs/ dolphinscheduler-alert/src/main/resources/alert.properties_bak dolphinscheduler-alert/src/main/resources/logback.xml dolphinscheduler-server/src/main/resources/logback.xml -dolphinscheduler-ui/dist +dolphinscheduler-ui/dist/ dolphinscheduler-ui/node -dolphinscheduler-ui/dist/css/common.16ac5d9.css -dolphinscheduler-ui/dist/css/home/index.b444b91.css -dolphinscheduler-ui/dist/css/login/index.5866c64.css -dolphinscheduler-ui/dist/js/0.ac94e5d.js -dolphinscheduler-ui/dist/js/0.ac94e5d.js.map -dolphinscheduler-ui/dist/js/1.0b043a3.js -dolphinscheduler-ui/dist/js/1.0b043a3.js.map -dolphinscheduler-ui/dist/js/10.1bce3dc.js -dolphinscheduler-ui/dist/js/10.1bce3dc.js.map -dolphinscheduler-ui/dist/js/11.79f04d8.js -dolphinscheduler-ui/dist/js/11.79f04d8.js.map -dolphinscheduler-ui/dist/js/12.420daa5.js -dolphinscheduler-ui/dist/js/12.420daa5.js.map -dolphinscheduler-ui/dist/js/13.e5bae1c.js -dolphinscheduler-ui/dist/js/13.e5bae1c.js.map -dolphinscheduler-ui/dist/js/14.f2a0dca.js -dolphinscheduler-ui/dist/js/14.f2a0dca.js.map -dolphinscheduler-ui/dist/js/15.45373e8.js -dolphinscheduler-ui/dist/js/15.45373e8.js.map -dolphinscheduler-ui/dist/js/16.fecb0fc.js -dolphinscheduler-ui/dist/js/16.fecb0fc.js.map -dolphinscheduler-ui/dist/js/17.84be279.js -dolphinscheduler-ui/dist/js/17.84be279.js.map -dolphinscheduler-ui/dist/js/18.307ea70.js -dolphinscheduler-ui/dist/js/18.307ea70.js.map -dolphinscheduler-ui/dist/js/19.144db9c.js -dolphinscheduler-ui/dist/js/19.144db9c.js.map -dolphinscheduler-ui/dist/js/2.8b4ef29.js -dolphinscheduler-ui/dist/js/2.8b4ef29.js.map -dolphinscheduler-ui/dist/js/20.4c527e9.js -dolphinscheduler-ui/dist/js/20.4c527e9.js.map -dolphinscheduler-ui/dist/js/21.831b2a2.js -dolphinscheduler-ui/dist/js/21.831b2a2.js.map -dolphinscheduler-ui/dist/js/22.2b4bb2a.js -dolphinscheduler-ui/dist/js/22.2b4bb2a.js.map -dolphinscheduler-ui/dist/js/23.81467ef.js -dolphinscheduler-ui/dist/js/23.81467ef.js.map -dolphinscheduler-ui/dist/js/24.54a00e4.js -dolphinscheduler-ui/dist/js/24.54a00e4.js.map -dolphinscheduler-ui/dist/js/25.8d7bd36.js -dolphinscheduler-ui/dist/js/25.8d7bd36.js.map -dolphinscheduler-ui/dist/js/26.2ec5e78.js -dolphinscheduler-ui/dist/js/26.2ec5e78.js.map -dolphinscheduler-ui/dist/js/27.3ab48c2.js -dolphinscheduler-ui/dist/js/27.3ab48c2.js.map -dolphinscheduler-ui/dist/js/28.363088a.js -dolphinscheduler-ui/dist/js/28.363088a.js.map -dolphinscheduler-ui/dist/js/29.6c5853a.js -dolphinscheduler-ui/dist/js/29.6c5853a.js.map -dolphinscheduler-ui/dist/js/3.a0edb5b.js -dolphinscheduler-ui/dist/js/3.a0edb5b.js.map -dolphinscheduler-ui/dist/js/30.940fdd3.js -dolphinscheduler-ui/dist/js/30.940fdd3.js.map -dolphinscheduler-ui/dist/js/31.168a460.js -dolphinscheduler-ui/dist/js/31.168a460.js.map -dolphinscheduler-ui/dist/js/32.8df6594.js -dolphinscheduler-ui/dist/js/32.8df6594.js.map -dolphinscheduler-ui/dist/js/33.4480bbe.js -dolphinscheduler-ui/dist/js/33.4480bbe.js.map -dolphinscheduler-ui/dist/js/34.b407fe1.js -dolphinscheduler-ui/dist/js/34.b407fe1.js.map -dolphinscheduler-ui/dist/js/35.f340b0a.js -dolphinscheduler-ui/dist/js/35.f340b0a.js.map -dolphinscheduler-ui/dist/js/36.8880c2d.js -dolphinscheduler-ui/dist/js/36.8880c2d.js.map -dolphinscheduler-ui/dist/js/37.ea2a25d.js -dolphinscheduler-ui/dist/js/37.ea2a25d.js.map -dolphinscheduler-ui/dist/js/38.98a59ee.js -dolphinscheduler-ui/dist/js/38.98a59ee.js.map -dolphinscheduler-ui/dist/js/39.a5e958a.js -dolphinscheduler-ui/dist/js/39.a5e958a.js.map -dolphinscheduler-ui/dist/js/4.4ca44db.js -dolphinscheduler-ui/dist/js/4.4ca44db.js.map -dolphinscheduler-ui/dist/js/40.e187b1e.js -dolphinscheduler-ui/dist/js/40.e187b1e.js.map -dolphinscheduler-ui/dist/js/41.0e89182.js -dolphinscheduler-ui/dist/js/41.0e89182.js.map -dolphinscheduler-ui/dist/js/42.341047c.js -dolphinscheduler-ui/dist/js/42.341047c.js.map -dolphinscheduler-ui/dist/js/43.27b8228.js -dolphinscheduler-ui/dist/js/43.27b8228.js.map -dolphinscheduler-ui/dist/js/44.e8869bc.js -dolphinscheduler-ui/dist/js/44.e8869bc.js.map -dolphinscheduler-ui/dist/js/45.8d54901.js -dolphinscheduler-ui/dist/js/45.8d54901.js.map -dolphinscheduler-ui/dist/js/5.e1ed7f3.js -dolphinscheduler-ui/dist/js/5.e1ed7f3.js.map -dolphinscheduler-ui/dist/js/6.241ba07.js -dolphinscheduler-ui/dist/js/6.241ba07.js.map -dolphinscheduler-ui/dist/js/7.ab2e297.js -dolphinscheduler-ui/dist/js/7.ab2e297.js.map -dolphinscheduler-ui/dist/js/8.83ff814.js -dolphinscheduler-ui/dist/js/8.83ff814.js.map -dolphinscheduler-ui/dist/js/9.39cb29f.js -dolphinscheduler-ui/dist/js/9.39cb29f.js.map -dolphinscheduler-ui/dist/js/common.733e342.js -dolphinscheduler-ui/dist/js/common.733e342.js.map -dolphinscheduler-ui/dist/js/home/index.78a5d12.js -dolphinscheduler-ui/dist/js/home/index.78a5d12.js.map -dolphinscheduler-ui/dist/js/login/index.291b8e3.js -dolphinscheduler-ui/dist/js/login/index.291b8e3.js.map -dolphinscheduler-ui/dist/lib/external/ -/dolphinscheduler-dao/src/main/resources/dao/data_source.properties +dolphinscheduler-dao/src/main/resources/dao/data_source.properties + +.mvn/wrapper/*.jar !/zookeeper_data/ diff --git a/ambari_plugin/README.md b/ambari_plugin/README.md new file mode 100644 index 0000000000..bd634d4a29 --- /dev/null +++ b/ambari_plugin/README.md @@ -0,0 +1,68 @@ +### Dolphin Scheduler的Ambari插件使用说明 + +##### 备注 + +1. 本文档适用于对Ambari中基本了解的用户 +2. 本文档是对已安装Ambari服务添加Dolphin Scheduler(1.3.0版本)服务的说明 + +##### 一 安装准备 + +1. 准备RPM包 + + - 在源码dolphinscheduler-dist目录下执行命令```mvn -U clean install rpm:attached-rpm -Prpmbuild -Dmaven.test.skip=true -X```即可生成(在目录 dolphinscheduler-dist/target/rpm/apache-dolphinscheduler-incubating/RPMS/noarch 下) + +2. 创建DS的安装用户--权限 + +3. 初始化数据库信息 + + ``` + -- 创建Dolphin Scheduler的数据库：dolphinscheduler + CREATE DATABASE dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE + utf8_general_ci; + + -- 初始化dolphinscheduler数据库的用户和密码，并分配权限 + -- 替换下面sql语句中的{user}为dolphinscheduler数据库的用户 + GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'%' IDENTIFIED BY '{password}'; + GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'localhost' IDENTIFIED BY + '{password}'; + flush privileges; + ``` + + + +##### 二 Ambari安装Dolphin Scheduler + +1. Ambari界面安装Dolphin Scheduler + + ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_001.png) + +2. 选择Dolphin Scheduler的Master安装的节点 + + ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_002.png) + +3. 配置Dolphin Scheduler的Worker、Api、Logger、Alert安装的节点 + + ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_003.png) + +4. 设置Dolphin Scheduler服务的安装用户（**步骤一中创建的**）及所属的用户组 + + ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_004.png) + +5. 配置数据库的信息（和步骤一中初始化数据库中一致） + + ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_005.png) + +6. 配置其它的信息--如果需要的话 + + ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_006.png) + + ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_007.png) + +7. 正常执行接下来的步骤 + + ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_008.png) + +8. 安装成功后的界面 + + ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_009.png) + diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/alerts.json b/ambari_plugin/common-services/DOLPHIN/2.0.0/alerts.json deleted file mode 100644 index 385c5d5599..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/alerts.json +++ /dev/null @@ -1,164 +0,0 @@ -{ - "DOLPHIN": { - "service": [], - "DOLPHIN_API": [ - { - "name": "dolphin_api_port_check", - "label": "dolphin_api_port_check", - "description": "dolphin_api_port_check.", - "interval": 10, - "scope": "ANY", - "source": { - "type": "PORT", - "uri": "{{dolphin-application-api/server.port}}", - "default_port": 12345, - "reporting": { - "ok": { - "text": "TCP OK - {0:.3f}s response on port {1}" - }, - "warning": { - "text": "TCP OK - {0:.3f}s response on port {1}", - "value": 1.5 - }, - "critical": { - "text": "Connection failed: {0} to {1}:{2}", - "value": 5.0 - } - } - } - } - ], - "DOLPHIN_MASTER": [ - { - "name": "DOLPHIN_MASTER_CHECK", - "label": "check dolphin scheduler master status", - "description": "", - "interval":10, - "scope": "HOST", - "enabled": true, - "source": { - "type": "SCRIPT", - "path": "DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py", - "parameters": [ - - { - "name": "connection.timeout", - "display_name": "Connection Timeout", - "value": 5.0, - "type": "NUMERIC", - "description": "The maximum time before this alert is considered to be CRITICAL", - "units": "seconds", - "threshold": "CRITICAL" - }, - { - "name": "alertName", - "display_name": "alertName", - "value": "DOLPHIN_MASTER", - "type": "STRING", - "description": "alert name" - } - ] - } - } - ], - "DOLPHIN_WORKER": [ - { - "name": "DOLPHIN_WORKER_CHECK", - "label": "check dolphin scheduler worker status", - "description": "", - "interval":10, - "scope": "HOST", - "enabled": true, - "source": { - "type": "SCRIPT", - "path": "DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py", - "parameters": [ - - { - "name": "connection.timeout", - "display_name": "Connection Timeout", - "value": 5.0, - "type": "NUMERIC", - "description": "The maximum time before this alert is considered to be CRITICAL", - "units": "seconds", - "threshold": "CRITICAL" - }, - { - "name": "alertName", - "display_name": "alertName", - "value": "DOLPHIN_WORKER", - "type": "STRING", - "description": "alert name" - } - ] - } - } - ], - "DOLPHIN_ALERT": [ - { - "name": "DOLPHIN_DOLPHIN_ALERT_CHECK", - "label": "check dolphin scheduler alert status", - "description": "", - "interval":10, - "scope": "HOST", - "enabled": true, - "source": { - "type": "SCRIPT", - "path": "DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py", - "parameters": [ - - { - "name": "connection.timeout", - "display_name": "Connection Timeout", - "value": 5.0, - "type": "NUMERIC", - "description": "The maximum time before this alert is considered to be CRITICAL", - "units": "seconds", - "threshold": "CRITICAL" - }, - { - "name": "alertName", - "display_name": "alertName", - "value": "DOLPHIN_ALERT", - "type": "STRING", - "description": "alert name" - } - ] - } - } - ], - "DOLPHIN_ALERT": [ - { - "name": "DOLPHIN_DOLPHIN_LOGGER_CHECK", - "label": "check dolphin scheduler alert status", - "description": "", - "interval":10, - "scope": "HOST", - "enabled": true, - "source": { - "type": "SCRIPT", - "path": "DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py", - "parameters": [ - - { - "name": "connection.timeout", - "display_name": "Connection Timeout", - "value": 5.0, - "type": "NUMERIC", - "description": "The maximum time before this alert is considered to be CRITICAL", - "units": "seconds", - "threshold": "CRITICAL" - }, - { - "name": "alertName", - "display_name": "alertName", - "value": "DOLPHIN_LOGGER", - "type": "STRING", - "description": "alert name" - } - ] - } - } - ] - } -} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-alert.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-alert.xml deleted file mode 100644 index 5f44a1a4c8..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-alert.xml +++ /dev/null @@ -1,143 +0,0 @@ - - - - alert.type - EMAIL - alert type is EMAIL/SMS - - - - alert.template - html - alter msg template, default is html template - - - - mail.protocol - SMTP - - - - - mail.server.host - xxx.xxx.com - - - - - mail.server.port - 25 - - int - - - - - - mail.sender - admin - - - - - mail.user - admin - - - - - mail.passwd - 000000 - - PASSWORD - - password - - - - - - mail.smtp.starttls.enable - true - - boolean - - - - - - mail.smtp.ssl.enable - true - - boolean - - - - - - mail.smtp.ssl.trust - xxx.xxx.com - - - - - - enterprise.wechat.enable - false - - - value-list - - - true - - - - false - - - - 1 - - - - - enterprise.wechat.corp.id - wechatId - - - - - enterprise.wechat.secret - secret - - - - - enterprise.wechat.agent.id - agentId - - - - - enterprise.wechat.users - wechatUsers - - - - \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-application-api.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-application-api.xml deleted file mode 100644 index 766c0f477d..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-application-api.xml +++ /dev/null @@ -1,87 +0,0 @@ - - - - server.port - 12345 - - server port - - - int - - - - server.servlet.session.timeout - 7200 - - int - - - - - - server.servlet.context-path - /dolphinscheduler/ - - - - - spring.servlet.multipart.max-file-size - 1024 - - MB - int - - - - - - spring.servlet.multipart.max-request-size - 1024 - - MB - int - - - - - - server.jetty.max-http-post-size - 5000000 - - int - - - - - - spring.messages.encoding - UTF-8 - - - - spring.messages.basename - i18n/messages - - - - security.authentication.type - PASSWORD - - - \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-common.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-common.xml deleted file mode 100644 index 439e21188a..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-common.xml +++ /dev/null @@ -1,158 +0,0 @@ - - - - resource.storage.type - Choose Resource Upload Startup Type - - Resource upload startup type : HDFS,S3,NONE - - NONE - - value-list - - - HDFS - - - - S3 - - - - NONE - - - - 1 - - - - - resource.upload.path - /dolphinscheduler - - resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended - - - - - data.basedir.path - /tmp/dolphinscheduler - - user data local directory path, please make sure the directory exists and have read write permissions - - - - - - hadoop.security.authentication.startup.state - false - - value-list - - - true - - - - false - - - - 1 - - whether kerberos starts - - - java.security.krb5.conf.path - /opt/krb5.conf - - java.security.krb5.conf path - - - - - login.user.keytab.username - hdfs-mycluster@ESZ.COM - - LoginUserFromKeytab user - - - - - login.user.keytab.path - /opt/hdfs.headless.keytab - - LoginUserFromKeytab path - - - - - resource.view.suffixs - txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties - - - - hdfs.root.user - hdfs - - Users who have permission to create directories under the HDFS root path - - - - - fs.defaultFS - hdfs://mycluster:8020 - - HA or single namenode, - If namenode ha needs to copy core-site.xml and hdfs-site.xml to the conf directory， - support s3，for example : s3a://dolphinscheduler - - - - - fs.s3a.endpoint - http://host:9010 - - s3 need，s3 endpoint - - - - - fs.s3a.access.key - A3DXS30FO22544RE - - s3 need，s3 access key - - - - - fs.s3a.secret.key - OloCLq3n+8+sdPHUhJ21XrSxTC+JK - - s3 need，s3 secret key - - - - - kerberos.expire.time - 7 - - - - \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-datasource.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-datasource.xml deleted file mode 100644 index 6e50a1b649..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-datasource.xml +++ /dev/null @@ -1,467 +0,0 @@ - - - - spring.datasource.initialSize - 5 - - Init connection number - - - int - - - - - spring.datasource.minIdle - 5 - - Min connection number - - - int - - - - - spring.datasource.maxActive - 50 - - Max connection number - - - int - - - - - spring.datasource.maxWait - 60000 - - Max wait time for get a connection in milliseconds. - If configuring maxWait, fair locks are enabled by default and concurrency efficiency decreases. - If necessary, unfair locks can be used by configuring the useUnfairLock attribute to true. - - - int - - - - - spring.datasource.timeBetweenEvictionRunsMillis - 60000 - - Milliseconds for check to close free connections - - - int - - - - - spring.datasource.timeBetweenConnectErrorMillis - 60000 - - The Destroy thread detects the connection interval and closes the physical connection in milliseconds - if the connection idle time is greater than or equal to minEvictableIdleTimeMillis. - - - int - - - - - spring.datasource.minEvictableIdleTimeMillis - 300000 - - The longest time a connection remains idle without being evicted, in milliseconds - - - int - - - - - spring.datasource.validationQuery - SELECT 1 - - The SQL used to check whether the connection is valid requires a query statement. - If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work. - - - - - spring.datasource.validationQueryTimeout - 3 - - int - - - Check whether the connection is valid for timeout, in seconds - - - - - spring.datasource.testWhileIdle - true - - boolean - - - When applying for a connection, - if it is detected that the connection is idle longer than time Between Eviction Runs Millis, - validation Query is performed to check whether the connection is valid - - - - - spring.datasource.testOnBorrow - true - - boolean - - - Execute validation to check if the connection is valid when applying for a connection - - - - - spring.datasource.testOnReturn - false - - boolean - - - Execute validation to check if the connection is valid when the connection is returned - - - - - spring.datasource.defaultAutoCommit - true - - boolean - - - - - - - spring.datasource.keepAlive - false - - boolean - - - - - - - - spring.datasource.poolPreparedStatements - true - - boolean - - - Open PSCache, specify count PSCache for every connection - - - - - spring.datasource.maxPoolPreparedStatementPerConnectionSize - 20 - - int - - - - - - spring.datasource.spring.datasource.filters - stat,wall,log4j - - - - - spring.datasource.connectionProperties - druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000 - - - - - - mybatis-plus.mapper-locations - classpath*:/org.apache.dolphinscheduler.dao.mapper/*.xml - - - - - mybatis-plus.typeEnumsPackage - org.apache.dolphinscheduler.*.enums - - - - - mybatis-plus.typeAliasesPackage - org.apache.dolphinscheduler.dao.entity - - Entity scan, where multiple packages are separated by a comma or semicolon - - - - - mybatis-plus.global-config.db-config.id-type - AUTO - - value-list - - - AUTO - - - - INPUT - - - - ID_WORKER - - - - UUID - - - - 1 - - - Primary key type AUTO:" database ID AUTO ", - INPUT:" user INPUT ID", - ID_WORKER:" global unique ID (numeric type unique ID)", - UUID:" global unique ID UUID"; - - - - - mybatis-plus.global-config.db-config.field-strategy - NOT_NULL - - value-list - - - IGNORED - - - - NOT_NULL - - - - NOT_EMPTY - - - - 1 - - - Field policy IGNORED:" ignore judgment ", - NOT_NULL:" not NULL judgment "), - NOT_EMPTY:" not NULL judgment" - - - - - mybatis-plus.global-config.db-config.column-underline - true - - boolean - - - - - - mybatis-plus.global-config.db-config.logic-delete-value - 1 - - int - - - - - - mybatis-plus.global-config.db-config.logic-not-delete-value - 0 - - int - - - - - - mybatis-plus.global-config.db-config.banner - true - - boolean - - - - - - - mybatis-plus.configuration.map-underscore-to-camel-case - true - - boolean - - - - - - mybatis-plus.configuration.cache-enabled - false - - boolean - - - - - - mybatis-plus.configuration.call-setters-on-nulls - true - - boolean - - - - - - mybatis-plus.configuration.jdbc-type-for-null - null - - - - - master.exec.threads - 100 - - int - - - - - - master.exec.task.num - 20 - - int - - - - - - master.heartbeat.interval - 10 - - int - - - - - - master.task.commit.retryTimes - 5 - - int - - - - - - master.task.commit.interval - 1000 - - int - - - - - - master.max.cpuload.avg - 100 - - int - - - - - - master.reserved.memory - 0.1 - - float - - - - - - worker.exec.threads - 100 - - int - - - - - - worker.heartbeat.interval - 10 - - int - - - - - - worker.fetch.task.num - 3 - - int - - - - - - worker.max.cpuload.avg - 100 - - int - - - - - - worker.reserved.memory - 0.1 - - float - - - - - - \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-env.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-env.xml deleted file mode 100644 index 8e14716d05..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-env.xml +++ /dev/null @@ -1,123 +0,0 @@ - - - - dolphin.database.type - mysql - Dolphin Scheduler DataBase Type Which Is Select - Dolphin Database Type - - value-list - - - mysql - - - - postgresql - - - - 1 - - - - - - dolphin.database.host - - Dolphin Database Host - - - - - dolphin.database.port - - Dolphin Database Port - - - - - dolphin.database.username - - Dolphin Database Username - - - - - dolphin.database.password - - Dolphin Database Password - PASSWORD - - password - - - - - - dolphin.user - - Which user to install and admin dolphin scheduler - Deploy User - - - - dolphin.group - - Which user to install and admin dolphin scheduler - Deploy Group - - - - - dolphinscheduler-env-content - Dolphinscheduler Env template - This is the jinja template for dolphinscheduler.env.sh file - # -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -export HADOOP_HOME=/opt/soft/hadoop -export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop -export SPARK_HOME1=/opt/soft/spark1 -export SPARK_HOME2=/opt/soft/spark2 -export PYTHON_HOME=/opt/soft/python -export JAVA_HOME=/opt/soft/java -export HIVE_HOME=/opt/soft/hive -export FLINK_HOME=/opt/soft/flink - - content - false - false - - - - \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-master.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-master.xml deleted file mode 100644 index c8eec047fc..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-master.xml +++ /dev/null @@ -1,88 +0,0 @@ - - - - master.exec.threads - 100 - - int - - master execute thread num - - - - master.exec.task.num - 20 - - int - - master execute task number in parallel - - - - master.heartbeat.interval - 10 - - int - - master heartbeat interval - - - - master.task.commit.retryTimes - 5 - - int - - master commit task retry times - - - - master.task.commit.interval - 1000 - - int - - master commit task interval - - - - master.max.cpuload.avg - 100 - - int - - only less than cpu avg load, master server can work. default value : the number of cpu cores * 2 - - - - master.reserved.memory - 0.3 - only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G. - - - - - master.listen.port - 5678 - - int - - master listen port - - - \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-quartz.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-quartz.xml deleted file mode 100644 index 7a0c68b051..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-quartz.xml +++ /dev/null @@ -1,126 +0,0 @@ - - - - org.quartz.scheduler.instanceName - DolphinScheduler - - - - - org.quartz.scheduler.instanceId - AUTO - - - - org.quartz.scheduler.makeSchedulerThreadDaemon - true - - boolean - - - - - org.quartz.jobStore.useProperties - false - - boolean - - - - - org.quartz.threadPool.class - org.quartz.simpl.SimpleThreadPool - - - - org.quartz.threadPool.makeThreadsDaemons - true - - boolean - - - - - org.quartz.threadPool.threadCount - 25 - - int - - - - - org.quartz.threadPool.threadPriority - 5 - - int - - - - - org.quartz.jobStore.class - org.quartz.impl.jdbcjobstore.JobStoreTX - - - - org.quartz.jobStore.tablePrefix - QRTZ_ - - - - org.quartz.jobStore.isClustered - true - - boolean - - - - - org.quartz.jobStore.misfireThreshold - 60000 - - int - - - - - org.quartz.jobStore.clusterCheckinInterval - 5000 - - int - - - - - org.quartz.jobStore.acquireTriggersWithinLock - true - - boolean - - - - - org.quartz.jobStore.dataSource - myDs - - - - org.quartz.dataSource.myDs.connectionProvider.class - org.apache.dolphinscheduler.service.quartz.DruidConnectionProvider - - - \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-worker.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-worker.xml deleted file mode 100644 index 97beade1bc..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-worker.xml +++ /dev/null @@ -1,76 +0,0 @@ - - - - worker.exec.threads - 100 - - int - - worker execute thread num - - - - worker.heartbeat.interval - 10 - - int - - worker heartbeat interval - - - - worker.fetch.task.num - 3 - - int - - submit the number of tasks at a time - - - - worker.max.cpuload.avg - 100 - - int - - only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2 - - - - worker.reserved.memory - 0.3 - only larger than reserved memory, worker server can work. default value : physical memory * 1/10, unit is G. - - - - - worker.listen.port - 1234 - - int - - worker listen port - - - - worker.group - default - default worker group - - - \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-zookeeper.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-zookeeper.xml deleted file mode 100644 index 5882162254..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-zookeeper.xml +++ /dev/null @@ -1,84 +0,0 @@ - - - - dolphinscheduler.queue.impl - zookeeper - - Task queue implementation, default "zookeeper" - - - - - zookeeper.dolphinscheduler.root - /dolphinscheduler - - dolphinscheduler root directory - - - - - zookeeper.session.timeout - 300 - - int - - - - - - - zookeeper.connection.timeout - 300 - - int - - - - - - - zookeeper.retry.base.sleep - 100 - - int - - - - - - - zookeeper.retry.max.sleep - 30000 - - int - - - - - - - zookeeper.retry.maxtime - 5 - - int - - - - - - \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/metainfo.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/metainfo.xml deleted file mode 100644 index b3c14e33cb..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/metainfo.xml +++ /dev/null @@ -1,137 +0,0 @@ - - - - 2.0 - - - DOLPHIN - Dolphin Scheduler - 分布式易扩展的可视化DAG工作流任务调度系统 - 2.0.0 - - - DOLPHIN_MASTER - DS Master - MASTER - 1+ - - - PYTHON - 600 - - - - - DOLPHIN_LOGGER - DS Logger - SLAVE - 1+ - - - PYTHON - 600 - - - - - DOLPHIN_WORKER - DS Worker - SLAVE - 1+ - - - DOLPHIN/DOLPHIN_LOGGER - host - - true - - - - - - PYTHON - 600 - - - - - DOLPHIN_ALERT - DS Alert - SLAVE - 1 - - - PYTHON - 600 - - - - - DOLPHIN_API - DS_Api - SLAVE - 1 - - - PYTHON - 600 - - - - - - ZOOKEEPER - - - - - any - - - apache-dolphinscheduler-incubating-1.2.1* - - - - - - - dolphin-alert - dolphin-app-api - dolphin-app-dao - dolphin-common - dolphin-env - dolphin-quartz - - - - - theme.json - true - - - - quicklinks - - - quicklinks.json - true - - - - - diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py deleted file mode 100644 index 87cc7b453b..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py +++ /dev/null @@ -1,124 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import socket -import urllib2 -import os -import logging -import ambari_simplejson as json -from resource_management.libraries.script.script import Script -import sys -reload(sys) -sys.setdefaultencoding('utf-8') - -logger = logging.getLogger('ambari_alerts') - -config = Script.get_config() - - -def get_tokens(): - """ - Returns a tuple of tokens in the format {{site/property}} that will be used - to build the dictionary passed into execute - - :rtype tuple - """ - -def get_info(url, connection_timeout): - response = None - - try: - response = urllib2.urlopen(url, timeout=connection_timeout) - json_data = response.read() - return json_data - finally: - if response is not None: - try: - response.close() - except: - pass - - -def execute(configurations={}, parameters={}, host_name=None): - """ - Returns a tuple containing the result code and a pre-formatted result label - - Keyword arguments: - configurations : a mapping of configuration key to value - parameters : a mapping of script parameter key to value - host_name : the name of this host where the alert is running - - :type configurations dict - :type parameters dict - :type host_name str - """ - - alert_name = parameters['alertName'] - - dolphin_pidfile_dir = "/opt/soft/run/dolphinscheduler" - - pid = "0" - - - from resource_management.core import sudo - - is_running = True - pid_file_path = "" - if alert_name == 'DOLPHIN_MASTER': - pid_file_path = dolphin_pidfile_dir + "/master-server.pid" - elif alert_name == 'DOLPHIN_WORKER': - pid_file_path = dolphin_pidfile_dir + "/worker-server.pid" - elif alert_name == 'DOLPHIN_ALERT': - pid_file_path = dolphin_pidfile_dir + "/alert-server.pid" - elif alert_name == 'DOLPHIN_LOGGER': - pid_file_path = dolphin_pidfile_dir + "/logger-server.pid" - elif alert_name == 'DOLPHIN_API': - pid_file_path = dolphin_pidfile_dir + "/api-server.pid" - - if not pid_file_path or not os.path.isfile(pid_file_path): - is_running = False - - try: - pid = int(sudo.read_file(pid_file_path)) - except: - is_running = False - - try: - # Kill will not actually kill the process - # From the doc: - # If sig is 0, then no signal is sent, but error checking is still - # performed; this can be used to check for the existence of a - # process ID or process group ID. - sudo.kill(pid, 0) - except OSError: - is_running = False - - if host_name is None: - host_name = socket.getfqdn() - - if not is_running: - result_code = "CRITICAL" - else: - result_code = "OK" - - label = "The comment {0} of DOLPHIN_SCHEDULER on {1} is {2}".format(alert_name, host_name, result_code) - - return ((result_code, [label])) - -if __name__ == "__main__": - pass diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_alert_service.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_alert_service.py deleted file mode 100644 index 62255a3432..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_alert_service.py +++ /dev/null @@ -1,61 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import time -from resource_management import * - -from dolphin_env import dolphin_env - - -class DolphinAlertService(Script): - def install(self, env): - import params - env.set_params(params) - self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) - - def configure(self, env): - import params - params.pika_slave = True - env.set_params(params) - - dolphin_env() - - def start(self, env): - import params - env.set_params(params) - self.configure(env) - no_op_test = format("ls {dolphin_pidfile_dir}/alert-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/alert-server.pid` | grep `cat {dolphin_pidfile_dir}/alert-server.pid` >/dev/null 2>&1") - - start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start alert-server") - Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) - - def stop(self, env): - import params - env.set_params(params) - stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop alert-server") - Execute(stop_cmd, user=params.dolphin_user) - time.sleep(5) - - def status(self, env): - import status_params - env.set_params(status_params) - check_process_status(status_params.dolphin_run_dir + "alert-server.pid") - - -if __name__ == "__main__": - DolphinAlertService().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_api_service.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_api_service.py deleted file mode 100644 index bdc18fb602..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_api_service.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import time -from resource_management import * - -from dolphin_env import dolphin_env - - -class DolphinApiService(Script): - def install(self, env): - import params - env.set_params(params) - self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) - - def configure(self, env): - import params - params.pika_slave = True - env.set_params(params) - - dolphin_env() - - def start(self, env): - import params - env.set_params(params) - self.configure(env) - - #init - init_cmd=format("sh " + params.dolphin_home + "/script/create-dolphinscheduler.sh") - Execute(init_cmd, user=params.dolphin_user) - - #upgrade - upgrade_cmd=format("sh " + params.dolphin_home + "/script/upgrade-dolphinscheduler.sh") - Execute(upgrade_cmd, user=params.dolphin_user) - - no_op_test = format("ls {dolphin_pidfile_dir}/api-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/api-server.pid` | grep `cat {dolphin_pidfile_dir}/api-server.pid` >/dev/null 2>&1") - - start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start api-server") - Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) - - def stop(self, env): - import params - env.set_params(params) - stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop api-server") - Execute(stop_cmd, user=params.dolphin_user) - time.sleep(5) - - def status(self, env): - import status_params - env.set_params(status_params) - check_process_status(status_params.dolphin_run_dir + "api-server.pid") - - -if __name__ == "__main__": - DolphinApiService().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_env.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_env.py deleted file mode 100644 index 1661d76c75..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_env.py +++ /dev/null @@ -1,123 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management import * - - -def dolphin_env(): - import params - - Directory(params.dolphin_pidfile_dir, - mode=0777, - owner=params.dolphin_user, - group=params.dolphin_group, - create_parents=True - ) - Directory(params.dolphin_log_dir, - mode=0777, - owner=params.dolphin_user, - group=params.dolphin_group, - create_parents=True - ) - Directory(params.dolphin_conf_dir, - mode=0777, - owner=params.dolphin_user, - group=params.dolphin_group, - create_parents=True - ) - - Directory(params.dolphin_common_map['data.basedir.path'], - mode=0777, - owner=params.dolphin_user, - group=params.dolphin_group, - create_parents=True - ) - - - File(format(params.dolphin_env_path), - mode=0777, - content=InlineTemplate(params.dolphin_env_content), - owner=params.dolphin_user, - group=params.dolphin_group - ) - - - File(format(params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh"), - mode=0755, - content=Template("dolphin-daemon.sh.j2"), - owner=params.dolphin_user, - group=params.dolphin_group - ) - - File(format(params.dolphin_conf_dir + "/master.properties"), - mode=0755, - content=Template("master.properties.j2"), - owner=params.dolphin_user, - group=params.dolphin_group - ) - - File(format(params.dolphin_conf_dir + "/worker.properties"), - mode=0755, - content=Template("worker.properties.j2"), - owner=params.dolphin_user, - group=params.dolphin_group - ) - - - File(format(params.dolphin_conf_dir + "/alert.properties"), - mode=0755, - content=Template("alert.properties.j2"), - owner=params.dolphin_user, - group=params.dolphin_group - ) - - File(format(params.dolphin_conf_dir + "/datasource.properties"), - mode=0755, - content=Template("datasource.properties.j2"), - owner=params.dolphin_user, - group=params.dolphin_group - ) - - File(format(params.dolphin_conf_dir + "/application-api.properties"), - mode=0755, - content=Template("application-api.properties.j2"), - owner=params.dolphin_user, - group=params.dolphin_group - ) - - File(format(params.dolphin_conf_dir + "/common.properties"), - mode=0755, - content=Template("common.properties.j2"), - owner=params.dolphin_user, - group=params.dolphin_group - ) - - File(format(params.dolphin_conf_dir + "/quartz.properties"), - mode=0755, - content=Template("quartz.properties.j2"), - owner=params.dolphin_user, - group=params.dolphin_group - ) - - File(format(params.dolphin_conf_dir + "/zookeeper.properties"), - mode=0755, - content=Template("zookeeper.properties.j2"), - owner=params.dolphin_user, - group=params.dolphin_group - ) diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_logger_service.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_logger_service.py deleted file mode 100644 index f1c19bd66f..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_logger_service.py +++ /dev/null @@ -1,61 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import time -from resource_management import * - -from dolphin_env import dolphin_env - - -class DolphinLoggerService(Script): - def install(self, env): - import params - env.set_params(params) - self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) - - def configure(self, env): - import params - params.pika_slave = True - env.set_params(params) - - dolphin_env() - - def start(self, env): - import params - env.set_params(params) - self.configure(env) - no_op_test = format("ls {dolphin_pidfile_dir}/logger-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/logger-server.pid` | grep `cat {dolphin_pidfile_dir}/logger-server.pid` >/dev/null 2>&1") - - start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start logger-server") - Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) - - def stop(self, env): - import params - env.set_params(params) - stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop logger-server") - Execute(stop_cmd, user=params.dolphin_user) - time.sleep(5) - - def status(self, env): - import status_params - env.set_params(status_params) - check_process_status(status_params.dolphin_run_dir + "logger-server.pid") - - -if __name__ == "__main__": - DolphinLoggerService().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_master_service.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_master_service.py deleted file mode 100644 index 6ee7ecfcf3..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_master_service.py +++ /dev/null @@ -1,61 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import time -from resource_management import * - -from dolphin_env import dolphin_env - - -class DolphinMasterService(Script): - def install(self, env): - import params - env.set_params(params) - self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) - - def configure(self, env): - import params - params.pika_slave = True - env.set_params(params) - - dolphin_env() - - def start(self, env): - import params - env.set_params(params) - self.configure(env) - no_op_test = format("ls {dolphin_pidfile_dir}/master-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/master-server.pid` | grep `cat {dolphin_pidfile_dir}/master-server.pid` >/dev/null 2>&1") - start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start master-server") - Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) - - def stop(self, env): - import params - env.set_params(params) - stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop master-server") - Execute(stop_cmd, user=params.dolphin_user) - time.sleep(5) - - def status(self, env): - import status_params - env.set_params(status_params) - check_process_status(status_params.dolphin_run_dir + "master-server.pid") - - -if __name__ == "__main__": - DolphinMasterService().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_worker_service.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_worker_service.py deleted file mode 100644 index 2d145ee730..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_worker_service.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import time -from resource_management import * - -from dolphin_env import dolphin_env - - -class DolphinWorkerService(Script): - def install(self, env): - import params - env.set_params(params) - self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) - - def configure(self, env): - import params - params.pika_slave = True - env.set_params(params) - - dolphin_env() - - def start(self, env): - import params - env.set_params(params) - self.configure(env) - no_op_test = format("ls {dolphin_pidfile_dir}/worker-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/worker-server.pid` | grep `cat {dolphin_pidfile_dir}/worker-server.pid` >/dev/null 2>&1") - start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start worker-server") - Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) - - def stop(self, env): - import params - env.set_params(params) - stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop worker-server") - Execute(stop_cmd, user=params.dolphin_user) - time.sleep(5) - - def status(self, env): - import status_params - env.set_params(status_params) - check_process_status(status_params.dolphin_run_dir + "worker-server.pid") - - -if __name__ == "__main__": - DolphinWorkerService().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/params.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/params.py deleted file mode 100644 index b09b2589f4..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/params.py +++ /dev/null @@ -1,154 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - - -import sys -from resource_management import * -from resource_management.core.logger import Logger -from resource_management.libraries.functions import default - -Logger.initialize_logger() -reload(sys) -sys.setdefaultencoding('utf-8') - -# server configurations -config = Script.get_config() - -# conf_dir = "/etc/" -dolphin_home = "/opt/soft/dolphinscheduler" -dolphin_conf_dir = dolphin_home + "/conf" -dolphin_log_dir = dolphin_home + "/logs" -dolphin_bin_dir = dolphin_home + "/bin" -dolphin_lib_jars = dolphin_home + "/lib/*" -dolphin_pidfile_dir = "/opt/soft/run/dolphinscheduler" - -rmHosts = default("/clusterHostInfo/rm_host", []) - -# dolphin-env -dolphin_env_map = {} -dolphin_env_map.update(config['configurations']['dolphin-env']) - -# which user to install and admin dolphin scheduler -dolphin_user = dolphin_env_map['dolphin.user'] -dolphin_group = dolphin_env_map['dolphin.group'] - -# .dolphinscheduler_env.sh -dolphin_env_path = dolphin_conf_dir + '/env/dolphinscheduler_env.sh' -dolphin_env_content = dolphin_env_map['dolphinscheduler-env-content'] - -# database config -dolphin_database_config = {} -dolphin_database_config['dolphin_database_type'] = dolphin_env_map['dolphin.database.type'] -dolphin_database_config['dolphin_database_username'] = dolphin_env_map['dolphin.database.username'] -dolphin_database_config['dolphin_database_password'] = dolphin_env_map['dolphin.database.password'] -if 'mysql' == dolphin_database_config['dolphin_database_type']: - dolphin_database_config['dolphin_database_driver'] = 'com.mysql.jdbc.Driver' - dolphin_database_config['driverDelegateClass'] = 'org.quartz.impl.jdbcjobstore.StdJDBCDelegate' - dolphin_database_config['dolphin_database_url'] = 'jdbc:mysql://' + dolphin_env_map['dolphin.database.host'] \ - + ':' + dolphin_env_map['dolphin.database.port'] \ - + '/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8' -else: - dolphin_database_config['dolphin_database_driver'] = 'org.postgresql.Driver' - dolphin_database_config['driverDelegateClass'] = 'org.quartz.impl.jdbcjobstore.PostgreSQLDelegate' - dolphin_database_config['dolphin_database_url'] = 'jdbc:postgresql://' + dolphin_env_map['dolphin.database.host'] \ - + ':' + dolphin_env_map['dolphin.database.port'] \ - + '/dolphinscheduler' - - - - - -# application-alert.properties -dolphin_alert_map = {} -wechat_push_url = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token' -wechat_token_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret' -wechat_team_send_msg = '{\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"}' -wechat_user_send_msg = '{\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}}' - -dolphin_alert_config_map = config['configurations']['dolphin-alert'] - -if dolphin_alert_config_map['enterprise.wechat.enable']: - dolphin_alert_map['enterprise.wechat.push.ur'] = wechat_push_url - dolphin_alert_map['enterprise.wechat.token.url'] = wechat_token_url - dolphin_alert_map['enterprise.wechat.team.send.msg'] = wechat_team_send_msg - dolphin_alert_map['enterprise.wechat.user.send.msg'] = wechat_user_send_msg - -dolphin_alert_map.update(dolphin_alert_config_map) - - - -# application-api.properties -dolphin_app_api_map = {} -dolphin_app_api_map.update(config['configurations']['dolphin-application-api']) - - -# common.properties -dolphin_common_map = {} - -if 'yarn-site' in config['configurations'] and \ - 'yarn.resourcemanager.webapp.address' in config['configurations']['yarn-site']: - yarn_resourcemanager_webapp_address = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'] - yarn_application_status_address = 'http://' + yarn_resourcemanager_webapp_address + '/ws/v1/cluster/apps/%s' - dolphin_common_map['yarn.application.status.address'] = yarn_application_status_address - -rmHosts = default("/clusterHostInfo/rm_host", []) -if len(rmHosts) > 1: - dolphin_common_map['yarn.resourcemanager.ha.rm.ids'] = ','.join(rmHosts) -else: - dolphin_common_map['yarn.resourcemanager.ha.rm.ids'] = '' - -dolphin_common_map_tmp = config['configurations']['dolphin-common'] -data_basedir_path = dolphin_common_map_tmp['data.basedir.path'] -process_exec_basepath = data_basedir_path + '/exec' -data_download_basedir_path = data_basedir_path + '/download' -dolphin_common_map['process.exec.basepath'] = process_exec_basepath -dolphin_common_map['data.download.basedir.path'] = data_download_basedir_path -dolphin_common_map['dolphinscheduler.env.path'] = dolphin_env_path -dolphin_common_map.update(config['configurations']['dolphin-common']) - -# datasource.properties -dolphin_datasource_map = {} -dolphin_datasource_map['spring.datasource.type'] = 'com.alibaba.druid.pool.DruidDataSource' -dolphin_datasource_map['spring.datasource.driver-class-name'] = dolphin_database_config['dolphin_database_driver'] -dolphin_datasource_map['spring.datasource.url'] = dolphin_database_config['dolphin_database_url'] -dolphin_datasource_map['spring.datasource.username'] = dolphin_database_config['dolphin_database_username'] -dolphin_datasource_map['spring.datasource.password'] = dolphin_database_config['dolphin_database_password'] -dolphin_datasource_map.update(config['configurations']['dolphin-datasource']) - -# master.properties -dolphin_master_map = config['configurations']['dolphin-master'] - -# quartz.properties -dolphin_quartz_map = {} -dolphin_quartz_map['org.quartz.jobStore.driverDelegateClass'] = dolphin_database_config['driverDelegateClass'] -dolphin_quartz_map.update(config['configurations']['dolphin-quartz']) - -# worker.properties -dolphin_worker_map = config['configurations']['dolphin-worker'] - -# zookeeper.properties -dolphin_zookeeper_map={} -zookeeperHosts = default("/clusterHostInfo/zookeeper_hosts", []) -if len(zookeeperHosts) > 0 and "clientPort" in config['configurations']['zoo.cfg']: - clientPort = config['configurations']['zoo.cfg']['clientPort'] - zookeeperPort = ":" + clientPort + "," - dolphin_zookeeper_map['zookeeper.quorum'] = zookeeperPort.join(zookeeperHosts) + ":" + clientPort -dolphin_zookeeper_map.update(config['configurations']['dolphin-zookeeper']) - - - diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/service_check.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/service_check.py deleted file mode 100644 index 0e12f69932..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/service_check.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management import * -from resource_management.libraries.functions import get_unique_id_and_date - -class ServiceCheck(Script): - def service_check(self, env): - import params - #env.set_params(params) - - # Execute(format("which pika_server")) - -if __name__ == "__main__": - ServiceCheck().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/status_params.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/status_params.py deleted file mode 100644 index 24b2c8b1bc..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/status_params.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -from resource_management import * - -config = Script.get_config() - -dolphin_run_dir = "/opt/soft/run/dolphinscheduler/" diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/application-api.properties.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/application-api.properties.j2 deleted file mode 100644 index 70118003b9..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/application-api.properties.j2 +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -{% for key, value in dolphin_app_api_map.iteritems() -%} - {{key}}={{value}} -{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/common.properties.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/common.properties.j2 deleted file mode 100644 index 2220c4effa..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/common.properties.j2 +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -{% for key, value in dolphin_common_map.iteritems() -%} - {{key}}={{value}} -{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/datasource.properties.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/datasource.properties.j2 deleted file mode 100644 index 40aed83543..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/datasource.properties.j2 +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -{% for key, value in dolphin_datasource_map.iteritems() -%} - {{key}}={{value}} -{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/dolphin-daemon.sh.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/dolphin-daemon.sh.j2 deleted file mode 100644 index 0802b74750..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/dolphin-daemon.sh.j2 +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/sh -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -usage="Usage: dolphinscheduler-daemon.sh (start|stop) " - -# if no args specified, show usage -if [ $# -le 1 ]; then - echo $usage - exit 1 -fi - -startStop=$1 -shift -command=$1 -shift - -echo "Begin $startStop $command......" - -BIN_DIR=`dirname $0` -BIN_DIR=`cd "$BIN_DIR"; pwd` -DOLPHINSCHEDULER_HOME=$BIN_DIR/.. - -export HOSTNAME=`hostname` - -DOLPHINSCHEDULER_LIB_JARS={{dolphin_lib_jars}} - -DOLPHINSCHEDULER_OPTS="-server -Xmx16g -Xms4g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" -STOP_TIMEOUT=5 - -log={{dolphin_log_dir}}/dolphinscheduler-$command-$HOSTNAME.out -pid={{dolphin_pidfile_dir}}/$command.pid - -cd $DOLPHINSCHEDULER_HOME - -if [ "$command" = "api-server" ]; then - LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-api.xml -Dspring.profiles.active=api" - CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer -elif [ "$command" = "master-server" ]; then - LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-master.xml -Ddruid.mysql.usePingMethod=false" - CLASS=org.apache.dolphinscheduler.server.master.MasterServer -elif [ "$command" = "worker-server" ]; then - LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-worker.xml -Ddruid.mysql.usePingMethod=false" - CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer -elif [ "$command" = "alert-server" ]; then - LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-alert.xml" - CLASS=org.apache.dolphinscheduler.alert.AlertServer -elif [ "$command" = "logger-server" ]; then - CLASS=org.apache.dolphinscheduler.server.log.LoggerServer -else - echo "Error: No command named \`$command' was found." - exit 1 -fi - -case $startStop in - (start) - - if [ -f $pid ]; then - if kill -0 `cat $pid` > /dev/null 2>&1; then - echo $command running as process `cat $pid`. Stop it first. - exit 1 - fi - fi - - echo starting $command, logging to $log - - exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath {{dolphin_conf_dir}}:{{dolphin_lib_jars}} $CLASS" - - echo "nohup java $exec_command > $log 2>&1 < /dev/null &" - nohup java $exec_command > $log 2>&1 < /dev/null & - echo $! > $pid - ;; - - (stop) - - if [ -f $pid ]; then - TARGET_PID=`cat $pid` - if kill -0 $TARGET_PID > /dev/null 2>&1; then - echo stopping $command - kill $TARGET_PID - sleep $STOP_TIMEOUT - if kill -0 $TARGET_PID > /dev/null 2>&1; then - echo "$command did not stop gracefully after $STOP_TIMEOUT seconds: killing with kill -9" - kill -9 $TARGET_PID - fi - else - echo no $command to stop - fi - rm -f $pid - else - echo no $command to stop - fi - ;; - - (*) - echo $usage - exit 1 - ;; - -esac - -echo "End $startStop $command." \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/master.properties.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/master.properties.j2 deleted file mode 100644 index d9b85e14cf..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/master.properties.j2 +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -{% for key, value in dolphin_master_map.iteritems() -%} - {{key}}={{value}} -{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/quartz.properties.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/quartz.properties.j2 deleted file mode 100644 index e027a263b5..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/quartz.properties.j2 +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -{% for key, value in dolphin_quartz_map.iteritems() -%} - {{key}}={{value}} -{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/worker.properties.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/worker.properties.j2 deleted file mode 100644 index a008b74084..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/worker.properties.j2 +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -{% for key, value in dolphin_worker_map.iteritems() -%} - {{key}}={{value}} -{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/zookeeper.properties.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/zookeeper.properties.j2 deleted file mode 100644 index 9eb14eaef3..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/zookeeper.properties.j2 +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -{% for key, value in dolphin_zookeeper_map.iteritems() -%} - {{key}}={{value}} -{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/quicklinks/quicklinks.json b/ambari_plugin/common-services/DOLPHIN/2.0.0/quicklinks/quicklinks.json deleted file mode 100755 index 8753004fef..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/quicklinks/quicklinks.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "name": "default", - "description": "default quick links configuration", - "configuration": { - "protocol": - { - "type":"http" - }, - - "links": [ - { - "name": "dolphin-application-ui", - "label": "DolphinApplication UI", - "requires_user_name": "false", - "component_name": "DOLPHIN_API", - "url": "%@://%@:%@/dolphinscheduler/ui/view/login/index.html", - "port":{ - "http_property": "server.port", - "http_default_port": "12345", - "regex": "^(\\d+)$", - "site": "dolphin-application-api" - } - } - ] - } -} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/themes/theme.json b/ambari_plugin/common-services/DOLPHIN/2.0.0/themes/theme.json deleted file mode 100644 index 953e2323f8..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/themes/theme.json +++ /dev/null @@ -1,661 +0,0 @@ -{ - "name": "default", - "description": "Default theme for Dolphin Scheduler service", - "configuration": { - "layouts": [ - { - "name": "default", - "tabs": [ - { - "name": "settings", - "display-name": "Settings", - "layout": { - "tab-rows": "3", - "tab-columns": "3", - "sections": [ - { - "name": "dolphin-env-config", - "display-name": "Dolphin Env Config", - "row-index": "0", - "column-index": "0", - "row-span": "1", - "column-span": "2", - "section-rows": "1", - "section-columns": "2", - "subsections": [ - { - "name": "env-row1-col1", - "display-name": "Deploy User Info", - "row-index": "0", - "column-index": "0", - "row-span": "1", - "column-span": "1" - }, - { - "name": "env-row1-col2", - "display-name": "System Env Optimization", - "row-index": "0", - "column-index": "1", - "row-span": "1", - "column-span": "1" - } - ] - }, - { - "name": "dolphin-database-config", - "display-name": "Database Config", - "row-index": "1", - "column-index": "0", - "row-span": "1", - "column-span": "2", - "section-rows": "1", - "section-columns": "3", - "subsections": [ - { - "name": "database-row1-col1", - "row-index": "0", - "column-index": "0", - "row-span": "1", - "column-span": "1" - }, - { - "name": "database-row1-col2", - "row-index": "0", - "column-index": "1", - "row-span": "1", - "column-span": "1" - }, - { - "name": "database-row1-col3", - "row-index": "0", - "column-index": "2", - "row-span": "1", - "column-span": "1" - } - ] - }, - { - "name": "dynamic-config", - "row-index": "2", - "column-index": "0", - "row-span": "1", - "column-span": "2", - "section-rows": "1", - "section-columns": "3", - "subsections": [ - { - "name": "dynamic-row1-col1", - "display-name": "Resource FS Config", - "row-index": "0", - "column-index": "0", - "row-span": "1", - "column-span": "1" - }, - { - "name": "dynamic-row1-col2", - "display-name": "Kerberos Info", - "row-index": "0", - "column-index": "1", - "row-span": "1", - "column-span": "1" - }, - { - "name": "dynamic-row1-col3", - "display-name": "Wechat Info", - "row-index": "0", - "column-index": "1", - "row-span": "1", - "column-span": "1" - } - ] - } - ] - } - } - ] - } - ], - "placement": { - "configuration-layout": "default", - "configs": [ - { - "config": "dolphin-env/dolphin.database.type", - "subsection-name": "database-row1-col1" - }, - { - "config": "dolphin-env/dolphin.database.host", - "subsection-name": "database-row1-col2" - }, - { - "config": "dolphin-env/dolphin.database.port", - "subsection-name": "database-row1-col2" - }, - { - "config": "dolphin-env/dolphin.database.username", - "subsection-name": "database-row1-col3" - }, - { - "config": "dolphin-env/dolphin.database.password", - "subsection-name": "database-row1-col3" - }, - { - "config": "dolphin-env/dolphin.user", - "subsection-name": "env-row1-col1" - }, - { - "config": "dolphin-env/dolphin.group", - "subsection-name": "env-row1-col1" - }, - { - "config": "dolphin-env/dolphinscheduler-env-content", - "subsection-name": "env-row1-col2" - }, - { - "config": "dolphin-common/resource.storage.type", - "subsection-name": "dynamic-row1-col1" - }, - { - "config": "dolphin-common/resource.upload.path", - "subsection-name": "dynamic-row1-col1", - "depends-on": [ - { - "configs":[ - "dolphin-common/resource.storage.type" - ], - "if": "${dolphin-common/resource.storage.type} === HDFS || ${dolphin-common/resource.storage.type} === S3", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-common/hdfs.root.user", - "subsection-name": "dynamic-row1-col1", - "depends-on": [ - { - "configs":[ - "dolphin-common/resource.storage.type" - ], - "if": "${dolphin-common/resource.storage.type} === HDFS", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-common/data.store2hdfs.basepath", - "subsection-name": "dynamic-row1-col1", - "depends-on": [ - { - "configs":[ - "dolphin-common/resource.storage.type" - ], - "if": "${dolphin-common/resource.storage.type} === HDFS", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-common/fs.defaultFS", - "subsection-name": "dynamic-row1-col1", - "depends-on": [ - { - "configs":[ - "dolphin-common/resource.storage.type" - ], - "if": "${dolphin-common/resource.storage.type} === HDFS", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-common/fs.s3a.endpoint", - "subsection-name": "dynamic-row1-col1", - "depends-on": [ - { - "configs":[ - "dolphin-common/resource.storage.type" - ], - "if": "${dolphin-common/resource.storage.type} === S3", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-common/fs.s3a.access.key", - "subsection-name": "dynamic-row1-col1", - "depends-on": [ - { - "configs":[ - "dolphin-common/resource.storage.type" - ], - "if": "${dolphin-common/resource.storage.type} === S3", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-common/fs.s3a.secret.key", - "subsection-name": "dynamic-row1-col1", - "depends-on": [ - { - "configs":[ - "dolphin-common/resource.storage.type" - ], - "if": "${dolphin-common/resource.storage.type} === S3", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-common/hadoop.security.authentication.startup.state", - "subsection-name": "dynamic-row1-col2" - }, - { - "config": "dolphin-common/java.security.krb5.conf.path", - "subsection-name": "dynamic-row1-col2", - "depends-on": [ - { - "configs":[ - "dolphin-common/hadoop.security.authentication.startup.state" - ], - "if": "${dolphin-common/hadoop.security.authentication.startup.state}", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-common/login.user.keytab.username", - "subsection-name": "dynamic-row1-col2", - "depends-on": [ - { - "configs":[ - "dolphin-common/hadoop.security.authentication.startup.state" - ], - "if": "${dolphin-common/hadoop.security.authentication.startup.state}", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-common/login.user.keytab.path", - "subsection-name": "dynamic-row1-col2", - "depends-on": [ - { - "configs":[ - "dolphin-common/hadoop.security.authentication.startup.state" - ], - "if": "${dolphin-common/hadoop.security.authentication.startup.state}", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-common/kerberos.expire.time", - "subsection-name": "dynamic-row1-col2", - "depends-on": [ - { - "configs":[ - "dolphin-common/hadoop.security.authentication.startup.state" - ], - "if": "${dolphin-common/hadoop.security.authentication.startup.state}", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-alert/enterprise.wechat.enable", - "subsection-name": "dynamic-row1-col3" - }, - { - "config": "dolphin-alert/enterprise.wechat.corp.id", - "subsection-name": "dynamic-row1-col3", - "depends-on": [ - { - "configs":[ - "dolphin-alert/enterprise.wechat.enable" - ], - "if": "${dolphin-alert/enterprise.wechat.enable}", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-alert/enterprise.wechat.secret", - "subsection-name": "dynamic-row1-col3", - "depends-on": [ - { - "configs":[ - "dolphin-alert/enterprise.wechat.enable" - ], - "if": "${dolphin-alert/enterprise.wechat.enable}", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-alert/enterprise.wechat.agent.id", - "subsection-name": "dynamic-row1-col3", - "depends-on": [ - { - "configs":[ - "dolphin-alert/enterprise.wechat.enable" - ], - "if": "${dolphin-alert/enterprise.wechat.enable}", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-alert/enterprise.wechat.users", - "subsection-name": "dynamic-row1-col3", - "depends-on": [ - { - "configs":[ - "dolphin-alert/enterprise.wechat.enable" - ], - "if": "${dolphin-alert/enterprise.wechat.enable}", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - } - ] - }, - "widgets": [ - { - "config": "dolphin-env/dolphin.database.type", - "widget": { - "type": "combo" - } - }, - { - "config": "dolphin-env/dolphin.database.host", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-env/dolphin.database.port", - "widget": { - "type": "text-field", - "units": [ - { - "unit-name": "int" - } - ] - } - }, - { - "config": "dolphin-env/dolphin.database.username", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-env/dolphin.database.password", - "widget": { - "type": "password" - } - }, - { - "config": "dolphin-env/dolphin.user", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-env/dolphin.group", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-env/dolphinscheduler-env-content", - "widget": { - "type": "text-area" - } - }, - { - "config": "dolphin-common/resource.storage.type", - "widget": { - "type": "combo" - } - }, - { - "config": "dolphin-common/resource.upload.path", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-common/hdfs.root.user", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-common/data.store2hdfs.basepath", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-common/fs.defaultFS", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-common/fs.s3a.endpoint", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-common/fs.s3a.access.key", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-common/fs.s3a.secret.key", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-common/hadoop.security.authentication.startup.state", - "widget": { - "type": "toggle" - } - }, - { - "config": "dolphin-common/java.security.krb5.conf.path", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-common/login.user.keytab.username", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-common/login.user.keytab.path", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-common/kerberos.expire.time", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-alert/enterprise.wechat.enable", - "widget": { - "type": "toggle" - } - }, - { - "config": "dolphin-alert/enterprise.wechat.corp.id", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-alert/enterprise.wechat.secret", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-alert/enterprise.wechat.agent.id", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-alert/enterprise.wechat.users", - "widget": { - "type": "text-field" - } - } - ] - } -} diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 7e9c4e57cb..26ffb5b2b5 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -23,6 +23,7 @@ services: - "2181:2181" environment: ZOO_MY_ID: 1 + ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons db: image: postgres container_name: postgres diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java index 58a37c2f41..347336cada 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java @@ -16,8 +16,11 @@ */ package org.apache.dolphinscheduler.alert; +import org.apache.dolphinscheduler.alert.plugin.EmailAlertPlugin; import org.apache.dolphinscheduler.alert.runner.AlertSender; import org.apache.dolphinscheduler.alert.utils.Constants; +import org.apache.dolphinscheduler.alert.utils.PropertyUtils; +import org.apache.dolphinscheduler.common.plugin.FilePluginManager; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.DaoFactory; @@ -41,34 +44,47 @@ public class AlertServer { private static AlertServer instance; - public AlertServer() { + private FilePluginManager alertPluginManager; + + private static final String[] whitePrefixes = new String[]{"org.apache.dolphinscheduler.plugin.utils."}; + private static final String[] excludePrefixes = new String[]{ + "org.apache.dolphinscheduler.plugin.", + "ch.qos.logback.", + "org.slf4j." + }; + + public AlertServer() { + alertPluginManager = + new FilePluginManager(PropertyUtils.getString(Constants.PLUGIN_DIR), whitePrefixes, excludePrefixes); + // add default alert plugins + alertPluginManager.addPlugin(new EmailAlertPlugin()); } - public synchronized static AlertServer getInstance(){ + public synchronized static AlertServer getInstance() { if (null == instance) { instance = new AlertServer(); } return instance; } - public void start(){ + public void start() { logger.info("alert server ready start "); - while (Stopper.isRunning()){ + while (Stopper.isRunning()) { try { Thread.sleep(Constants.ALERT_SCAN_INTERVAL); } catch (InterruptedException e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); Thread.currentThread().interrupt(); } List alerts = alertDao.listWaitExecutionAlert(); - alertSender = new AlertSender(alerts, alertDao); + alertSender = new AlertSender(alerts, alertDao, alertPluginManager); alertSender.run(); } } - public static void main(String[] args){ + public static void main(String[] args) { AlertServer alertServer = AlertServer.getInstance(); alertServer.start(); } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EmailManager.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EmailManager.java index 047ee8bfed..96feb7f09e 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EmailManager.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EmailManager.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.alert.manager; import org.apache.dolphinscheduler.alert.utils.MailUtils; -import org.apache.dolphinscheduler.common.enums.ShowType; import java.util.List; import java.util.Map; @@ -35,7 +34,7 @@ public class EmailManager { * @param showType the showType * @return the send result */ - public Map send(List receviersList,List receviersCcList,String title,String content,ShowType showType){ + public Map send(List receviersList,List receviersCcList,String title,String content,String showType){ return MailUtils.sendMails(receviersList, receviersCcList, title, content, showType); } @@ -48,7 +47,7 @@ public class EmailManager { * @param showType the showType * @return the send result */ - public Map send(List receviersList,String title,String content,ShowType showType){ + public Map send(List receviersList,String title,String content,String showType){ return MailUtils.sendMails(receviersList,title, content, showType); } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java index bb06be6561..43649d6758 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.alert.manager; import org.apache.dolphinscheduler.alert.utils.Constants; import org.apache.dolphinscheduler.alert.utils.EnterpriseWeChatUtils; -import org.apache.dolphinscheduler.dao.entity.Alert; +import org.apache.dolphinscheduler.plugin.model.AlertInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,18 +35,18 @@ public class EnterpriseWeChatManager { private static final Logger logger = LoggerFactory.getLogger(EnterpriseWeChatManager.class); /** * Enterprise We Chat send - * @param alert the alert + * @param alertInfo the alert info * @param token the token * @return the send result */ - public Map send(Alert alert, String token){ + public Map send(AlertInfo alertInfo, String token){ Map retMap = new HashMap<>(); retMap.put(Constants.STATUS, false); String agentId = EnterpriseWeChatUtils.ENTERPRISE_WE_CHAT_AGENT_ID; String users = EnterpriseWeChatUtils.ENTERPRISE_WE_CHAT_USERS; List userList = Arrays.asList(users.split(",")); - logger.info("send message {}",alert); - String msg = EnterpriseWeChatUtils.makeUserSendMsg(userList, agentId,EnterpriseWeChatUtils.markdownByAlert(alert)); + logger.info("send message {}", alertInfo.getAlertData().getTitle()); + String msg = EnterpriseWeChatUtils.makeUserSendMsg(userList, agentId,EnterpriseWeChatUtils.markdownByAlert(alertInfo.getAlertData())); try { EnterpriseWeChatUtils.sendEnterpriseWeChat(Constants.UTF_8, msg, token); } catch (IOException e) { diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java new file mode 100644 index 0000000000..d20306b153 --- /dev/null +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.alert.plugin; + +import org.apache.dolphinscheduler.alert.manager.EmailManager; +import org.apache.dolphinscheduler.alert.manager.EnterpriseWeChatManager; +import org.apache.dolphinscheduler.alert.utils.Constants; +import org.apache.dolphinscheduler.alert.utils.EnterpriseWeChatUtils; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.plugin.api.AlertPlugin; +import org.apache.dolphinscheduler.plugin.model.AlertData; +import org.apache.dolphinscheduler.plugin.model.AlertInfo; +import org.apache.dolphinscheduler.plugin.model.PluginName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +/** + * EmailAlertPlugin + * + * This plugin is a default plugin, and mix up email and enterprise wechat, because adapt with former alert behavior + */ +public class EmailAlertPlugin implements AlertPlugin { + + private static final Logger logger = LoggerFactory.getLogger(EmailAlertPlugin.class); + + private PluginName pluginName; + + private static final EmailManager emailManager = new EmailManager(); + private static final EnterpriseWeChatManager weChatManager = new EnterpriseWeChatManager(); + + public EmailAlertPlugin() { + this.pluginName = new PluginName(); + this.pluginName.setEnglish(Constants.PLUGIN_DEFAULT_EMAIL_EN); + this.pluginName.setChinese(Constants.PLUGIN_DEFAULT_EMAIL_CH); + } + + @Override + public String getId() { + return Constants.PLUGIN_DEFAULT_EMAIL; + } + + @Override + public PluginName getName() { + return pluginName; + } + + @Override + @SuppressWarnings("unchecked") + public Map process(AlertInfo info) { + Map retMaps = new HashMap<>(); + + AlertData alert = info.getAlertData(); + + List receviersList = (List) info.getProp(Constants.PLUGIN_DEFAULT_EMAIL_RECEIVERS); + + // receiving group list + // custom receiver + String receivers = alert.getReceivers(); + if (StringUtils.isNotEmpty(receivers)) { + String[] splits = receivers.split(","); + receviersList.addAll(Arrays.asList(splits)); + } + + List receviersCcList = new ArrayList<>(); + // Custom Copier + String receiversCc = alert.getReceiversCc(); + if (StringUtils.isNotEmpty(receiversCc)) { + String[] splits = receiversCc.split(","); + receviersCcList.addAll(Arrays.asList(splits)); + } + + if (CollectionUtils.isEmpty(receviersList) && CollectionUtils.isEmpty(receviersCcList)) { + logger.warn("alert send error : At least one receiver address required"); + retMaps.put(Constants.STATUS, "false"); + retMaps.put(Constants.MESSAGE, "execution failure,At least one receiver address required."); + return retMaps; + } + + retMaps = emailManager.send(receviersList, receviersCcList, alert.getTitle(), alert.getContent(), + alert.getShowType()); + + //send flag + boolean flag = false; + + if (retMaps == null) { + retMaps = new HashMap<>(); + retMaps.put(Constants.MESSAGE, "alert send error."); + retMaps.put(Constants.STATUS, "false"); + logger.info("alert send error : {}", retMaps.get(Constants.MESSAGE)); + return retMaps; + } + + flag = Boolean.parseBoolean(String.valueOf(retMaps.get(Constants.STATUS))); + + if (flag) { + logger.info("alert send success"); + retMaps.put(Constants.MESSAGE, "email send success."); + if (EnterpriseWeChatUtils.isEnable()) { + logger.info("Enterprise WeChat is enable!"); + try { + String token = EnterpriseWeChatUtils.getToken(); + weChatManager.send(info, token); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + } else { + retMaps.put(Constants.MESSAGE, "alert send error."); + logger.info("alert send error : {}", retMaps.get(Constants.MESSAGE)); + } + + return retMaps; + } + +} diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java index 5feb36b60f..d3ac852b0f 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java @@ -16,139 +16,85 @@ */ package org.apache.dolphinscheduler.alert.runner; -import org.apache.dolphinscheduler.alert.manager.EmailManager; -import org.apache.dolphinscheduler.alert.manager.EnterpriseWeChatManager; import org.apache.dolphinscheduler.alert.utils.Constants; -import org.apache.dolphinscheduler.alert.utils.EnterpriseWeChatUtils; import org.apache.dolphinscheduler.common.enums.AlertStatus; -import org.apache.dolphinscheduler.common.enums.AlertType; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.plugin.PluginManager; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.entity.Alert; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.api.AlertPlugin; +import org.apache.dolphinscheduler.plugin.model.AlertData; +import org.apache.dolphinscheduler.plugin.model.AlertInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; /** * alert sender */ -public class AlertSender{ +public class AlertSender { private static final Logger logger = LoggerFactory.getLogger(AlertSender.class); - private static final EmailManager emailManager= new EmailManager(); - private static final EnterpriseWeChatManager weChatManager= new EnterpriseWeChatManager(); - - private List alertList; private AlertDao alertDao; + private PluginManager pluginManager; + + public AlertSender() { + } - public AlertSender(){} - public AlertSender(List alertList, AlertDao alertDao){ + public AlertSender(List alertList, AlertDao alertDao, PluginManager pluginManager) { super(); this.alertList = alertList; this.alertDao = alertDao; + this.pluginManager = pluginManager; } public void run() { - List users; - Map retMaps = null; - for(Alert alert:alertList){ + for (Alert alert : alertList) { users = alertDao.listUserByAlertgroupId(alert.getAlertGroupId()); // receiving group list List receviersList = new ArrayList<>(); - for(User user:users){ + for (User user : users) { receviersList.add(user.getEmail()); } - // custom receiver - String receivers = alert.getReceivers(); - if (StringUtils.isNotEmpty(receivers)){ - String[] splits = receivers.split(","); - receviersList.addAll(Arrays.asList(splits)); - } - // copy list - List receviersCcList = new ArrayList<>(); - - - // Custom Copier - String receiversCc = alert.getReceiversCc(); - - if (StringUtils.isNotEmpty(receiversCc)){ - String[] splits = receiversCc.split(","); - receviersCcList.addAll(Arrays.asList(splits)); - } - - if (CollectionUtils.isEmpty(receviersList) && CollectionUtils.isEmpty(receviersCcList)) { - logger.warn("alert send error : At least one receiver address required"); - alertDao.updateAlert(AlertStatus.EXECUTION_FAILURE, "execution failure,At least one receiver address required.", alert.getId()); - continue; - } - - if (alert.getAlertType() == AlertType.EMAIL){ - retMaps = emailManager.send(receviersList,receviersCcList, alert.getTitle(), alert.getContent(),alert.getShowType()); - - alert.setInfo(retMaps); - }else if (alert.getAlertType() == AlertType.SMS){ - retMaps = emailManager.send(getReciversForSMS(users), alert.getTitle(), alert.getContent(),alert.getShowType()); - alert.setInfo(retMaps); + AlertData alertData = new AlertData(); + alertData.setId(alert.getId()) + .setAlertGroupId(alert.getAlertGroupId()) + .setContent(alert.getContent()) + .setLog(alert.getLog()) + .setReceivers(alert.getReceivers()) + .setReceiversCc(alert.getReceiversCc()) + .setShowType(alert.getShowType().getDescp()) + .setTitle(alert.getTitle()); + + AlertInfo alertInfo = new AlertInfo(); + alertInfo.setAlertData(alertData); + + alertInfo.addProp("receivers", receviersList); + + AlertPlugin emailPlugin = pluginManager.findOne(Constants.PLUGIN_DEFAULT_EMAIL); + retMaps = emailPlugin.process(alertInfo); + + if (retMaps == null) { + alertDao.updateAlert(AlertStatus.EXECUTION_FAILURE, "alert send error", alert.getId()); + logger.info("alert send error : return value is null"); + } else if (!Boolean.parseBoolean(String.valueOf(retMaps.get(Constants.STATUS)))) { + alertDao.updateAlert(AlertStatus.EXECUTION_FAILURE, String.valueOf(retMaps.get(Constants.MESSAGE)), alert.getId()); + logger.info("alert send error : {}", retMaps.get(Constants.MESSAGE)); } else { - logger.error("AlertType is not defined. code: {}, descp: {}", - alert.getAlertType().getCode(), - alert.getAlertType().getDescp()); - return; - } - - //send flag - boolean flag = false; - - if (null != retMaps) { - flag = Boolean.parseBoolean(String.valueOf(retMaps.get(Constants.STATUS))); - } - - if (flag) { - alertDao.updateAlert(AlertStatus.EXECUTION_SUCCESS, "execution success", alert.getId()); + alertDao.updateAlert(AlertStatus.EXECUTION_SUCCESS, (String) retMaps.get(Constants.MESSAGE), alert.getId()); logger.info("alert send success"); - if (EnterpriseWeChatUtils.isEnable()) { - logger.info("Enterprise WeChat is enable!"); - try { - String token = EnterpriseWeChatUtils.getToken(); - weChatManager.send(alert, token); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - } else { - if (null != retMaps) { - alertDao.updateAlert(AlertStatus.EXECUTION_FAILURE, String.valueOf(retMaps.get(Constants.MESSAGE)), alert.getId()); - logger.info("alert send error : {}", retMaps.get(Constants.MESSAGE)); - } } } } - - /** - * get a list of SMS users - * @param users - * @return - */ - private List getReciversForSMS(List users){ - List list = new ArrayList<>(); - for (User user : users){ - list.add(user.getPhone()); - } - return list; - } } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java index 28be8aa195..8fa38c62fc 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java @@ -156,4 +156,21 @@ public class Constants { public static final String ENTERPRISE_WECHAT_AGENT_ID = "enterprise.wechat.agent.id"; public static final String ENTERPRISE_WECHAT_USERS = "enterprise.wechat.users"; + + /** + * plugin config + */ + public static final String PLUGIN_DIR = "plugin.dir"; + + public static final String PLUGIN_DEFAULT_EMAIL = "email"; + + public static final String PLUGIN_DEFAULT_EMAIL_CH = "邮件"; + + public static final String PLUGIN_DEFAULT_EMAIL_EN = "email"; + + public static final String PLUGIN_DEFAULT_EMAIL_RECEIVERS = "receivers"; + + public static final String PLUGIN_DEFAULT_EMAIL_RECEIVERCCS = "receiverCcs"; + + public static final String RETMAP_MSG = "msg"; } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java index 170c0dd37e..d199d154aa 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java @@ -18,10 +18,10 @@ package org.apache.dolphinscheduler.alert.utils; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.dao.entity.Alert; import com.alibaba.fastjson.JSON; import com.google.common.reflect.TypeToken; +import org.apache.dolphinscheduler.plugin.model.AlertData; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; @@ -66,15 +66,17 @@ public class EnterpriseWeChatUtils { * get Enterprise WeChat is enable * @return isEnable */ - public static Boolean isEnable(){ - Boolean isEnable = false; + public static boolean isEnable(){ + Boolean isEnable = null; try { isEnable = PropertyUtils.getBoolean(Constants.ENTERPRISE_WECHAT_ENABLE); } catch (Exception e) { logger.error(e.getMessage(),e); } + if (isEnable == null) { + return false; + } return isEnable; - } /** @@ -253,14 +255,13 @@ public class EnterpriseWeChatUtils { /** * Determine the mardown style based on the show type of the alert - * @param alert the alert * @return the markdown alert table/text */ - public static String markdownByAlert(Alert alert){ + public static String markdownByAlert(AlertData alert){ String result = ""; - if (alert.getShowType() == ShowType.TABLE) { + if (alert.getShowType().equals(ShowType.TABLE.getDescp())) { result = markdownTable(alert.getTitle(),alert.getContent()); - }else if(alert.getShowType() == ShowType.TEXT){ + }else if(alert.getShowType().equals(ShowType.TEXT.getDescp())){ result = markdownText(alert.getTitle(),alert.getContent()); } return result; diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java index ef364cb1c2..bb565f5133 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java @@ -74,7 +74,7 @@ public class MailUtils { * @param showType the show type * @return the result map */ - public static Map sendMails(Collection receivers, String title, String content,ShowType showType) { + public static Map sendMails(Collection receivers, String title, String content,String showType) { return sendMails(receivers, null, title, content, showType); } @@ -87,7 +87,7 @@ public class MailUtils { * @param showType the show type * @return the send result */ - public static Map sendMails(Collection receivers, Collection receiversCc, String title, String content, ShowType showType) { + public static Map sendMails(Collection receivers, Collection receiversCc, String title, String content, String showType) { Map retMap = new HashMap<>(); retMap.put(Constants.STATUS, false); @@ -98,7 +98,7 @@ public class MailUtils { receivers.removeIf(StringUtils::isEmpty); - if (showType == ShowType.TABLE || showType == ShowType.TEXT){ + if (showType.equals(ShowType.TABLE.getDescp()) || showType.equals(ShowType.TEXT.getDescp())) { // send email HtmlEmail email = new HtmlEmail(); @@ -125,10 +125,10 @@ public class MailUtils { } catch (Exception e) { handleException(receivers, retMap, e); } - }else if (showType == ShowType.ATTACHMENT || showType == ShowType.TABLEATTACHMENT){ + }else if (showType.equals(ShowType.ATTACHMENT.getDescp()) || showType.equals(ShowType.TABLEATTACHMENT.getDescp())) { try { - String partContent = (showType == ShowType.ATTACHMENT ? "Please see the attachment " + title + Constants.EXCEL_SUFFIX_XLS : htmlTable(content,false)); + String partContent = (showType.equals(ShowType.ATTACHMENT.getDescp()) ? "Please see the attachment " + title + Constants.EXCEL_SUFFIX_XLS : htmlTable(content,false)); attachment(receivers,receiversCc,title,content,partContent); @@ -290,7 +290,7 @@ public class MailUtils { * @return the result map * @throws EmailException */ - private static Map getStringObjectMap(String title, String content, ShowType showType, Map retMap, HtmlEmail email) throws EmailException { + private static Map getStringObjectMap(String title, String content, String showType, Map retMap, HtmlEmail email) throws EmailException { /** * the subject of the message to be sent @@ -299,9 +299,9 @@ public class MailUtils { /** * to send information, you can use HTML tags in mail content because of the use of HtmlEmail */ - if (showType == ShowType.TABLE) { + if (showType.equals(ShowType.TABLE.getDescp())) { email.setMsg(htmlTable(content)); - } else if (showType == ShowType.TEXT) { + } else if (showType.equals(ShowType.TEXT.getDescp())) { email.setMsg(htmlText(content)); } diff --git a/dolphinscheduler-alert/src/main/resources/alert.properties b/dolphinscheduler-alert/src/main/resources/alert.properties index 3e83c01235..19b55fec97 100644 --- a/dolphinscheduler-alert/src/main/resources/alert.properties +++ b/dolphinscheduler-alert/src/main/resources/alert.properties @@ -45,5 +45,5 @@ enterprise.wechat.enable=false #enterprise.wechat.team.send.msg={\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"} #enterprise.wechat.user.send.msg={\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}} - +plugin.dir=/Users/xx/your/path/to/plugin/dir diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java new file mode 100644 index 0000000000..52c2fc050e --- /dev/null +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.alert.plugin; + +import org.apache.dolphinscheduler.alert.utils.Constants; +import org.apache.dolphinscheduler.common.enums.ShowType; +import org.apache.dolphinscheduler.plugin.api.AlertPlugin; +import org.apache.dolphinscheduler.plugin.model.AlertData; +import org.apache.dolphinscheduler.plugin.model.AlertInfo; +import org.apache.dolphinscheduler.plugin.model.PluginName; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.*; + +public class EmailAlertPluginTest { + + private static final Logger logger = LoggerFactory.getLogger(EmailAlertPluginTest.class); + + private AlertPlugin plugin; + + @Before + public void before() { + plugin = new EmailAlertPlugin(); + } + + @Test + public void getId() { + String id = plugin.getId(); + assertEquals(Constants.PLUGIN_DEFAULT_EMAIL, id); + } + + @Test + public void getName() { + PluginName pluginName = plugin.getName(); + assertEquals(Constants.PLUGIN_DEFAULT_EMAIL_CH, pluginName.getChinese()); + assertEquals(Constants.PLUGIN_DEFAULT_EMAIL_EN, pluginName.getEnglish()); + } + + @Test + public void process() { + AlertInfo alertInfo = new AlertInfo(); + AlertData alertData = new AlertData(); + alertData.setId(1) + .setAlertGroupId(1) + .setContent("[\"alarm time：2018-02-05\", \"service name：MYSQL_ALTER\", \"alarm name：MYSQL_ALTER_DUMP\", " + + "\"get the alarm exception.！，interface error，exception information：timed out\", \"request address：http://blog.csdn.net/dreamInTheWorld/article/details/78539286\"]") + .setLog("test log") + .setReceivers("xx@xx.com") + .setReceiversCc("xx@xx.com") + .setShowType(ShowType.TEXT.getDescp()) + .setTitle("test title"); + + alertInfo.setAlertData(alertData); + List list = new ArrayList(){{ add("xx@xx.com"); }}; + alertInfo.addProp("receivers", list); + Map ret = plugin.process(alertInfo); + assertFalse(Boolean.parseBoolean(String.valueOf(ret.get(Constants.STATUS)))); + } +} \ No newline at end of file diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java index d0f3538c1b..2b405cc436 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java @@ -20,6 +20,7 @@ import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.dao.entity.Alert; +import org.apache.dolphinscheduler.plugin.model.AlertData; import org.junit.Assert; import org.junit.Before; import org.junit.Ignore; @@ -120,14 +121,22 @@ public class EnterpriseWeChatUtilsTest { @Test public void testMarkdownByAlertForText(){ Alert alertForText = createAlertForText(); - String result = EnterpriseWeChatUtils.markdownByAlert(alertForText); + AlertData alertData = new AlertData(); + alertData.setTitle(alertForText.getTitle()) + .setShowType(alertForText.getShowType().getDescp()) + .setContent(alertForText.getContent()); + String result = EnterpriseWeChatUtils.markdownByAlert(alertData); Assert.assertNotNull(result); } @Test public void testMarkdownByAlertForTable(){ Alert alertForText = createAlertForTable(); - String result = EnterpriseWeChatUtils.markdownByAlert(alertForText); + AlertData alertData = new AlertData(); + alertData.setTitle(alertForText.getTitle()) + .setShowType(alertForText.getShowType().getDescp()) + .setContent(alertForText.getContent()); + String result = EnterpriseWeChatUtils.markdownByAlert(alertData); Assert.assertNotNull(result); } diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java index 1820a1ef89..11322da0e3 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java @@ -58,7 +58,7 @@ public class MailUtilsTest { alert.setAlertType(AlertType.EMAIL); alert.setAlertGroupId(4); - MailUtils.sendMails(Arrays.asList(receivers),Arrays.asList(receiversCc),alert.getTitle(),alert.getContent(), ShowType.TEXT); + MailUtils.sendMails(Arrays.asList(receivers),Arrays.asList(receiversCc),alert.getTitle(),alert.getContent(), ShowType.TEXT.getDescp()); } @@ -70,7 +70,7 @@ public class MailUtilsTest { String[] mails = new String[]{"xx@xx.com"}; for(Alert alert : alerts){ - MailUtils.sendMails(Arrays.asList(mails),"gaojing", alert.getContent(), alert.getShowType()); + MailUtils.sendMails(Arrays.asList(mails),"gaojing", alert.getContent(), ShowType.TABLE.getDescp()); } } @@ -111,7 +111,7 @@ public class MailUtilsTest { alert.setContent(content); alert.setAlertType(AlertType.EMAIL); alert.setAlertGroupId(1); - MailUtils.sendMails(Arrays.asList(mails),"gaojing", alert.getContent(), ShowType.TABLE); + MailUtils.sendMails(Arrays.asList(mails),"gaojing", alert.getContent(), ShowType.TABLE.getDescp()); } /** @@ -170,7 +170,7 @@ public class MailUtilsTest { alert.setContent(content); alert.setAlertType(AlertType.EMAIL); alert.setAlertGroupId(1); - MailUtils.sendMails(Arrays.asList(mails),"gaojing",alert.getContent(),ShowType.ATTACHMENT); + MailUtils.sendMails(Arrays.asList(mails),"gaojing",alert.getContent(),ShowType.ATTACHMENT.getDescp()); } @Test @@ -183,7 +183,7 @@ public class MailUtilsTest { alert.setContent(content); alert.setAlertType(AlertType.EMAIL); alert.setAlertGroupId(1); - MailUtils.sendMails(Arrays.asList(mails),"gaojing",alert.getContent(),ShowType.TABLEATTACHMENT); + MailUtils.sendMails(Arrays.asList(mails),"gaojing",alert.getContent(),ShowType.TABLEATTACHMENT.getDescp()); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index 51f5420ac5..1aab0953da 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.api.enums.ExecuteType; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.*; +import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; @@ -59,7 +60,7 @@ public class ExecutorService extends BaseService{ private ProcessDefinitionMapper processDefinitionMapper; @Autowired - private ProcessDefinitionService processDefinitionService; + private MonitorService monitorService; @Autowired @@ -123,6 +124,14 @@ public class ExecutorService extends BaseService{ return result; } + // check master server exists + List masterServers = monitorService.getServerListFromZK(true); + + + if (masterServers.size() == 0) { + putMsg(result, Status.MASTER_NOT_EXISTS); + return result; + } /** * create command */ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java index cb07ffbbe3..9328fe0375 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java @@ -365,6 +365,7 @@ public class SchedulerService extends BaseService { if (masterServers.size() == 0) { putMsg(result, Status.MASTER_NOT_EXISTS); + return result; } // set status diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java index a33754397d..55bb3d336c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java @@ -33,7 +33,7 @@ public class ZooKeeperState { private final String host; private final int port; - private int minLatency = -1, avgLatency = -1, maxLatency = -1; + private float minLatency = -1, avgLatency = -1, maxLatency = -1; private long received = -1; private long sent = -1; private int outStanding = -1; @@ -60,9 +60,9 @@ public class ZooKeeperState { String line = scannerForStat.nextLine(); if (line.startsWith("Latency min/avg/max:")) { String[] latencys = getStringValueFromLine(line).split("/"); - minLatency = Integer.parseInt(latencys[0]); - avgLatency = Integer.parseInt(latencys[1]); - maxLatency = Integer.parseInt(latencys[2]); + minLatency = Float.parseFloat(latencys[0]); + avgLatency = Float.parseFloat(latencys[1]); + maxLatency = Float.parseFloat(latencys[2]); } else if (line.startsWith("Received:")) { received = Long.parseLong(getStringValueFromLine(line)); } else if (line.startsWith("Sent:")) { @@ -165,15 +165,15 @@ public class ZooKeeperState { return port; } - public int getMinLatency() { + public float getMinLatency() { return minLatency; } - public int getAvgLatency() { + public float getAvgLatency() { return avgLatency; } - public int getMaxLatency() { + public float getMaxLatency() { return maxLatency; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitor.java index f91d3bc68c..7ba704916c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitor.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitor.java @@ -88,9 +88,9 @@ public class ZookeeperMonitor extends AbstractZKClient { long sent = state.getSent(); long received = state.getReceived(); String mode = state.getMode(); - int minLatency = state.getMinLatency(); - int avgLatency = state.getAvgLatency(); - int maxLatency = state.getMaxLatency(); + float minLatency = state.getMinLatency(); + float avgLatency = state.getAvgLatency(); + float maxLatency = state.getMaxLatency(); int nodeCount = state.getNodeCount(); int status = ok ? 1 : 0; Date date = new Date(); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java index a8777541b7..59523bdd11 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.RunMode; +import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; @@ -63,6 +64,9 @@ public class ExecutorService2Test { @Mock private ProjectService projectService; + @Mock + private MonitorService monitorService; + private int processDefinitionId = 1; private int tenantId = 1; @@ -102,6 +106,7 @@ public class ExecutorService2Test { Mockito.when(processDefinitionMapper.selectById(processDefinitionId)).thenReturn(processDefinition); Mockito.when(processService.getTenantForProcess(tenantId, userId)).thenReturn(new Tenant()); Mockito.when(processService.createCommand(any(Command.class))).thenReturn(1); + Mockito.when(monitorService.getServerListFromZK(true)).thenReturn(getMasterServersList()); } /** @@ -121,7 +126,6 @@ public class ExecutorService2Test { Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); verify(processService, times(1)).createCommand(any(Command.class)); }catch (Exception e){ - Assert.assertTrue(false); } } @@ -142,7 +146,6 @@ public class ExecutorService2Test { Assert.assertEquals(Status.START_PROCESS_INSTANCE_ERROR, result.get(Constants.STATUS)); verify(processService, times(0)).createCommand(any(Command.class)); }catch (Exception e){ - Assert.assertTrue(false); } } @@ -163,7 +166,6 @@ public class ExecutorService2Test { Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); verify(processService, times(1)).createCommand(any(Command.class)); }catch (Exception e){ - Assert.assertTrue(false); } } @@ -184,7 +186,6 @@ public class ExecutorService2Test { Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); verify(processService, times(31)).createCommand(any(Command.class)); }catch (Exception e){ - Assert.assertTrue(false); } } @@ -205,10 +206,42 @@ public class ExecutorService2Test { Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); verify(processService, times(15)).createCommand(any(Command.class)); }catch (Exception e){ - Assert.assertTrue(false); } } + + @Test + public void testNoMsterServers() throws ParseException{ + Mockito.when(monitorService.getServerListFromZK(true)).thenReturn(new ArrayList()); + + Map result = executorService.execProcessInstance(loginUser, projectName, + processDefinitionId, cronTime, CommandType.COMPLEMENT_DATA, + null, null, + null, null, 0, + "", "", RunMode.RUN_MODE_PARALLEL, + Priority.LOW, Constants.DEFAULT_WORKER_GROUP, 110); + Assert.assertEquals(result.get(Constants.STATUS),Status.MASTER_NOT_EXISTS); + + } + + private List getMasterServersList(){ + List masterServerList = new ArrayList<>(); + Server masterServer1 = new Server(); + masterServer1.setId(1); + masterServer1.setHost("192.168.220.188"); + masterServer1.setPort(1121); + masterServerList.add(masterServer1); + + Server masterServer2 = new Server(); + masterServer2.setId(2); + masterServer2.setHost("192.168.220.189"); + masterServer2.setPort(1122); + masterServerList.add(masterServer2); + + return masterServerList; + + } + private List zeroSchedulerList(){ return Collections.EMPTY_LIST; } diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index ca75a84a62..3954159dfa 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -32,12 +32,15 @@ 3.1.0 + + org.apache.dolphinscheduler + dolphinscheduler-plugin-api + com.alibaba fastjson compile - org.apache.httpcomponents httpclient diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 853ab95d1c..effa4f0f8e 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -954,4 +954,11 @@ public final class Constants { * authorize readable perm */ public static final int AUTHORIZE_READABLE_PERM=4; + + + /** + * plugin configurations + */ + public static final String PLUGIN_JAR_SUFFIX = ".jar"; + } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/FilePluginManager.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/FilePluginManager.java new file mode 100644 index 0000000000..d4dd4bf283 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/FilePluginManager.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.plugin; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.plugin.api.AlertPlugin; +import org.apache.dolphinscheduler.plugin.spi.AlertPluginProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Map; +import java.util.ServiceLoader; +import java.util.concurrent.ConcurrentHashMap; + +/** + * FilePluginManager + */ +public class FilePluginManager implements PluginManager { + + private static final Logger logger = LoggerFactory.getLogger(FilePluginManager.class); + + private Map pluginMap = new ConcurrentHashMap<>(); + + private Map> pluginLoaderMap = new ConcurrentHashMap<>(); + + private Map classLoaderMap = new ConcurrentHashMap<>(); + + private String[] whitePrefixes; + + private String[] excludePrefixes; + + public FilePluginManager(String dirPath, String[] whitePrefixes, String[] excludePrefixes) { + this.whitePrefixes = whitePrefixes; + this.excludePrefixes = excludePrefixes; + try { + load(dirPath); + } catch (MalformedURLException e) { + logger.error("load plugins failed.", e); + } + } + + private void load(String dirPath) throws MalformedURLException { + logger.info("start to load jar files in {}", dirPath); + if (dirPath == null) { + logger.error("not a valid path - {}", dirPath); + return; + } + File[] files = new File(dirPath).listFiles(); + if (files == null) { + logger.error("not a valid path - {}", dirPath); + return; + } + for (File file : files) { + if (file.isDirectory() && !file.getPath().endsWith(Constants.PLUGIN_JAR_SUFFIX)) { + continue; + } + String pluginName = file.getName() + .substring(0, file.getName().length() - Constants.PLUGIN_JAR_SUFFIX.length()); + URL[] urls = new URL[]{ file.toURI().toURL() }; + PluginClassLoader classLoader = + new PluginClassLoader(urls, Thread.currentThread().getContextClassLoader(), whitePrefixes, excludePrefixes); + classLoaderMap.put(pluginName, classLoader); + + ServiceLoader loader = ServiceLoader.load(AlertPluginProvider.class, classLoader); + pluginLoaderMap.put(pluginName, loader); + + loader.forEach(provider -> { + AlertPlugin plugin = provider.createPlugin(); + pluginMap.put(plugin.getId(), plugin); + logger.info("loaded plugin - {}", plugin.getId()); + }); + } + } + + @Override + public AlertPlugin findOne(String name) { + return pluginMap.get(name); + } + + @Override + public Map findAll() { + return pluginMap; + } + + @Override + public void addPlugin(AlertPlugin plugin) { + pluginMap.put(plugin.getId(), plugin); + } + +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/PluginClassLoader.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/PluginClassLoader.java new file mode 100644 index 0000000000..528e83a59c --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/PluginClassLoader.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.plugin; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Enumeration; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; + +/** + * Plugin Class Loader + */ +public class PluginClassLoader extends URLClassLoader { + + private static final Logger logger = LoggerFactory.getLogger(PluginClassLoader.class); + + private static final String JAVA_PACKAGE_PREFIX = "java."; + private static final String JAVAX_PACKAGE_PREFIX = "javax."; + + private final String[] whitePrefixes; + + private final String[] excludePrefixes; + + public PluginClassLoader(URL[] urls, ClassLoader parent, String[] whitePrefix, String[] excludePreifx) { + super(urls, parent); + this.whitePrefixes = whitePrefix; + this.excludePrefixes = excludePreifx; + } + + @Override + public Class loadClass(String name) throws ClassNotFoundException { + logger.trace("Received request to load class '{}'", name); + synchronized (getClassLoadingLock(name)) { + if (name.startsWith(JAVA_PACKAGE_PREFIX) || name.startsWith(JAVAX_PACKAGE_PREFIX)) { + return findSystemClass(name); + } + + boolean isWhitePrefixes = fromWhitePrefix(name); + boolean isExcludePrefixed = fromExcludePrefix(name); + + // if the class is part of the plugin engine use parent class loader + if (!isWhitePrefixes && isExcludePrefixed) { + return getParent().loadClass(name); + } + + // check whether it's already been loaded + Class loadedClass = findLoadedClass(name); + if (loadedClass != null) { + logger.debug("Found loaded class '{}'", name); + return loadedClass; + } + + // nope, try to load locally + try { + loadedClass = findClass(name); + logger.debug("Found class '{}' in plugin classpath", name); + return loadedClass; + } catch (ClassNotFoundException e) { + // try next step + } + + // use the standard ClassLoader (which follows normal parent delegation) + return super.loadClass(name); + } + } + + private boolean fromWhitePrefix(String name) { + if (this.whitePrefixes == null) { + return false; + } + for (String whitePrefix : this.whitePrefixes) { + if (name.startsWith(whitePrefix)) { + return true; + } + } + return false; + } + + private boolean fromExcludePrefix(String name) { + if (this.excludePrefixes == null) { + return false; + } + for (String excludePrefix : this.excludePrefixes) { + if (name.startsWith(excludePrefix)) { + return true; + } + } + return false; + } + + @Override + public Enumeration getResources(String name) throws IOException { + List allRes = new LinkedList<>(); + + Enumeration thisRes = findResources(name); + if (thisRes != null) { + while (thisRes.hasMoreElements()) { + allRes.add(thisRes.nextElement()); + } + } + + Enumeration parentRes = super.findResources(name); + if (parentRes != null) { + while (parentRes.hasMoreElements()) { + allRes.add(parentRes.nextElement()); + } + } + + return new Enumeration() { + Iterator it = allRes.iterator(); + + @Override + public boolean hasMoreElements() { + return it.hasNext(); + } + + @Override + public URL nextElement() { + return it.next(); + } + }; + } + + @Override + public URL getResource(String name) { + URL res = null; + + res = findResource(name); + if (res == null) { + res = super.getResource(name); + } + return res; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/PluginManager.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/PluginManager.java new file mode 100644 index 0000000000..f8078841e4 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/PluginManager.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.plugin; + +import org.apache.dolphinscheduler.plugin.api.AlertPlugin; + +import java.util.Map; + +/** + * PluginManager + */ +public interface PluginManager { + + AlertPlugin findOne(String name); + + Map findAll(); + + void addPlugin(AlertPlugin plugin); +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 02f00ce330..963aff5f31 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -503,6 +503,9 @@ public class HadoopUtils implements Closeable { * @return hdfs file name */ public static String getHdfsFileName(ResourceType resourceType, String tenantCode, String fileName) { + if (fileName.startsWith("/")) { + fileName = fileName.replaceFirst("/",""); + } return String.format("%s/%s", getHdfsDir(resourceType,tenantCode), fileName); } @@ -514,6 +517,9 @@ public class HadoopUtils implements Closeable { * @return get absolute path and name for file on hdfs */ public static String getHdfsResourceFileName(String tenantCode, String fileName) { + if (fileName.startsWith("/")) { + fileName = fileName.replaceFirst("/",""); + } return String.format("%s/%s", getHdfsResDir(tenantCode), fileName); } @@ -525,6 +531,9 @@ public class HadoopUtils implements Closeable { * @return get absolute path and name for udf file on hdfs */ public static String getHdfsUdfFileName(String tenantCode, String fileName) { + if (fileName.startsWith("/")) { + fileName = fileName.replaceFirst("/",""); + } return String.format("%s/%s", getHdfsUdfDir(tenantCode), fileName); } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/plugin/FilePluginManagerTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/plugin/FilePluginManagerTest.java new file mode 100644 index 0000000000..1a57cb10fa --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/plugin/FilePluginManagerTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.plugin; + +import org.apache.dolphinscheduler.plugin.api.AlertPlugin; +import org.apache.dolphinscheduler.plugin.model.AlertInfo; +import org.apache.dolphinscheduler.plugin.model.PluginName; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.*; + +public class FilePluginManagerTest { + + private FilePluginManager filePluginManager; + private AlertPlugin alertPlugin; + + @Before + public void before() { + filePluginManager = new FilePluginManager(null, null, null); + alertPlugin = new AlertPlugin() { + @Override + public String getId() { + return "test"; + } + + @Override + public PluginName getName() { + return new PluginName().setChinese("ch").setEnglish("en"); + } + + @Override + public Map process(AlertInfo info) { + return new HashMap<>(); + } + }; + } + + @Test + public void findOne() { + filePluginManager.addPlugin(alertPlugin); + assertEquals(alertPlugin, filePluginManager.findOne(alertPlugin.getId())); + } + + @Test + public void findAll() { + assertNotNull(filePluginManager.findAll()); + } + + @Test + public void addPlugin() { + filePluginManager.addPlugin(alertPlugin); + assertNotNull(filePluginManager.findAll()); + } +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/plugin/PluginClassLoaderTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/plugin/PluginClassLoaderTest.java new file mode 100644 index 0000000000..8a6bfaee13 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/plugin/PluginClassLoaderTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.plugin; + +import org.junit.Before; +import org.junit.Test; + +import java.net.URL; + +import static org.junit.Assert.*; + +public class PluginClassLoaderTest { + + private PluginClassLoader pluginClassLoader; + private ClassLoader parent; + + @Before + public void setUp() { + parent = Thread.currentThread().getContextClassLoader(); + pluginClassLoader = new PluginClassLoader( + new URL[]{}, parent, + null, null); + } + + @Test + public void loadClassNull() { + Class clazz = null; + try { + clazz = pluginClassLoader.loadClass("java.lang.Object"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + assertEquals(null, clazz.getClassLoader()); + } + + @Test + public void loadClassApp() { + Class clazz = null; + try { + clazz = pluginClassLoader.loadClass("org.apache.dolphinscheduler.common.Constants"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + assertEquals(parent, clazz.getClassLoader()); + } + +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java index 00b8f1c5c6..e239fe7cb0 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java @@ -127,6 +127,18 @@ public class HadoopUtilsTest { Assert.assertEquals("/dolphinscheduler/11000/resources/aa.txt", result); } + @Test + public void getHdfsResourceFileName() { + String result = hadoopUtils.getHdfsResourceFileName("11000","aa.txt"); + Assert.assertEquals("/dolphinscheduler/11000/resources/aa.txt", result); + } + + @Test + public void getHdfsUdfFileName() { + String result = hadoopUtils.getHdfsFileName(ResourceType.UDF,"11000","aa.txt"); + Assert.assertEquals("/dolphinscheduler/11000/udfs/aa.txt", result); + } + @Test public void isYarnEnabled() { boolean result = hadoopUtils.isYarnEnabled(); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ZookeeperRecord.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ZookeeperRecord.java index 6b86ac5c11..7d45c66940 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ZookeeperRecord.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ZookeeperRecord.java @@ -56,17 +56,17 @@ public class ZookeeperRecord { /** * min Latency */ - private int minLatency; + private float minLatency; /** * avg Latency */ - private int avgLatency; + private float avgLatency; /** * max Latency */ - private int maxLatency; + private float maxLatency; /** * node count @@ -85,7 +85,7 @@ public class ZookeeperRecord { private int state; - public ZookeeperRecord(String hostname,int connections, int watches, long sent, long received, String mode, int minLatency, int avgLatency, int maxLatency, int nodeCount, int state,Date date) { + public ZookeeperRecord(String hostname,int connections, int watches, long sent, long received, String mode, float minLatency, float avgLatency, float maxLatency, int nodeCount, int state,Date date) { this.hostname = hostname; this.connections = connections; this.watches = watches; @@ -149,23 +149,23 @@ public class ZookeeperRecord { this.mode = mode; } - public int getMinLatency() { + public float getMinLatency() { return minLatency; } - public void setMinLatency(int minLatency) { + public void setMinLatency(float minLatency) { this.minLatency = minLatency; } - public int getAvgLatency() { + public float getAvgLatency() { return avgLatency; } - public void setAvgLatency(int avgLatency) { + public void setAvgLatency(float avgLatency) { this.avgLatency = avgLatency; } - public int getMaxLatency() { + public float getMaxLatency() { return maxLatency; } diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index 82e641ec72..52563f5573 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -474,7 +474,6 @@ The following components are provided under a MIT 2.0 license. See project link The text of each license is also included at licenses/LICENSE-[project].txt. jul-to-slf4j 1.7.25: https://mvnrepository.com/artifact/org.slf4j/jul-to-slf4j/1.7.25, MIT - lombok 1.18.6: https://mvnrepository.com/artifact/org.projectlombok/lombok/1.18.6, MIT mssql-jdbc 6.1.0.jre8: https://mvnrepository.com/artifact/com.microsoft.sqlserver/mssql-jdbc/6.1.0.jre8, MIT slf4j-api 1.7.5: https://mvnrepository.com/artifact/org.slf4j/slf4j-api/1.7.5, MIT diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-lombok.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-lombok.txt deleted file mode 100644 index 5da7dbeff3..0000000000 --- a/dolphinscheduler-dist/release-docs/licenses/LICENSE-lombok.txt +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (C) 2009-2015 The Project Lombok Authors. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/dolphinscheduler-plugin-api/pom.xml b/dolphinscheduler-plugin-api/pom.xml new file mode 100644 index 0000000000..b958b21616 --- /dev/null +++ b/dolphinscheduler-plugin-api/pom.xml @@ -0,0 +1,52 @@ + + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler + 1.2.1-SNAPSHOT + + dolphinscheduler-plugin-api + ${project.artifactId} + jar + + + UTF-8 + + + + + org.slf4j + slf4j-api + + + junit + junit + test + + + commons-io + commons-io + + + + diff --git a/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/api/AlertPlugin.java b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/api/AlertPlugin.java new file mode 100644 index 0000000000..deb7ff6aa4 --- /dev/null +++ b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/api/AlertPlugin.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.api; + +import org.apache.dolphinscheduler.plugin.model.AlertInfo; +import org.apache.dolphinscheduler.plugin.model.PluginName; + +import java.util.Map; + +/** + * Plugin + */ +public interface AlertPlugin { + + /** + * Get alert plugin id + * + * @return alert plugin id, which should be unique + */ + String getId(); + + /** + * Get alert plugin name, which will show in front end portal + * + * @return plugin name + */ + PluginName getName(); + + Map process(AlertInfo info); + +} diff --git a/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/AlertData.java b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/AlertData.java new file mode 100644 index 0000000000..89ab5c4279 --- /dev/null +++ b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/AlertData.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.model; + +/** + * AlertData + */ +public class AlertData { + + /** + * alert primary key + */ + private int id; + /** + * title + */ + private String title; + /** + * content + */ + private String content; + /** + * log + */ + private String log; + /** + * alertgroup_id + */ + private int alertGroupId; + /** + * receivers + */ + private String receivers; + /** + * show_type + */ + private String showType; + /** + * receivers_cc + */ + private String receiversCc; + + public int getId() { + return id; + } + + public AlertData setId(int id) { + this.id = id; + return this; + } + + public String getTitle() { + return title; + } + + public AlertData setTitle(String title) { + this.title = title; + return this; + } + + public String getContent() { + return content; + } + + public AlertData setContent(String content) { + this.content = content; + return this; + } + + public String getLog() { + return log; + } + + public AlertData setLog(String log) { + this.log = log; + return this; + } + + public int getAlertGroupId() { + return alertGroupId; + } + + public AlertData setAlertGroupId(int alertGroupId) { + this.alertGroupId = alertGroupId; + return this; + } + + public String getReceivers() { + return receivers; + } + + public AlertData setReceivers(String receivers) { + this.receivers = receivers; + return this; + } + + public String getReceiversCc() { + return receiversCc; + } + + public AlertData setReceiversCc(String receiversCc) { + this.receiversCc = receiversCc; + return this; + } + + public String getShowType() { + return showType; + } + + public AlertData setShowType(String showType) { + this.showType = showType; + return this; + } + +} diff --git a/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/AlertInfo.java b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/AlertInfo.java new file mode 100644 index 0000000000..1d71ed7d8a --- /dev/null +++ b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/AlertInfo.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.model; + +import java.util.HashMap; +import java.util.Map; + +/** + * AlertInfo + */ +public class AlertInfo { + + private Map alertProps; + + private AlertData alertData; + + public AlertInfo() { + this.alertProps = new HashMap<>(); + } + + public Map getAlertProps() { + return alertProps; + } + + public AlertInfo setAlertProps(Map alertProps) { + this.alertProps = alertProps; + return this; + } + + public AlertInfo addProp(String key, Object value) { + this.alertProps.put(key, value); + return this; + } + + public Object getProp(String key) { + return this.alertProps.get(key); + } + + public AlertData getAlertData() { + return alertData; + } + + public AlertInfo setAlertData(AlertData alertData) { + this.alertData = alertData; + return this; + } +} diff --git a/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/PluginName.java b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/PluginName.java new file mode 100644 index 0000000000..8066e45f1d --- /dev/null +++ b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/PluginName.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.model; + +/** + * PluginName + */ +public class PluginName { + + private String chinese; + + private String english; + + public String getChinese() { + return chinese; + } + + public PluginName setChinese(String chinese) { + this.chinese = chinese; + return this; + } + + public String getEnglish() { + return english; + } + + public PluginName setEnglish(String english) { + this.english = english; + return this; + } +} diff --git a/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/spi/AlertPluginProvider.java b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/spi/AlertPluginProvider.java new file mode 100644 index 0000000000..594636f4eb --- /dev/null +++ b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/spi/AlertPluginProvider.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.spi; + +import org.apache.dolphinscheduler.plugin.api.AlertPlugin; + +/** + * PluginProvider + */ +public interface AlertPluginProvider { + + /** + * create an alert plugin + * + * @return an alert plugin + */ + AlertPlugin createPlugin(); + +} diff --git a/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/utils/PropertyUtils.java b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/utils/PropertyUtils.java new file mode 100644 index 0000000000..a244dd491e --- /dev/null +++ b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/utils/PropertyUtils.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.utils; + +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +/** + * property utils + * single instance + */ +public class PropertyUtils { + + private static final Logger logger = LoggerFactory.getLogger(PropertyUtils.class); + + private static final Properties properties = new Properties(); + + private PropertyUtils() { + throw new IllegalStateException("PropertyUtils class"); + } + + static { + String propertyFiles = "/plugin.properties"; + InputStream fis = null; + try { + fis = PropertyUtils.class.getResourceAsStream(propertyFiles); + properties.load(fis); + } catch (IOException e) { + logger.error(e.getMessage(), e); + if (fis != null) { + IOUtils.closeQuietly(fis); + } + } finally { + IOUtils.closeQuietly(fis); + } + } + + /** + * get property value + * + * @param key property name + * @param defaultVal default value + * @return property value + */ + public static String getString(String key, String defaultVal) { + String val = properties.getProperty(key.trim()); + return val == null ? defaultVal : val; + } + + /** + * get property value + * + * @param key property name + * @return property value + */ + public static String getString(String key) { + if (key == null) { + return null; + } + return properties.getProperty(key.trim()); + } + + /** + * get property value + * + * @param key property name + * @return get property int value , if key == null, then return -1 + */ + public static int getInt(String key) { + return getInt(key, -1); + } + + /** + * get int + * + * @param key key + * @param defaultValue default value + * @return property value + */ + public static int getInt(String key, int defaultValue) { + String value = properties.getProperty(key.trim()); + if (value == null) { + return defaultValue; + } + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + logger.info(e.getMessage(), e); + } + return defaultValue; + } + + /** + * get property value + * + * @param key property name + * @return property value + */ + public static boolean getBoolean(String key) { + String value = properties.getProperty(key.trim()); + if (value == null) { + return false; + } + + return Boolean.parseBoolean(value); + } + + /** + * get property value + * + * @param key property name + * @param defaultValue default value + * @return property value + */ + public static Boolean getBoolean(String key, boolean defaultValue) { + String value = properties.getProperty(key.trim()); + if (value == null) { + return defaultValue; + } + + return Boolean.parseBoolean(value); + } + + /** + * get property long value + * + * @param key key + * @param defaultVal default value + * @return property value + */ + public static long getLong(String key, long defaultVal) { + String val = getString(key); + return val == null ? defaultVal : Long.parseLong(val); + } + + /** + * get long + * + * @param key key + * @return property value + */ + public static long getLong(String key) { + return getLong(key, -1); + } + + /** + * get double + * + * @param key key + * @param defaultVal default value + * @return property value + */ + public static double getDouble(String key, double defaultVal) { + String val = properties.getProperty(key.trim()); + return val == null ? defaultVal : Double.parseDouble(val); + } + + /** + * @param key key + * @param type type + * @param defaultValue default value + * @param T + * @return get enum value + */ + public > T getEnum(String key, Class type, + T defaultValue) { + String val = properties.getProperty(key.trim()); + return val == null ? defaultValue : Enum.valueOf(type, val); + } + +} diff --git a/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/model/AlertDataTest.java b/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/model/AlertDataTest.java new file mode 100644 index 0000000000..c19b5bc29a --- /dev/null +++ b/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/model/AlertDataTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.model; + +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class AlertDataTest { + + private AlertData alertData; + + @Before + public void before() { + alertData = new AlertData(); + alertData.setId(1) + .setContent("content") + .setShowType("email") + .setTitle("title") + .setReceivers("receivers") + .setReceiversCc("cc") + .setLog("log") + .setAlertGroupId(1); + } + + @Test + public void getId() { + assertEquals(1, alertData.getId()); + } + + @Test + public void getTitle() { + assertEquals("title", alertData.getTitle()); + } + + @Test + public void getContent() { + assertEquals("content", alertData.getContent()); + } + + @Test + public void getLog() { + assertEquals("log", alertData.getLog()); + } + + @Test + public void getAlertGroupId() { + assertEquals(1, alertData.getAlertGroupId()); + } + + @Test + public void getReceivers() { + assertEquals("receivers", alertData.getReceivers()); + } + + @Test + public void getReceiversCc() { + assertEquals("cc", alertData.getReceiversCc()); + } + + @Test + public void getShowType() { + assertEquals("email", alertData.getShowType()); + } +} \ No newline at end of file diff --git a/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/model/AlertInfoTest.java b/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/model/AlertInfoTest.java new file mode 100644 index 0000000000..13eb595ac3 --- /dev/null +++ b/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/model/AlertInfoTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.model; + +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.*; + +public class AlertInfoTest { + + private AlertInfo alertInfo; + + @Before + public void before() { + alertInfo = new AlertInfo(); + } + + @Test + public void getAlertProps() { + Map map = new HashMap<>(); + alertInfo.setAlertProps(map); + assertNotNull(alertInfo.getAlertProps()); + } + + @Test + public void getProp() { + alertInfo.addProp("k", "v"); + assertEquals("v", alertInfo.getProp("k")); + } + + @Test + public void getAlertData() { + alertInfo.setAlertData(new AlertData()); + assertNotNull(alertInfo.getAlertData()); + } +} \ No newline at end of file diff --git a/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/utils/PropertyUtilsTest.java b/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/utils/PropertyUtilsTest.java new file mode 100644 index 0000000000..614a7c009c --- /dev/null +++ b/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/utils/PropertyUtilsTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.utils; + +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.junit.Assert.*; + +public class PropertyUtilsTest { + + private static final Logger logger = LoggerFactory.getLogger(PropertyUtilsTest.class); + + /** + * Test getString + */ + @Test + public void testGetString() { + + String result = PropertyUtils.getString("test.string"); + logger.info(result); + assertEquals("teststring", result); + + //If key is null, then return null + result = PropertyUtils.getString(null); + assertNull(result); + } + + + /** + * Test getBoolean + */ + @Test + public void testGetBoolean() { + + //Expected true + Boolean result = PropertyUtils.getBoolean("test.true"); + assertTrue(result); + + //Expected false + result = PropertyUtils.getBoolean("test.false"); + assertFalse(result); + } + + /** + * Test getLong + */ + @Test + public void testGetLong() { + long result = PropertyUtils.getLong("test.long"); + assertSame(100L, result); + } + + /** + * Test getDouble + */ + @Test + public void testGetDouble() { + + //If key is undefine in alert.properties, and there is a defaultval, then return defaultval + double result = PropertyUtils.getDouble("abc", 5.0); + assertEquals(5.0, result, 0); + + result = PropertyUtils.getDouble("cba", 5.0); + assertEquals(3.1, result, 0.01); + } + +} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/alert.properties.j2 b/dolphinscheduler-plugin-api/src/test/resources/plugin.properties similarity index 90% rename from ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/alert.properties.j2 rename to dolphinscheduler-plugin-api/src/test/resources/plugin.properties index 73840b8c18..d2ea3831be 100644 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/alert.properties.j2 +++ b/dolphinscheduler-plugin-api/src/test/resources/plugin.properties @@ -15,6 +15,8 @@ # limitations under the License. # -{% for key, value in dolphin_alert_map.iteritems() -%} - {{key}}={{value}} -{% endfor %} \ No newline at end of file +test.string=teststring +test.false=false +test.true=true +cba=3.1 +test.long=100 \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java index 480d6657c2..3314789fdb 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java @@ -120,19 +120,31 @@ public class TaskPriorityQueueConsumer extends Thread{ Boolean result = false; while (Stopper.isRunning()){ try { - result = dispatcher.dispatch(executionContext); + result = dispatcher.dispatch(executionContext); } catch (ExecuteException e) { logger.error("dispatch error",e); ThreadUtils.sleep(SLEEP_TIME_MILLIS); } - if (result){ + if (result || taskInstanceIsFinalState(taskInstanceId)){ break; } } return result; } + + /** + * taskInstance is final state + * success，failure，kill，stop，pause，threadwaiting is final state + * @param taskInstanceId taskInstanceId + * @return taskInstance is final state + */ + public Boolean taskInstanceIsFinalState(int taskInstanceId){ + TaskInstance taskInstance = processService.findTaskInstanceById(taskInstanceId); + return taskInstance.getState().typeIsFinished(); + } + /** * get TaskExecutionContext * @param taskInstanceId taskInstanceId diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java index 9986b07319..105584fe99 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java @@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter; import org.apache.dolphinscheduler.common.thread.Stopper; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.remote.command.TaskKillRequestCommand; @@ -35,9 +36,12 @@ import org.apache.dolphinscheduler.server.master.cache.impl.TaskInstanceCacheMan import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; import org.apache.dolphinscheduler.server.master.dispatch.enums.ExecutorType; import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager; +import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.springframework.beans.factory.annotation.Autowired; import java.util.Date; +import java.util.Set; /** @@ -53,6 +57,12 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { private NettyExecutorManager nettyExecutorManager; + + /** + * zookeeper register center + */ + private ZookeeperRegistryCenter zookeeperRegistryCenter; + /** * constructor of MasterTaskExecThread * @param taskInstance task instance @@ -61,6 +71,7 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { super(taskInstance); this.taskInstanceCacheManager = SpringApplicationContext.getBean(TaskInstanceCacheManagerImpl.class); this.nettyExecutorManager = SpringApplicationContext.getBean(NettyExecutorManager.class); + this.zookeeperRegistryCenter = SpringApplicationContext.getBean(ZookeeperRegistryCenter.class); } /** @@ -175,6 +186,16 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { } alreadyKilled = true; + String taskInstanceWorkerGroup = taskInstance.getWorkerGroup(); + + // not exists + if (!existsValidWorkerGroup(taskInstanceWorkerGroup)){ + taskInstance.setState(ExecutionStatus.KILL); + taskInstance.setEndTime(new Date()); + processService.updateTaskInstance(taskInstance); + return; + } + TaskKillRequestCommand killCommand = new TaskKillRequestCommand(); killCommand.setTaskInstanceId(taskInstance.getId()); @@ -185,10 +206,33 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { nettyExecutorManager.executeDirectly(executionContext); - logger.info("master add kill task :{} id:{} to kill queue", + logger.info("master kill taskInstance name :{} taskInstance id:{}", taskInstance.getName(), taskInstance.getId() ); } + /** + * whether exists valid worker group + * @param taskInstanceWorkerGroup taskInstanceWorkerGroup + * @return whether exists + */ + public Boolean existsValidWorkerGroup(String taskInstanceWorkerGroup){ + Set workerGroups = zookeeperRegistryCenter.getWorkerGroupDirectly(); + // not worker group + if (CollectionUtils.isEmpty(workerGroups)){ + return false; + } + + // has worker group , but not taskInstance assigned worker group + if (!workerGroups.contains(taskInstanceWorkerGroup)){ + return false; + } + Set workers = zookeeperRegistryCenter.getWorkerGroupNodesDirectly(taskInstanceWorkerGroup); + if (CollectionUtils.isEmpty(workers)) { + return false; + } + return true; + } + /** * get task timeout parameter * @return TaskTimeoutParameter diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index 84e4e54a50..ccd4bb2214 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -457,7 +457,7 @@ public class SqlTask extends AbstractTask { String showTypeName = sqlParameters.getShowType().replace(COMMA,"").trim(); if(EnumUtils.isValidEnum(ShowType.class,showTypeName)){ Map mailResult = MailUtils.sendMails(receviersList, - receviersCcList, title, content, ShowType.valueOf(showTypeName)); + receviersCcList, title, content, ShowType.valueOf(showTypeName).getDescp()); if(!(boolean) mailResult.get(STATUS)){ throw new RuntimeException("send mail failed!"); } diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/dolphinscheduler-server/src/main/resources/config/install_config.conf index 4671be7371..cba117e048 100644 --- a/dolphinscheduler-server/src/main/resources/config/install_config.conf +++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf @@ -63,7 +63,8 @@ mailPassword="xxxxxxxxxx" # TLS mail protocol support starttlsEnable="false" -sslTrust="xxxxxxxxxx" +#note: sslTrust is the same as mailServerHost +sslTrust="smtp.exmail.qq.com" # SSL mail protocol support # note: The SSL protocol is enabled by default. diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java new file mode 100644 index 0000000000..b6f118a734 --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.consumer; + +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.enums.Priority; +import org.apache.dolphinscheduler.dao.entity.*; +import org.apache.dolphinscheduler.server.master.dispatch.ExecutorDispatcher; +import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager; +import org.apache.dolphinscheduler.server.registry.DependencyConfig; +import org.apache.dolphinscheduler.server.registry.ZookeeperNodeManager; +import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; +import org.apache.dolphinscheduler.server.zk.SpringZKServer; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; +import org.apache.dolphinscheduler.service.queue.TaskPriorityQueue; +import org.apache.dolphinscheduler.service.queue.TaskPriorityQueueImpl; +import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; +import org.apache.dolphinscheduler.service.zk.ZookeeperConfig; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.util.Date; + + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes={DependencyConfig.class, SpringApplicationContext.class, SpringZKServer.class, + NettyExecutorManager.class, ExecutorDispatcher.class, ZookeeperRegistryCenter.class, TaskPriorityQueueConsumer.class, + ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class}) +public class TaskPriorityQueueConsumerTest { + + + @Autowired + private TaskPriorityQueue taskPriorityQueue; + + @Autowired + private TaskPriorityQueueConsumer taskPriorityQueueConsumer; + + @Autowired + private ProcessService processService; + + @Autowired + private ExecutorDispatcher dispatcher; + + @Before + public void init(){ + + Tenant tenant = new Tenant(); + tenant.setId(1); + tenant.setTenantCode("journey"); + tenant.setTenantName("journey"); + tenant.setDescription("journey"); + tenant.setQueueId(1); + tenant.setCreateTime(new Date()); + tenant.setUpdateTime(new Date()); + + Mockito.when(processService.getTenantForProcess(1,2)).thenReturn(tenant); + + Mockito.when(processService.queryUserQueueByProcessInstanceId(1)).thenReturn("default"); + } + + + @Test + public void testSHELLTask() throws Exception { + TaskInstance taskInstance = new TaskInstance(); + taskInstance.setId(1); + taskInstance.setTaskType("SHELL"); + taskInstance.setProcessDefinitionId(1); + taskInstance.setProcessInstanceId(1); + taskInstance.setState(ExecutionStatus.KILL); + taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\",\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"tasks-55201\",\"maxRetryTimes\":0,\"name\":\"测试任务\",\"params\":\"{\\\"rawScript\\\":\\\"echo \\\\\\\"测试任务\\\\\\\"\\\",\\\"localParams\\\":[],\\\"resourceList\\\":[]}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"SHELL\",\"workerGroup\":\"default\"}"); + taskInstance.setProcessInstancePriority(Priority.MEDIUM); + taskInstance.setWorkerGroup("default"); + taskInstance.setExecutorId(2); + + ProcessInstance processInstance = new ProcessInstance(); + processInstance.setTenantId(1); + processInstance.setCommandType(CommandType.START_PROCESS); + taskInstance.setProcessInstance(processInstance); + + ProcessDefinition processDefinition = new ProcessDefinition(); + processDefinition.setUserId(2); + processDefinition.setProjectId(1); + taskInstance.setProcessDefine(processDefinition); + + Mockito.when(processService.getTaskInstanceDetailByTaskId(1)).thenReturn(taskInstance); + taskPriorityQueue.put("2_1_2_1_default"); + + Thread.sleep(10000); + } + + + @Test + public void testSQLTask() throws Exception { + TaskInstance taskInstance = new TaskInstance(); + taskInstance.setId(1); + taskInstance.setTaskType("SQL"); + taskInstance.setProcessDefinitionId(1); + taskInstance.setProcessInstanceId(1); + taskInstance.setState(ExecutionStatus.KILL); + taskInstance.setTaskJson("{\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"tasks-3655\",\"maxRetryTimes\":0,\"name\":\"UDF测试\",\"params\":\"{\\\"postStatements\\\":[],\\\"connParams\\\":\\\"\\\",\\\"receiversCc\\\":\\\"\\\",\\\"udfs\\\":\\\"1\\\",\\\"type\\\":\\\"HIVE\\\",\\\"title\\\":\\\"test\\\",\\\"sql\\\":\\\"select id,name,ds,zodia(ds) from t_journey_user\\\",\\\"preStatements\\\":[],\\\"sqlType\\\":0,\\\"receivers\\\":\\\"825193156@qq.com\\\",\\\"datasource\\\":3,\\\"showType\\\":\\\"TABLE\\\",\\\"localParams\\\":[]}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"SQL\"}"); + taskInstance.setProcessInstancePriority(Priority.MEDIUM); + taskInstance.setWorkerGroup("default"); + taskInstance.setExecutorId(2); + + ProcessInstance processInstance = new ProcessInstance(); + processInstance.setTenantId(1); + processInstance.setCommandType(CommandType.START_PROCESS); + taskInstance.setProcessInstance(processInstance); + + ProcessDefinition processDefinition = new ProcessDefinition(); + processDefinition.setUserId(2); + processDefinition.setProjectId(1); + taskInstance.setProcessDefine(processDefinition); + + Mockito.when(processService.getTaskInstanceDetailByTaskId(1)).thenReturn(taskInstance); + taskPriorityQueue.put("2_1_2_1_default"); + + DataSource dataSource = new DataSource(); + dataSource.setId(1); + dataSource.setName("sqlDatasource"); + dataSource.setType(DbType.MYSQL); + dataSource.setUserId(2); + dataSource.setConnectionParams("{\"address\":\"jdbc:mysql://192.168.221.185:3306\",\"database\":\"dolphinscheduler_qiaozhanwei\",\"jdbcUrl\":\"jdbc:mysql://192.168.221.185:3306/dolphinscheduler_qiaozhanwei\",\"user\":\"root\",\"password\":\"root@123\"}"); + dataSource.setCreateTime(new Date()); + dataSource.setUpdateTime(new Date()); + + Mockito.when(processService.findDataSourceById(1)).thenReturn(dataSource); + + Thread.sleep(10000); + } + + + @Test + public void testDataxTask() throws Exception { + TaskInstance taskInstance = new TaskInstance(); + taskInstance.setId(1); + taskInstance.setTaskType("DATAX"); + taskInstance.setProcessDefinitionId(1); + taskInstance.setProcessInstanceId(1); + taskInstance.setState(ExecutionStatus.KILL); + taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\",\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"tasks-97625\",\"maxRetryTimes\":0,\"name\":\"MySQL数据相互导入\",\"params\":\"{\\\"targetTable\\\":\\\"pv2\\\",\\\"postStatements\\\":[],\\\"jobSpeedRecord\\\":1000,\\\"customConfig\\\":0,\\\"dtType\\\":\\\"MYSQL\\\",\\\"dsType\\\":\\\"MYSQL\\\",\\\"jobSpeedByte\\\":0,\\\"dataSource\\\":80,\\\"dataTarget\\\":80,\\\"sql\\\":\\\"SELECT dt,count FROM pv\\\",\\\"preStatements\\\":[]}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"DATAX\",\"workerGroup\":\"default\"}"); + taskInstance.setProcessInstancePriority(Priority.MEDIUM); + taskInstance.setWorkerGroup("default"); + taskInstance.setExecutorId(2); + + ProcessInstance processInstance = new ProcessInstance(); + processInstance.setTenantId(1); + processInstance.setCommandType(CommandType.START_PROCESS); + taskInstance.setProcessInstance(processInstance); + + ProcessDefinition processDefinition = new ProcessDefinition(); + processDefinition.setUserId(2); + processDefinition.setProjectId(1); + taskInstance.setProcessDefine(processDefinition); + + Mockito.when(processService.getTaskInstanceDetailByTaskId(1)).thenReturn(taskInstance); + taskPriorityQueue.put("2_1_2_1_default"); + + + + DataSource dataSource = new DataSource(); + dataSource.setId(80); + dataSource.setName("datax"); + dataSource.setType(DbType.MYSQL); + dataSource.setUserId(2); + dataSource.setConnectionParams("{\"address\":\"jdbc:mysql://192.168.221.185:3306\",\"database\":\"dolphinscheduler_qiaozhanwei\",\"jdbcUrl\":\"jdbc:mysql://192.168.221.185:3306/dolphinscheduler_qiaozhanwei\",\"user\":\"root\",\"password\":\"root@123\"}"); + dataSource.setCreateTime(new Date()); + dataSource.setUpdateTime(new Date()); + + Mockito.when(processService.findDataSourceById(80)).thenReturn(dataSource); + + Thread.sleep(10000); + } + + + @Test + public void testSqoopTask() throws Exception { + TaskInstance taskInstance = new TaskInstance(); + taskInstance.setId(1); + taskInstance.setTaskType("SQOOP"); + taskInstance.setProcessDefinitionId(1); + taskInstance.setProcessInstanceId(1); + taskInstance.setState(ExecutionStatus.KILL); + taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\",\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"tasks-63634\",\"maxRetryTimes\":0,\"name\":\"MySQL数据导入HDSF\",\"params\":\"{\\\"sourceType\\\":\\\"MYSQL\\\",\\\"targetType\\\":\\\"HDFS\\\",\\\"targetParams\\\":\\\"{\\\\\\\"targetPath\\\\\\\":\\\\\\\"/test/datatest\\\\\\\",\\\\\\\"deleteTargetDir\\\\\\\":true,\\\\\\\"fileType\\\\\\\":\\\\\\\"--as-textfile\\\\\\\",\\\\\\\"compressionCodec\\\\\\\":\\\\\\\"\\\\\\\",\\\\\\\"fieldsTerminated\\\\\\\":\\\\\\\",\\\\\\\",\\\\\\\"linesTerminated\\\\\\\":\\\\\\\"\\\\\\\\\\\\\\\\n\\\\\\\"}\\\",\\\"modelType\\\":\\\"import\\\",\\\"sourceParams\\\":\\\"{\\\\\\\"srcType\\\\\\\":\\\\\\\"MYSQL\\\\\\\",\\\\\\\"srcDatasource\\\\\\\":1,\\\\\\\"srcTable\\\\\\\":\\\\\\\"t_ds_user\\\\\\\",\\\\\\\"srcQueryType\\\\\\\":\\\\\\\"0\\\\\\\",\\\\\\\"srcQuerySql\\\\\\\":\\\\\\\"\\\\\\\",\\\\\\\"srcColumnType\\\\\\\":\\\\\\\"0\\\\\\\",\\\\\\\"srcColumns\\\\\\\":\\\\\\\"\\\\\\\",\\\\\\\"srcConditionList\\\\\\\":[],\\\\\\\"mapColumnHive\\\\\\\":[],\\\\\\\"mapColumnJava\\\\\\\":[]}\\\",\\\"localParams\\\":[],\\\"concurrency\\\":1}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"SQOOP\",\"workerGroup\":\"default\"}"); + taskInstance.setProcessInstancePriority(Priority.MEDIUM); + taskInstance.setWorkerGroup("default"); + taskInstance.setExecutorId(2); + + ProcessInstance processInstance = new ProcessInstance(); + processInstance.setTenantId(1); + processInstance.setCommandType(CommandType.START_PROCESS); + taskInstance.setProcessInstance(processInstance); + + ProcessDefinition processDefinition = new ProcessDefinition(); + processDefinition.setUserId(2); + processDefinition.setProjectId(1); + taskInstance.setProcessDefine(processDefinition); + + Mockito.when(processService.getTaskInstanceDetailByTaskId(1)).thenReturn(taskInstance); + taskPriorityQueue.put("2_1_2_1_default"); + + + + DataSource dataSource = new DataSource(); + dataSource.setId(1); + dataSource.setName("datax"); + dataSource.setType(DbType.MYSQL); + dataSource.setUserId(2); + dataSource.setConnectionParams("{\"address\":\"jdbc:mysql://192.168.221.185:3306\",\"database\":\"dolphinscheduler_qiaozhanwei\",\"jdbcUrl\":\"jdbc:mysql://192.168.221.185:3306/dolphinscheduler_qiaozhanwei\",\"user\":\"root\",\"password\":\"root@123\"}"); + dataSource.setCreateTime(new Date()); + dataSource.setUpdateTime(new Date()); + + Mockito.when(processService.findDataSourceById(1)).thenReturn(dataSource); + + Thread.sleep(10000); + } + + + @Test + public void testTaskInstanceIsFinalState(){ + TaskInstance taskInstance = new TaskInstance(); + taskInstance.setId(1); + taskInstance.setTaskType("SHELL"); + taskInstance.setProcessDefinitionId(1); + taskInstance.setProcessInstanceId(1); + taskInstance.setState(ExecutionStatus.KILL); + taskInstance.setTaskJson("{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\",\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false,\"id\":\"tasks-55201\",\"maxRetryTimes\":0,\"name\":\"测试任务\",\"params\":\"{\\\"rawScript\\\":\\\"echo \\\\\\\"测试任务\\\\\\\"\\\",\\\"localParams\\\":[],\\\"resourceList\\\":[]}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\",\"taskInstancePriority\":\"MEDIUM\",\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0},\"timeout\":\"{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}\",\"type\":\"SHELL\",\"workerGroup\":\"default\"}"); + taskInstance.setProcessInstancePriority(Priority.MEDIUM); + taskInstance.setWorkerGroup("default"); + taskInstance.setExecutorId(2); + + + Mockito.when( processService.findTaskInstanceById(1)).thenReturn(taskInstance); + + taskPriorityQueueConsumer.taskInstanceIsFinalState(1); + } + + +} diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java new file mode 100644 index 0000000000..ebddec4fb1 --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.runner; + +import junit.framework.Assert; +import org.apache.dolphinscheduler.server.master.config.MasterConfig; +import org.apache.dolphinscheduler.server.master.consumer.TaskPriorityQueueConsumer; +import org.apache.dolphinscheduler.server.master.dispatch.ExecutorDispatcher; +import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager; +import org.apache.dolphinscheduler.server.registry.DependencyConfig; +import org.apache.dolphinscheduler.server.registry.ZookeeperNodeManager; +import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; +import org.apache.dolphinscheduler.server.zk.SpringZKServer; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.queue.TaskPriorityQueueImpl; +import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; +import org.apache.dolphinscheduler.service.zk.ZookeeperConfig; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.util.HashSet; +import java.util.Set; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes={DependencyConfig.class, SpringApplicationContext.class, SpringZKServer.class, + NettyExecutorManager.class, ExecutorDispatcher.class, ZookeeperRegistryCenter.class, TaskPriorityQueueConsumer.class, + ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class, MasterConfig.class}) +public class MasterTaskExecThreadTest { + + @Test + public void testExistsValidWorkerGroup1(){ + ZookeeperRegistryCenter zookeeperRegistryCenter = Mockito.mock(ZookeeperRegistryCenter.class); + Mockito.when(zookeeperRegistryCenter.getWorkerGroupDirectly()).thenReturn(null); + MasterTaskExecThread masterTaskExecThread = new MasterTaskExecThread(null); + masterTaskExecThread.existsValidWorkerGroup("default"); + } + @Test + public void testExistsValidWorkerGroup2(){ + ZookeeperRegistryCenter zookeeperRegistryCenter = Mockito.mock(ZookeeperRegistryCenter.class); + Set workerGorups = new HashSet<>(); + workerGorups.add("test1"); + workerGorups.add("test2"); + + Mockito.when(zookeeperRegistryCenter.getWorkerGroupDirectly()).thenReturn(workerGorups); + MasterTaskExecThread masterTaskExecThread = new MasterTaskExecThread(null); + masterTaskExecThread.existsValidWorkerGroup("default"); + } + + @Test + public void testExistsValidWorkerGroup3(){ + ZookeeperRegistryCenter zookeeperRegistryCenter = Mockito.mock(ZookeeperRegistryCenter.class); + Set workerGorups = new HashSet<>(); + workerGorups.add("test1"); + + Mockito.when(zookeeperRegistryCenter.getWorkerGroupDirectly()).thenReturn(workerGorups); + Mockito.when(zookeeperRegistryCenter.getWorkerGroupNodesDirectly("test1")).thenReturn(workerGorups); + MasterTaskExecThread masterTaskExecThread = new MasterTaskExecThread(null); + masterTaskExecThread.existsValidWorkerGroup("test1"); + } + + +} diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/DependencyConfig.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/DependencyConfig.java index 0adea44cfd..93d2b03010 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/DependencyConfig.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/DependencyConfig.java @@ -20,11 +20,14 @@ package org.apache.dolphinscheduler.server.registry; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.mapper.*; import org.apache.dolphinscheduler.server.master.cache.impl.TaskInstanceCacheManagerImpl; +import org.apache.dolphinscheduler.server.master.dispatch.ExecutorDispatcher; import org.apache.dolphinscheduler.server.master.dispatch.host.HostManager; import org.apache.dolphinscheduler.server.master.dispatch.host.RandomHostManager; import org.apache.dolphinscheduler.server.master.processor.queue.TaskResponseService; import org.apache.dolphinscheduler.server.worker.processor.TaskCallbackService; import org.apache.dolphinscheduler.service.process.ProcessService; +import org.apache.dolphinscheduler.service.queue.TaskPriorityQueue; +import org.apache.dolphinscheduler.service.queue.TaskPriorityQueueImpl; import org.mockito.Mockito; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -144,4 +147,9 @@ public class DependencyConfig { public TaskResponseService taskResponseService(){ return Mockito.mock(TaskResponseService.class); } + + @Bean + public TaskPriorityQueue taskPriorityQueue(){ + return new TaskPriorityQueueImpl(); + } } diff --git a/dolphinscheduler-ui/package.json b/dolphinscheduler-ui/package.json index b23969803b..5dc1cb44db 100644 --- a/dolphinscheduler-ui/package.json +++ b/dolphinscheduler-ui/package.json @@ -55,7 +55,7 @@ "html-loader": "^0.5.5", "html-webpack-plugin": "^3.2.0", "mini-css-extract-plugin": "^0.8.2", - "node-sass": "^4.13.1", + "node-sass": "^4.14.0", "postcss-loader": "^3.0.0", "progress-bar-webpack-plugin": "^1.12.1", "rimraf": "^2.6.2", diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue index 6f630071c1..ae15e20f74 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue @@ -259,8 +259,15 @@ if (v2.name === v1.name) { let dom = $(`#${v2.id}`) let state = dom.find('.state-p') + let depState = '' + taskList.forEach(item=>{ + if(item.name==v1.name) { + depState = item.state + } + }) dom.attr('data-state-id', v1.stateId) dom.attr('data-dependent-result', v1.dependentResult || '') + dom.attr('data-dependent-depState', depState) state.append(``) state.find('strong').attr('title', titleTpl(v2, v1.desc)) } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue index 0c3f7433a3..549cb119f9 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue @@ -28,8 +28,8 @@ @@ -176,8 +176,7 @@ } }) }, - mounted () { - }, + mounted () {}, components: {} } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue index fb3f2c295c..621de81037 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue @@ -143,13 +143,20 @@ created () { let o = this.backfillItem let dependentResult = $(`#${o.id}`).data('dependent-result') || {} + // Does not represent an empty object backfill if (!_.isEmpty(o)) { this.relation = _.cloneDeep(o.dependence.relation) || 'AND' this.dependTaskList = _.cloneDeep(o.dependence.dependTaskList) || [] let defaultState = this.isDetails ? 'WAITING' : '' // Process instance return status display matches by key - _.map(this.dependTaskList, v => _.map(v.dependItemList, v1 => v1.state = dependentResult[`${v1.definitionId}-${v1.depTasks}-${v1.cycle}-${v1.dateValue}`] || defaultState)) + _.map(this.dependTaskList, v => _.map(v.dependItemList, v1 => { + $(`#${o.id}`).siblings().each(function(){ + if(v1.depTasks == $(this).text()) { + v1.state = $(this).attr('data-dependent-depstate') + } + }); + })) } }, mounted () { diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue index 95bdc2930c..dd4530786d 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue @@ -19,34 +19,36 @@
- - - - - - - - - - diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/index.vue index 1f28578504..5fcb3f6f21 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/index.vue @@ -15,29 +15,30 @@ * limitations under the License. */ @@ -66,6 +66,7 @@ pageNo: 1, searchVal: '' }, + isLeft: true, isADMIN: store.state.user.userInfo.userType === 'ADMIN_USER' } }, @@ -116,6 +117,11 @@ }) }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getQueueListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -143,6 +149,9 @@ mounted () { this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mList, mListConstruction, mConditions, mSpin, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/tenement/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/tenement/index.vue index 0c38f0a911..ca180b1718 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/tenement/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/tenement/index.vue @@ -39,7 +39,7 @@ - + @@ -67,6 +67,7 @@ pageNo: 1, searchVal: '' }, + isLeft: true, isADMIN: store.state.user.userInfo.userType === 'ADMIN_USER' } }, @@ -120,6 +121,11 @@ }) }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getTenantListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -147,6 +153,9 @@ mounted () { this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mList, mListConstruction, mConditions, mSpin, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue index 39dc27855a..7a1cf5bca4 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue @@ -37,7 +37,7 @@ @@ -65,7 +65,8 @@ pageSize: 10, pageNo: 1, searchVal: '' - } + }, + isLeft: true } }, mixins: [listUrlParamHandle], @@ -118,6 +119,11 @@ }) }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getUsersListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -145,6 +151,9 @@ mounted () { this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mList, mListConstruction, mConditions, mSpin, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/warningGroups/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/warningGroups/index.vue index e70ead46ce..ad604ba3a6 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/warningGroups/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/warningGroups/index.vue @@ -39,7 +39,7 @@ - + @@ -67,6 +67,7 @@ pageNo: 1, searchVal: '' }, + isLeft: true, isADMIN: store.state.user.userInfo.userType === 'ADMIN_USER' } }, @@ -120,6 +121,11 @@ }) }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getAlertgroupP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -147,6 +153,9 @@ mounted () { this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mList, mListConstruction, mConditions, mSpin, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/token/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/token/index.vue index 21d2becd32..3398acca13 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/token/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/token/index.vue @@ -40,7 +40,7 @@ - + @@ -67,7 +67,8 @@ pageSize: 10, pageNo: 1, searchVal: '' - } + }, + isLeft: true } }, mixins: [listUrlParamHandle], @@ -120,6 +121,11 @@ }) }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getTokenListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -147,6 +153,9 @@ mounted () { this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mSecondaryMenu, mList, mListConstruction, mConditions, mSpin, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/login/App.vue b/dolphinscheduler-ui/src/js/conf/login/App.vue index a79743a262..8d065920a8 100644 --- a/dolphinscheduler-ui/src/js/conf/login/App.vue +++ b/dolphinscheduler-ui/src/js/conf/login/App.vue @@ -84,6 +84,7 @@ this._gLogin().then(res => { setTimeout(() => { this.spinnerLoading = false + sessionStorage.setItem('isLeft',1); if (res.data.hasOwnProperty("sessionId")) { let sessionId=res.data.sessionId sessionStorage.setItem("sessionId", sessionId) diff --git a/dolphinscheduler-ui/src/js/module/components/secondaryMenu/secondaryMenu.vue b/dolphinscheduler-ui/src/js/module/components/secondaryMenu/secondaryMenu.vue index 102df8f910..5336555c21 100644 --- a/dolphinscheduler-ui/src/js/module/components/secondaryMenu/secondaryMenu.vue +++ b/dolphinscheduler-ui/src/js/module/components/secondaryMenu/secondaryMenu.vue @@ -63,7 +63,8 @@ menuList: menu(this.type), index: 0, id: this.$route.params.id, - isTogHide: false + isTogHide: false, + isLeft: true } }, props: { @@ -82,6 +83,11 @@ }, _toggleMenu () { this.isTogHide = !this.isTogHide + if(this.isTogHide) { + sessionStorage.setItem('isLeft',0) + } else { + sessionStorage.setItem('isLeft',1) + } } }, mounted () { diff --git a/dolphinscheduler-ui/src/js/module/components/spin/spin.vue b/dolphinscheduler-ui/src/js/module/components/spin/spin.vue index 95fc8e924e..7c6a9c3acf 100644 --- a/dolphinscheduler-ui/src/js/module/components/spin/spin.vue +++ b/dolphinscheduler-ui/src/js/module/components/spin/spin.vue @@ -47,7 +47,7 @@ #spin-model { position: fixed; left: 20px; - top: 80px; + top: 60px; background: #fff; z-index: 99; border-radius: 3px; @@ -69,11 +69,11 @@ } &.spin-sp1 { width: calc(100% - 40px); - height: calc(100% - 100px); + height: calc(100% - 60px); } &.spin-sp2 { - width: calc(100% - 240px); - height: calc(100% - 100px); + width: calc(100% - 220px); + height: calc(100% - 60px); left: 220px; } } diff --git a/dolphinscheduler-ui/src/sass/conf/home/index.scss b/dolphinscheduler-ui/src/sass/conf/home/index.scss index 76b9f6da01..16f588363d 100644 --- a/dolphinscheduler-ui/src/sass/conf/home/index.scss +++ b/dolphinscheduler-ui/src/sass/conf/home/index.scss @@ -24,8 +24,13 @@ body { .ans-message-box,.ans-message-wrapper { z-index: 121 !important; } + &::-webkit-scrollbar { + width: 0; + height: 0; + } } + @media screen and (max-width: 960px){ .nav-model { .logo-box, diff --git a/pom.xml b/pom.xml index 0647724ed0..7e291d2c4e 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,6 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - 4.0.0 @@ -205,6 +204,11 @@ dolphinscheduler-server ${project.version} + + org.apache.dolphinscheduler + dolphinscheduler-plugin-api + ${project.version} + org.apache.dolphinscheduler dolphinscheduler-common @@ -690,6 +694,7 @@ **/alert/utils/FuncUtilsTest.java **/alert/utils/JSONUtilsTest.java **/alert/utils/MailUtilsTest.java + **/alert/plugin/EmailAlertPluginTest.java **/api/dto/resources/filter/ResourceFilterTest.java **/api/dto/resources/visitor/ResourceTreeVisitorTest.java **/api/enums/testGetEnum.java @@ -765,6 +770,8 @@ **/common/utils/HttpUtilsTest.java **/common/ConstantsTest.java **/common/utils/HadoopUtils.java + **/common/plugin/FilePluginManagerTest + **/common/plugin/PluginClassLoaderTest **/dao/mapper/AccessTokenMapperTest.java **/dao/mapper/AlertGroupMapperTest.java **/dao/mapper/CommandMapperTest.java @@ -778,6 +785,8 @@ **/server/log/TaskLogDiscriminatorTest.java **/server/log/TaskLogFilterTest.java **/server/log/WorkerLogFilterTest.java + **/server/master/consumer/TaskPriorityQueueConsumerTest.java + **/server/master/runner/MasterTaskExecThreadTest.java **/server/master/dispatch/executor/NettyExecutorManagerTest.java **/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java **/server/master/dispatch/host/assign/RandomSelectorTest.java @@ -829,6 +838,9 @@ **/dao/mapper/UserMapperTest.java **/dao/utils/DagHelperTest.java **/dao/AlertDaoTest.java + **/plugin/model/AlertDataTest.java + **/plugin/model/AlertInfoTest.java + **/plugin/utils/PropertyUtilsTest.java @@ -864,6 +876,22 @@ apache-rat-plugin ${apache.rat.version} + false + false + + + AL20 + Apache License, 2.0 + + Licensed to the Apache Software Foundation (ASF) + + + + + + Apache License, 2.0 + + **/node_modules/** **/node/** @@ -882,7 +910,7 @@ **/*.babelrc **/*.eslintrc **/.mvn/jvm.config - **/.mvn/wrapper/maven-wrapper.properties + **/.mvn/wrapper/** true @@ -972,6 +1000,6 @@ dolphinscheduler-dist dolphinscheduler-remote dolphinscheduler-service + dolphinscheduler-plugin-api - diff --git a/script/env/dolphinscheduler_env.sh b/script/env/dolphinscheduler_env.sh index e5b99e2857..026b620203 100644 --- a/script/env/dolphinscheduler_env.sh +++ b/script/env/dolphinscheduler_env.sh @@ -23,4 +23,6 @@ export PYTHON_HOME=/opt/soft/python export JAVA_HOME=/opt/soft/java export HIVE_HOME=/opt/soft/hive export FLINK_HOME=/opt/soft/flink -export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$PATH +export DATAX_HOME=/opt/soft/datax/bin/datax.py + +export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH diff --git a/tools/dependencies/check-LICENSE.sh b/tools/dependencies/check-LICENSE.sh new file mode 100755 index 0000000000..d414bd40c6 --- /dev/null +++ b/tools/dependencies/check-LICENSE.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +mkdir dist || true + +tar -zxf dolphinscheduler-dist/target/apache-dolphinscheduler*-bin.tar.gz --strip=1 -C dist + +# List all modules(jars) that belong to the DolphinScheduler itself, these will be ignored when checking the dependency +# licenses +echo '=== Self modules: ' && ./mvnw --batch-mode --quiet -Dexec.executable='echo' -Dexec.args='${project.artifactId}-${project.version}.jar' exec:exec | tee self-modules.txt + +echo '=== Distributed dependencies: ' && ls dist/lib | tee all-dependencies.txt + +# Exclude all self modules(jars) to generate all third-party dependencies +echo '=== Third party dependencies: ' && grep -vf self-modules.txt all-dependencies.txt | tee third-party-dependencies.txt + +# 1. Compare the third-party dependencies with known dependencies, expect that all third-party dependencies are KNOWN +# and the exit code of the command is 0, otherwise we should add its license to LICENSE file and add the dependency to +# known-dependencies.txt. 2. Unify the `sort` behaviour: here we'll sort them again in case that the behaviour of `sort` +# command in target OS is different from what we used to sort the file `known-dependencies.txt`, i.e. "sort the two file +# using the same command (and default arguments)" + +diff -w -B -U0 <(sort < tools/dependencies/known-dependencies.txt) <(sort < third-party-dependencies.txt) diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt new file mode 100755 index 0000000000..6601cf9c34 --- /dev/null +++ b/tools/dependencies/known-dependencies.txt @@ -0,0 +1,211 @@ +HikariCP-3.2.0.jar +activation-1.1.jar +ant-1.6.5.jar +aopalliance-1.0.jar +apache-el-8.5.35.1.jar +apacheds-i18n-2.0.0-M15.jar +apacheds-kerberos-codec-2.0.0-M15.jar +api-asn1-api-1.0.0-M20.jar +api-util-1.0.0-M20.jar +asm-3.1.jar +aspectjweaver-1.9.2.jar +audience-annotations-0.5.0.jar +avro-1.7.4.jar +aws-java-sdk-1.7.4.jar +bonecp-0.8.0.RELEASE.jar +byte-buddy-1.9.10.jar +classmate-1.4.0.jar +clickhouse-jdbc-0.1.52.jar +commons-cli-1.2.jar +commons-codec-1.6.jar +commons-collections-3.2.2.jar +commons-collections4-4.1.jar +commons-compiler-3.0.12.jar +commons-compress-1.4.1.jar +commons-configuration-1.10.jar +commons-daemon-1.0.13.jar +commons-dbcp-1.4.jar +commons-email-1.5.jar +commons-httpclient-3.0.1.jar +commons-io-2.4.jar +commons-lang-2.6.jar +commons-logging-1.1.1.jar +commons-math3-3.1.1.jar +commons-net-3.1.jar +commons-pool-1.6.jar +core-3.1.1.jar +cron-utils-5.0.5.jar +curator-client-4.3.0.jar +curator-framework-4.3.0.jar +curator-recipes-4.3.0.jar +datanucleus-api-jdo-4.2.1.jar +datanucleus-core-4.1.6.jar +datanucleus-rdbms-4.1.7.jar +derby-10.14.2.0.jar +druid-1.1.14.jar +fastjson-1.2.61.jar +gson-2.8.5.jar +guava-20.0.jar +guice-3.0.jar +guice-servlet-3.0.jar +h2-1.4.200.jar +hadoop-annotations-2.7.3.jar +hadoop-auth-2.7.3.jar +hadoop-aws-2.7.3.jar +hadoop-client-2.7.3.jar +hadoop-common-2.7.3.jar +hadoop-hdfs-2.7.3.jar +hadoop-mapreduce-client-app-2.7.3.jar +hadoop-mapreduce-client-common-2.7.3.jar +hadoop-mapreduce-client-core-2.7.3.jar +hadoop-mapreduce-client-jobclient-2.7.3.jar +hadoop-mapreduce-client-shuffle-2.7.3.jar +hadoop-yarn-api-2.7.3.jar +hadoop-yarn-client-2.7.3.jar +hadoop-yarn-common-2.7.3.jar +hadoop-yarn-server-common-2.7.3.jar +hamcrest-core-1.3.jar +hibernate-validator-6.0.14.Final.jar +hive-common-2.1.0.jar +hive-jdbc-2.1.0.jar +hive-metastore-2.1.0.jar +hive-orc-2.1.0.jar +hive-serde-2.1.0.jar +hive-service-2.1.0.jar +hive-service-rpc-2.1.0.jar +hive-storage-api-2.1.0.jar +htrace-core-3.1.0-incubating.jar +httpclient-4.4.1.jar +httpcore-4.4.1.jar +httpmime-4.5.7.jar +jackson-annotations-2.9.8.jar +jackson-core-2.9.8.jar +jackson-core-asl-1.9.13.jar +jackson-databind-2.9.8.jar +jackson-datatype-jdk8-2.9.8.jar +jackson-datatype-jsr310-2.9.8.jar +jackson-jaxrs-1.9.13.jar +jackson-mapper-asl-1.9.13.jar +jackson-module-parameter-names-2.9.8.jar +jackson-xc-1.9.13.jar +jamon-runtime-2.3.1.jar +janino-3.0.12.jar +java-xmlbuilder-0.4.jar +javax.activation-api-1.2.0.jar +javax.annotation-api-1.3.2.jar +javax.inject-1.jar +javax.jdo-3.2.0-m3.jar +javax.mail-1.6.2.jar +javax.servlet-api-3.1.0.jar +javolution-5.5.1.jar +jaxb-api-2.3.1.jar +jaxb-impl-2.2.3-1.jar +jboss-logging-3.3.2.Final.jar +jdo-api-3.0.1.jar +jersey-client-1.9.jar +jersey-core-1.9.jar +jersey-guice-1.9.jar +jersey-json-1.9.jar +jersey-server-1.9.jar +jets3t-0.9.0.jar +jettison-1.1.jar +jetty-6.1.26.jar +jetty-continuation-9.4.14.v20181114.jar +jetty-http-9.4.14.v20181114.jar +jetty-io-9.4.14.v20181114.jar +jetty-security-9.4.14.v20181114.jar +jetty-server-9.4.14.v20181114.jar +jetty-servlet-9.4.14.v20181114.jar +jetty-servlets-9.4.14.v20181114.jar +jetty-util-6.1.26.jar +jetty-util-9.4.14.v20181114.jar +jetty-webapp-9.4.14.v20181114.jar +jetty-xml-9.4.14.v20181114.jar +jline-0.9.94.jar +jna-4.5.2.jar +jna-platform-4.5.2.jar +joda-time-2.10.1.jar +jpam-1.1.jar +jsch-0.1.42.jar +jsp-2.1-6.1.14.jar +jsp-api-2.1-6.1.14.jar +jsp-api-2.1.jar +jsqlparser-2.1.jar +jsr305-3.0.0.jar +jta-1.1.jar +jul-to-slf4j-1.7.25.jar +junit-4.12.jar +leveldbjni-all-1.8.jar +libfb303-0.9.3.jar +libthrift-0.9.3.jar +log4j-1.2-api-2.11.2.jar +log4j-1.2.17.jar +log4j-api-2.11.2.jar +log4j-core-2.11.2.jar +logback-classic-1.2.3.jar +logback-core-1.2.3.jar +lz4-1.3.0.jar +mapstruct-1.2.0.Final.jar +mssql-jdbc-6.1.0.jre8.jar +mybatis-3.5.2.jar +mybatis-plus-3.2.0.jar +mybatis-plus-annotation-3.2.0.jar +mybatis-plus-boot-starter-3.2.0.jar +mybatis-plus-core-3.2.0.jar +mybatis-plus-extension-3.2.0.jar +mybatis-spring-2.0.2.jar +netty-3.6.2.Final.jar +netty-all-4.1.33.Final.jar +opencsv-2.3.jar +oshi-core-3.5.0.jar +paranamer-2.3.jar +parquet-hadoop-bundle-1.8.1.jar +poi-3.17.jar +postgresql-42.1.4.jar +protobuf-java-2.5.0.jar +quartz-2.2.3.jar +quartz-jobs-2.2.3.jar +slf4j-api-1.7.5.jar +snakeyaml-1.23.jar +snappy-0.2.jar +snappy-java-1.0.4.1.jar +spring-aop-5.1.5.RELEASE.jar +spring-beans-5.1.5.RELEASE.jar +spring-boot-2.1.3.RELEASE.jar +spring-boot-autoconfigure-2.1.3.RELEASE.jar +spring-boot-starter-2.1.3.RELEASE.jar +spring-boot-starter-aop-2.1.3.RELEASE.jar +spring-boot-starter-jdbc-2.1.3.RELEASE.jar +spring-boot-starter-jetty-2.1.3.RELEASE.jar +spring-boot-starter-json-2.1.3.RELEASE.jar +spring-boot-starter-logging-2.1.3.RELEASE.jar +spring-boot-starter-web-2.1.3.RELEASE.jar +spring-context-5.1.5.RELEASE.jar +spring-core-5.1.5.RELEASE.jar +spring-expression-5.1.5.RELEASE.jar +spring-jcl-5.1.5.RELEASE.jar +spring-jdbc-5.1.5.RELEASE.jar +spring-plugin-core-1.2.0.RELEASE.jar +spring-plugin-metadata-1.2.0.RELEASE.jar +spring-tx-5.1.5.RELEASE.jar +spring-web-5.1.5.RELEASE.jar +spring-webmvc-5.1.5.RELEASE.jar +springfox-core-2.9.2.jar +springfox-schema-2.9.2.jar +springfox-spi-2.9.2.jar +springfox-spring-web-2.9.2.jar +springfox-swagger-common-2.9.2.jar +springfox-swagger-ui-2.9.2.jar +springfox-swagger2-2.9.2.jar +swagger-annotations-1.5.20.jar +swagger-bootstrap-ui-1.9.3.jar +swagger-models-1.5.20.jar +tephra-api-0.6.0.jar +threetenbp-1.3.6.jar +transaction-api-1.1.jar +validation-api-2.0.1.Final.jar +xercesImpl-2.9.1.jar +xml-apis-1.4.01.jar +xmlenc-0.52.jar +xz-1.0.jar +zookeeper-3.4.14.jar
+ + {{$t('#')}} + {{$t('Process Name')}} + {{$t('State')}} + {{$t('Create Time')}} + {{$t('Update Time')}} + {{$t('Description')}} + {{$t('Modify User')}} - {{$t('Timing state')}} + +
+ {{$t('Timing state')}} +
+ {{$t('Operation')}}