diff --git a/.github/workflows/ci_backend.yml b/.github/workflows/ci_backend.yml index 0273251e99..c24a3ecf07 100644 --- a/.github/workflows/ci_backend.yml +++ b/.github/workflows/ci_backend.yml @@ -56,21 +56,9 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.8 + - name: Check license + run: ./mvnw -B apache-rat:check - name: Compile - run: mvn -B clean compile package -Prelease -Dmaven.test.skip=true - License-check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. - - name: checkout submodules - shell: bash - run: | - git submodule sync --recursive - git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - name: Check - run: mvn -B apache-rat:check + run: mvn -B clean compile install -Prelease -Dmaven.test.skip=true + - name: Check dependency license + run: tools/dependencies/check-LICENSE.sh diff --git a/.github/workflows/ci_frontend.yml b/.github/workflows/ci_frontend.yml index 494d12dbae..b8e11252c8 100644 --- a/.github/workflows/ci_frontend.yml +++ b/.github/workflows/ci_frontend.yml @@ -50,6 +50,7 @@ jobs: cd dolphinscheduler-ui npm install node-sass --unsafe-perm npm install + npm run lint npm run build License-check: diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 1c2952b440..6d0b2a2a1d 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -75,6 +75,7 @@ jobs: -Dsonar.core.codeCoveragePlugin=jacoco -Dsonar.projectKey=apache-dolphinscheduler -Dsonar.login=e4058004bc6be89decf558ac819aa1ecbee57682 + -Dsonar.exclusions=dolphinscheduler-ui/src/**/i18n/locale/*.js env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} @@ -83,4 +84,4 @@ jobs: mkdir -p ${LOG_DIR} cd ${DOCKER_DIR} docker-compose logs db > ${LOG_DIR}/db.txt - continue-on-error: true \ No newline at end of file + continue-on-error: true diff --git a/.gitignore b/.gitignore index 6dd99201a9..17b0dc6610 100644 --- a/.gitignore +++ b/.gitignore @@ -4,27 +4,24 @@ .zip .gz .DS_Store -.idea .idea/ -.idea/* -.target -.target/ -**/**/target/** -target/* -*/target -*/target/* +dist/ +all-dependencies.txt +self-modules.txt +third-party-dependencies.txt +**/target/ .settings .nbproject .classpath .project -*.iml +**/*.iml *.ipr *.iws *.tgz .*.swp .vim .tmp -node_modules +**/node_modules npm-debug.log .vscode logs/* @@ -41,110 +38,10 @@ dolphinscheduler-alert/logs/ dolphinscheduler-alert/src/main/resources/alert.properties_bak dolphinscheduler-alert/src/main/resources/logback.xml dolphinscheduler-server/src/main/resources/logback.xml -dolphinscheduler-ui/dist +dolphinscheduler-ui/dist/ dolphinscheduler-ui/node -dolphinscheduler-ui/dist/css/common.16ac5d9.css -dolphinscheduler-ui/dist/css/home/index.b444b91.css -dolphinscheduler-ui/dist/css/login/index.5866c64.css -dolphinscheduler-ui/dist/js/0.ac94e5d.js -dolphinscheduler-ui/dist/js/0.ac94e5d.js.map -dolphinscheduler-ui/dist/js/1.0b043a3.js -dolphinscheduler-ui/dist/js/1.0b043a3.js.map -dolphinscheduler-ui/dist/js/10.1bce3dc.js -dolphinscheduler-ui/dist/js/10.1bce3dc.js.map -dolphinscheduler-ui/dist/js/11.79f04d8.js -dolphinscheduler-ui/dist/js/11.79f04d8.js.map -dolphinscheduler-ui/dist/js/12.420daa5.js -dolphinscheduler-ui/dist/js/12.420daa5.js.map -dolphinscheduler-ui/dist/js/13.e5bae1c.js -dolphinscheduler-ui/dist/js/13.e5bae1c.js.map -dolphinscheduler-ui/dist/js/14.f2a0dca.js -dolphinscheduler-ui/dist/js/14.f2a0dca.js.map -dolphinscheduler-ui/dist/js/15.45373e8.js -dolphinscheduler-ui/dist/js/15.45373e8.js.map -dolphinscheduler-ui/dist/js/16.fecb0fc.js -dolphinscheduler-ui/dist/js/16.fecb0fc.js.map -dolphinscheduler-ui/dist/js/17.84be279.js -dolphinscheduler-ui/dist/js/17.84be279.js.map -dolphinscheduler-ui/dist/js/18.307ea70.js -dolphinscheduler-ui/dist/js/18.307ea70.js.map -dolphinscheduler-ui/dist/js/19.144db9c.js -dolphinscheduler-ui/dist/js/19.144db9c.js.map -dolphinscheduler-ui/dist/js/2.8b4ef29.js -dolphinscheduler-ui/dist/js/2.8b4ef29.js.map -dolphinscheduler-ui/dist/js/20.4c527e9.js -dolphinscheduler-ui/dist/js/20.4c527e9.js.map -dolphinscheduler-ui/dist/js/21.831b2a2.js -dolphinscheduler-ui/dist/js/21.831b2a2.js.map -dolphinscheduler-ui/dist/js/22.2b4bb2a.js -dolphinscheduler-ui/dist/js/22.2b4bb2a.js.map -dolphinscheduler-ui/dist/js/23.81467ef.js -dolphinscheduler-ui/dist/js/23.81467ef.js.map -dolphinscheduler-ui/dist/js/24.54a00e4.js -dolphinscheduler-ui/dist/js/24.54a00e4.js.map -dolphinscheduler-ui/dist/js/25.8d7bd36.js -dolphinscheduler-ui/dist/js/25.8d7bd36.js.map -dolphinscheduler-ui/dist/js/26.2ec5e78.js -dolphinscheduler-ui/dist/js/26.2ec5e78.js.map -dolphinscheduler-ui/dist/js/27.3ab48c2.js -dolphinscheduler-ui/dist/js/27.3ab48c2.js.map -dolphinscheduler-ui/dist/js/28.363088a.js -dolphinscheduler-ui/dist/js/28.363088a.js.map -dolphinscheduler-ui/dist/js/29.6c5853a.js -dolphinscheduler-ui/dist/js/29.6c5853a.js.map -dolphinscheduler-ui/dist/js/3.a0edb5b.js -dolphinscheduler-ui/dist/js/3.a0edb5b.js.map -dolphinscheduler-ui/dist/js/30.940fdd3.js -dolphinscheduler-ui/dist/js/30.940fdd3.js.map -dolphinscheduler-ui/dist/js/31.168a460.js -dolphinscheduler-ui/dist/js/31.168a460.js.map -dolphinscheduler-ui/dist/js/32.8df6594.js -dolphinscheduler-ui/dist/js/32.8df6594.js.map -dolphinscheduler-ui/dist/js/33.4480bbe.js -dolphinscheduler-ui/dist/js/33.4480bbe.js.map -dolphinscheduler-ui/dist/js/34.b407fe1.js -dolphinscheduler-ui/dist/js/34.b407fe1.js.map -dolphinscheduler-ui/dist/js/35.f340b0a.js -dolphinscheduler-ui/dist/js/35.f340b0a.js.map -dolphinscheduler-ui/dist/js/36.8880c2d.js -dolphinscheduler-ui/dist/js/36.8880c2d.js.map -dolphinscheduler-ui/dist/js/37.ea2a25d.js -dolphinscheduler-ui/dist/js/37.ea2a25d.js.map -dolphinscheduler-ui/dist/js/38.98a59ee.js -dolphinscheduler-ui/dist/js/38.98a59ee.js.map -dolphinscheduler-ui/dist/js/39.a5e958a.js -dolphinscheduler-ui/dist/js/39.a5e958a.js.map -dolphinscheduler-ui/dist/js/4.4ca44db.js -dolphinscheduler-ui/dist/js/4.4ca44db.js.map -dolphinscheduler-ui/dist/js/40.e187b1e.js -dolphinscheduler-ui/dist/js/40.e187b1e.js.map -dolphinscheduler-ui/dist/js/41.0e89182.js -dolphinscheduler-ui/dist/js/41.0e89182.js.map -dolphinscheduler-ui/dist/js/42.341047c.js -dolphinscheduler-ui/dist/js/42.341047c.js.map -dolphinscheduler-ui/dist/js/43.27b8228.js -dolphinscheduler-ui/dist/js/43.27b8228.js.map -dolphinscheduler-ui/dist/js/44.e8869bc.js -dolphinscheduler-ui/dist/js/44.e8869bc.js.map -dolphinscheduler-ui/dist/js/45.8d54901.js -dolphinscheduler-ui/dist/js/45.8d54901.js.map -dolphinscheduler-ui/dist/js/5.e1ed7f3.js -dolphinscheduler-ui/dist/js/5.e1ed7f3.js.map -dolphinscheduler-ui/dist/js/6.241ba07.js -dolphinscheduler-ui/dist/js/6.241ba07.js.map -dolphinscheduler-ui/dist/js/7.ab2e297.js -dolphinscheduler-ui/dist/js/7.ab2e297.js.map -dolphinscheduler-ui/dist/js/8.83ff814.js -dolphinscheduler-ui/dist/js/8.83ff814.js.map -dolphinscheduler-ui/dist/js/9.39cb29f.js -dolphinscheduler-ui/dist/js/9.39cb29f.js.map -dolphinscheduler-ui/dist/js/common.733e342.js -dolphinscheduler-ui/dist/js/common.733e342.js.map -dolphinscheduler-ui/dist/js/home/index.78a5d12.js -dolphinscheduler-ui/dist/js/home/index.78a5d12.js.map -dolphinscheduler-ui/dist/js/login/index.291b8e3.js -dolphinscheduler-ui/dist/js/login/index.291b8e3.js.map -dolphinscheduler-ui/dist/lib/external/ -/dolphinscheduler-dao/src/main/resources/dao/data_source.properties +dolphinscheduler-dao/src/main/resources/dao/data_source.properties + +.mvn/wrapper/*.jar !/zookeeper_data/ diff --git a/README.md b/README.md index 84f9ccfa66..b88fdbed53 100644 --- a/README.md +++ b/README.md @@ -99,9 +99,9 @@ It is because of the shoulders of these open source projects that the birth of t ### Get Help 1. Submit an issue 1. Subscribe the mail list : https://dolphinscheduler.apache.org/en-us/docs/development/subscribe.html. then send mail to dev@dolphinscheduler.apache.org +1. Slack channel: [![Slack Status](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://join.slack.com/share/zt-do3gvfhj-UUhrAX2GxkVX_~JJt1jpKA) 1. Contact WeChat(dailidong66). This is just for Mandarin(CN) discussion. ### License Please refer to [LICENSE](https://github.com/apache/incubator-dolphinscheduler/blob/dev/LICENSE) file. - diff --git a/ambari_plugin/README.md b/ambari_plugin/README.md new file mode 100644 index 0000000000..bd634d4a29 --- /dev/null +++ b/ambari_plugin/README.md @@ -0,0 +1,68 @@ +### Dolphin Scheduler的Ambari插件使用说明 + +##### 备注 + +1. 本文档适用于对Ambari中基本了解的用户 +2. 本文档是对已安装Ambari服务添加Dolphin Scheduler(1.3.0版本)服务的说明 + +##### 一 安装准备 + +1. 准备RPM包 + + - 在源码dolphinscheduler-dist目录下执行命令```mvn -U clean install rpm:attached-rpm -Prpmbuild -Dmaven.test.skip=true -X```即可生成(在目录 dolphinscheduler-dist/target/rpm/apache-dolphinscheduler-incubating/RPMS/noarch 下) + +2. 创建DS的安装用户--权限 + +3. 初始化数据库信息 + + ``` + -- 创建Dolphin Scheduler的数据库:dolphinscheduler + CREATE DATABASE dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE + utf8_general_ci; + + -- 初始化dolphinscheduler数据库的用户和密码,并分配权限 + -- 替换下面sql语句中的{user}为dolphinscheduler数据库的用户 + GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'%' IDENTIFIED BY '{password}'; + GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'localhost' IDENTIFIED BY + '{password}'; + flush privileges; + ``` + + + +##### 二 Ambari安装Dolphin Scheduler + +1. Ambari界面安装Dolphin Scheduler + + ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_001.png) + +2. 选择Dolphin Scheduler的Master安装的节点 + + ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_002.png) + +3. 配置Dolphin Scheduler的Worker、Api、Logger、Alert安装的节点 + + ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_003.png) + +4. 设置Dolphin Scheduler服务的安装用户(**步骤一中创建的**)及所属的用户组 + + ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_004.png) + +5. 配置数据库的信息(和步骤一中初始化数据库中一致) + + ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_005.png) + +6. 配置其它的信息--如果需要的话 + + ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_006.png) + + ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_007.png) + +7. 正常执行接下来的步骤 + + ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_008.png) + +8. 安装成功后的界面 + + ![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_009.png) + diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/alerts.json b/ambari_plugin/common-services/DOLPHIN/2.0.0/alerts.json deleted file mode 100644 index 385c5d5599..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/alerts.json +++ /dev/null @@ -1,164 +0,0 @@ -{ - "DOLPHIN": { - "service": [], - "DOLPHIN_API": [ - { - "name": "dolphin_api_port_check", - "label": "dolphin_api_port_check", - "description": "dolphin_api_port_check.", - "interval": 10, - "scope": "ANY", - "source": { - "type": "PORT", - "uri": "{{dolphin-application-api/server.port}}", - "default_port": 12345, - "reporting": { - "ok": { - "text": "TCP OK - {0:.3f}s response on port {1}" - }, - "warning": { - "text": "TCP OK - {0:.3f}s response on port {1}", - "value": 1.5 - }, - "critical": { - "text": "Connection failed: {0} to {1}:{2}", - "value": 5.0 - } - } - } - } - ], - "DOLPHIN_MASTER": [ - { - "name": "DOLPHIN_MASTER_CHECK", - "label": "check dolphin scheduler master status", - "description": "", - "interval":10, - "scope": "HOST", - "enabled": true, - "source": { - "type": "SCRIPT", - "path": "DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py", - "parameters": [ - - { - "name": "connection.timeout", - "display_name": "Connection Timeout", - "value": 5.0, - "type": "NUMERIC", - "description": "The maximum time before this alert is considered to be CRITICAL", - "units": "seconds", - "threshold": "CRITICAL" - }, - { - "name": "alertName", - "display_name": "alertName", - "value": "DOLPHIN_MASTER", - "type": "STRING", - "description": "alert name" - } - ] - } - } - ], - "DOLPHIN_WORKER": [ - { - "name": "DOLPHIN_WORKER_CHECK", - "label": "check dolphin scheduler worker status", - "description": "", - "interval":10, - "scope": "HOST", - "enabled": true, - "source": { - "type": "SCRIPT", - "path": "DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py", - "parameters": [ - - { - "name": "connection.timeout", - "display_name": "Connection Timeout", - "value": 5.0, - "type": "NUMERIC", - "description": "The maximum time before this alert is considered to be CRITICAL", - "units": "seconds", - "threshold": "CRITICAL" - }, - { - "name": "alertName", - "display_name": "alertName", - "value": "DOLPHIN_WORKER", - "type": "STRING", - "description": "alert name" - } - ] - } - } - ], - "DOLPHIN_ALERT": [ - { - "name": "DOLPHIN_DOLPHIN_ALERT_CHECK", - "label": "check dolphin scheduler alert status", - "description": "", - "interval":10, - "scope": "HOST", - "enabled": true, - "source": { - "type": "SCRIPT", - "path": "DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py", - "parameters": [ - - { - "name": "connection.timeout", - "display_name": "Connection Timeout", - "value": 5.0, - "type": "NUMERIC", - "description": "The maximum time before this alert is considered to be CRITICAL", - "units": "seconds", - "threshold": "CRITICAL" - }, - { - "name": "alertName", - "display_name": "alertName", - "value": "DOLPHIN_ALERT", - "type": "STRING", - "description": "alert name" - } - ] - } - } - ], - "DOLPHIN_ALERT": [ - { - "name": "DOLPHIN_DOLPHIN_LOGGER_CHECK", - "label": "check dolphin scheduler alert status", - "description": "", - "interval":10, - "scope": "HOST", - "enabled": true, - "source": { - "type": "SCRIPT", - "path": "DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py", - "parameters": [ - - { - "name": "connection.timeout", - "display_name": "Connection Timeout", - "value": 5.0, - "type": "NUMERIC", - "description": "The maximum time before this alert is considered to be CRITICAL", - "units": "seconds", - "threshold": "CRITICAL" - }, - { - "name": "alertName", - "display_name": "alertName", - "value": "DOLPHIN_LOGGER", - "type": "STRING", - "description": "alert name" - } - ] - } - } - ] - } -} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-alert.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-alert.xml deleted file mode 100644 index 5f44a1a4c8..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-alert.xml +++ /dev/null @@ -1,143 +0,0 @@ - - - - alert.type - EMAIL - alert type is EMAIL/SMS - - - - alert.template - html - alter msg template, default is html template - - - - mail.protocol - SMTP - - - - - mail.server.host - xxx.xxx.com - - - - - mail.server.port - 25 - - int - - - - - - mail.sender - admin - - - - - mail.user - admin - - - - - mail.passwd - 000000 - - PASSWORD - - password - - - - - - mail.smtp.starttls.enable - true - - boolean - - - - - - mail.smtp.ssl.enable - true - - boolean - - - - - - mail.smtp.ssl.trust - xxx.xxx.com - - - - - - enterprise.wechat.enable - false - - - value-list - - - true - - - - false - - - - 1 - - - - - enterprise.wechat.corp.id - wechatId - - - - - enterprise.wechat.secret - secret - - - - - enterprise.wechat.agent.id - agentId - - - - - enterprise.wechat.users - wechatUsers - - - - \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-application-api.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-application-api.xml deleted file mode 100644 index 766c0f477d..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-application-api.xml +++ /dev/null @@ -1,87 +0,0 @@ - - - - server.port - 12345 - - server port - - - int - - - - server.servlet.session.timeout - 7200 - - int - - - - - - server.servlet.context-path - /dolphinscheduler/ - - - - - spring.servlet.multipart.max-file-size - 1024 - - MB - int - - - - - - spring.servlet.multipart.max-request-size - 1024 - - MB - int - - - - - - server.jetty.max-http-post-size - 5000000 - - int - - - - - - spring.messages.encoding - UTF-8 - - - - spring.messages.basename - i18n/messages - - - - security.authentication.type - PASSWORD - - - \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-common.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-common.xml deleted file mode 100644 index 439e21188a..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-common.xml +++ /dev/null @@ -1,158 +0,0 @@ - - - - resource.storage.type - Choose Resource Upload Startup Type - - Resource upload startup type : HDFS,S3,NONE - - NONE - - value-list - - - HDFS - - - - S3 - - - - NONE - - - - 1 - - - - - resource.upload.path - /dolphinscheduler - - resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended - - - - - data.basedir.path - /tmp/dolphinscheduler - - user data local directory path, please make sure the directory exists and have read write permissions - - - - - - hadoop.security.authentication.startup.state - false - - value-list - - - true - - - - false - - - - 1 - - whether kerberos starts - - - java.security.krb5.conf.path - /opt/krb5.conf - - java.security.krb5.conf path - - - - - login.user.keytab.username - hdfs-mycluster@ESZ.COM - - LoginUserFromKeytab user - - - - - login.user.keytab.path - /opt/hdfs.headless.keytab - - LoginUserFromKeytab path - - - - - resource.view.suffixs - txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties - - - - hdfs.root.user - hdfs - - Users who have permission to create directories under the HDFS root path - - - - - fs.defaultFS - hdfs://mycluster:8020 - - HA or single namenode, - If namenode ha needs to copy core-site.xml and hdfs-site.xml to the conf directory, - support s3,for example : s3a://dolphinscheduler - - - - - fs.s3a.endpoint - http://host:9010 - - s3 need,s3 endpoint - - - - - fs.s3a.access.key - A3DXS30FO22544RE - - s3 need,s3 access key - - - - - fs.s3a.secret.key - OloCLq3n+8+sdPHUhJ21XrSxTC+JK - - s3 need,s3 secret key - - - - - kerberos.expire.time - 7 - - - - \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-datasource.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-datasource.xml deleted file mode 100644 index 6e50a1b649..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-datasource.xml +++ /dev/null @@ -1,467 +0,0 @@ - - - - spring.datasource.initialSize - 5 - - Init connection number - - - int - - - - - spring.datasource.minIdle - 5 - - Min connection number - - - int - - - - - spring.datasource.maxActive - 50 - - Max connection number - - - int - - - - - spring.datasource.maxWait - 60000 - - Max wait time for get a connection in milliseconds. - If configuring maxWait, fair locks are enabled by default and concurrency efficiency decreases. - If necessary, unfair locks can be used by configuring the useUnfairLock attribute to true. - - - int - - - - - spring.datasource.timeBetweenEvictionRunsMillis - 60000 - - Milliseconds for check to close free connections - - - int - - - - - spring.datasource.timeBetweenConnectErrorMillis - 60000 - - The Destroy thread detects the connection interval and closes the physical connection in milliseconds - if the connection idle time is greater than or equal to minEvictableIdleTimeMillis. - - - int - - - - - spring.datasource.minEvictableIdleTimeMillis - 300000 - - The longest time a connection remains idle without being evicted, in milliseconds - - - int - - - - - spring.datasource.validationQuery - SELECT 1 - - The SQL used to check whether the connection is valid requires a query statement. - If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work. - - - - - spring.datasource.validationQueryTimeout - 3 - - int - - - Check whether the connection is valid for timeout, in seconds - - - - - spring.datasource.testWhileIdle - true - - boolean - - - When applying for a connection, - if it is detected that the connection is idle longer than time Between Eviction Runs Millis, - validation Query is performed to check whether the connection is valid - - - - - spring.datasource.testOnBorrow - true - - boolean - - - Execute validation to check if the connection is valid when applying for a connection - - - - - spring.datasource.testOnReturn - false - - boolean - - - Execute validation to check if the connection is valid when the connection is returned - - - - - spring.datasource.defaultAutoCommit - true - - boolean - - - - - - - spring.datasource.keepAlive - false - - boolean - - - - - - - - spring.datasource.poolPreparedStatements - true - - boolean - - - Open PSCache, specify count PSCache for every connection - - - - - spring.datasource.maxPoolPreparedStatementPerConnectionSize - 20 - - int - - - - - - spring.datasource.spring.datasource.filters - stat,wall,log4j - - - - - spring.datasource.connectionProperties - druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000 - - - - - - mybatis-plus.mapper-locations - classpath*:/org.apache.dolphinscheduler.dao.mapper/*.xml - - - - - mybatis-plus.typeEnumsPackage - org.apache.dolphinscheduler.*.enums - - - - - mybatis-plus.typeAliasesPackage - org.apache.dolphinscheduler.dao.entity - - Entity scan, where multiple packages are separated by a comma or semicolon - - - - - mybatis-plus.global-config.db-config.id-type - AUTO - - value-list - - - AUTO - - - - INPUT - - - - ID_WORKER - - - - UUID - - - - 1 - - - Primary key type AUTO:" database ID AUTO ", - INPUT:" user INPUT ID", - ID_WORKER:" global unique ID (numeric type unique ID)", - UUID:" global unique ID UUID"; - - - - - mybatis-plus.global-config.db-config.field-strategy - NOT_NULL - - value-list - - - IGNORED - - - - NOT_NULL - - - - NOT_EMPTY - - - - 1 - - - Field policy IGNORED:" ignore judgment ", - NOT_NULL:" not NULL judgment "), - NOT_EMPTY:" not NULL judgment" - - - - - mybatis-plus.global-config.db-config.column-underline - true - - boolean - - - - - - mybatis-plus.global-config.db-config.logic-delete-value - 1 - - int - - - - - - mybatis-plus.global-config.db-config.logic-not-delete-value - 0 - - int - - - - - - mybatis-plus.global-config.db-config.banner - true - - boolean - - - - - - - mybatis-plus.configuration.map-underscore-to-camel-case - true - - boolean - - - - - - mybatis-plus.configuration.cache-enabled - false - - boolean - - - - - - mybatis-plus.configuration.call-setters-on-nulls - true - - boolean - - - - - - mybatis-plus.configuration.jdbc-type-for-null - null - - - - - master.exec.threads - 100 - - int - - - - - - master.exec.task.num - 20 - - int - - - - - - master.heartbeat.interval - 10 - - int - - - - - - master.task.commit.retryTimes - 5 - - int - - - - - - master.task.commit.interval - 1000 - - int - - - - - - master.max.cpuload.avg - 100 - - int - - - - - - master.reserved.memory - 0.1 - - float - - - - - - worker.exec.threads - 100 - - int - - - - - - worker.heartbeat.interval - 10 - - int - - - - - - worker.fetch.task.num - 3 - - int - - - - - - worker.max.cpuload.avg - 100 - - int - - - - - - worker.reserved.memory - 0.1 - - float - - - - - - \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-env.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-env.xml deleted file mode 100644 index 8e14716d05..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-env.xml +++ /dev/null @@ -1,123 +0,0 @@ - - - - dolphin.database.type - mysql - Dolphin Scheduler DataBase Type Which Is Select - Dolphin Database Type - - value-list - - - mysql - - - - postgresql - - - - 1 - - - - - - dolphin.database.host - - Dolphin Database Host - - - - - dolphin.database.port - - Dolphin Database Port - - - - - dolphin.database.username - - Dolphin Database Username - - - - - dolphin.database.password - - Dolphin Database Password - PASSWORD - - password - - - - - - dolphin.user - - Which user to install and admin dolphin scheduler - Deploy User - - - - dolphin.group - - Which user to install and admin dolphin scheduler - Deploy Group - - - - - dolphinscheduler-env-content - Dolphinscheduler Env template - This is the jinja template for dolphinscheduler.env.sh file - # -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -export HADOOP_HOME=/opt/soft/hadoop -export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop -export SPARK_HOME1=/opt/soft/spark1 -export SPARK_HOME2=/opt/soft/spark2 -export PYTHON_HOME=/opt/soft/python -export JAVA_HOME=/opt/soft/java -export HIVE_HOME=/opt/soft/hive -export FLINK_HOME=/opt/soft/flink - - content - false - false - - - - \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-master.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-master.xml deleted file mode 100644 index c8eec047fc..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-master.xml +++ /dev/null @@ -1,88 +0,0 @@ - - - - master.exec.threads - 100 - - int - - master execute thread num - - - - master.exec.task.num - 20 - - int - - master execute task number in parallel - - - - master.heartbeat.interval - 10 - - int - - master heartbeat interval - - - - master.task.commit.retryTimes - 5 - - int - - master commit task retry times - - - - master.task.commit.interval - 1000 - - int - - master commit task interval - - - - master.max.cpuload.avg - 100 - - int - - only less than cpu avg load, master server can work. default value : the number of cpu cores * 2 - - - - master.reserved.memory - 0.3 - only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G. - - - - - master.listen.port - 5678 - - int - - master listen port - - - \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-quartz.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-quartz.xml deleted file mode 100644 index 7a0c68b051..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-quartz.xml +++ /dev/null @@ -1,126 +0,0 @@ - - - - org.quartz.scheduler.instanceName - DolphinScheduler - - - - - org.quartz.scheduler.instanceId - AUTO - - - - org.quartz.scheduler.makeSchedulerThreadDaemon - true - - boolean - - - - - org.quartz.jobStore.useProperties - false - - boolean - - - - - org.quartz.threadPool.class - org.quartz.simpl.SimpleThreadPool - - - - org.quartz.threadPool.makeThreadsDaemons - true - - boolean - - - - - org.quartz.threadPool.threadCount - 25 - - int - - - - - org.quartz.threadPool.threadPriority - 5 - - int - - - - - org.quartz.jobStore.class - org.quartz.impl.jdbcjobstore.JobStoreTX - - - - org.quartz.jobStore.tablePrefix - QRTZ_ - - - - org.quartz.jobStore.isClustered - true - - boolean - - - - - org.quartz.jobStore.misfireThreshold - 60000 - - int - - - - - org.quartz.jobStore.clusterCheckinInterval - 5000 - - int - - - - - org.quartz.jobStore.acquireTriggersWithinLock - true - - boolean - - - - - org.quartz.jobStore.dataSource - myDs - - - - org.quartz.dataSource.myDs.connectionProvider.class - org.apache.dolphinscheduler.service.quartz.DruidConnectionProvider - - - \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-worker.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-worker.xml deleted file mode 100644 index 97beade1bc..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-worker.xml +++ /dev/null @@ -1,76 +0,0 @@ - - - - worker.exec.threads - 100 - - int - - worker execute thread num - - - - worker.heartbeat.interval - 10 - - int - - worker heartbeat interval - - - - worker.fetch.task.num - 3 - - int - - submit the number of tasks at a time - - - - worker.max.cpuload.avg - 100 - - int - - only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2 - - - - worker.reserved.memory - 0.3 - only larger than reserved memory, worker server can work. default value : physical memory * 1/10, unit is G. - - - - - worker.listen.port - 1234 - - int - - worker listen port - - - - worker.group - default - default worker group - - - \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-zookeeper.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-zookeeper.xml deleted file mode 100644 index 5882162254..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/configuration/dolphin-zookeeper.xml +++ /dev/null @@ -1,84 +0,0 @@ - - - - dolphinscheduler.queue.impl - zookeeper - - Task queue implementation, default "zookeeper" - - - - - zookeeper.dolphinscheduler.root - /dolphinscheduler - - dolphinscheduler root directory - - - - - zookeeper.session.timeout - 300 - - int - - - - - - - zookeeper.connection.timeout - 300 - - int - - - - - - - zookeeper.retry.base.sleep - 100 - - int - - - - - - - zookeeper.retry.max.sleep - 30000 - - int - - - - - - - zookeeper.retry.maxtime - 5 - - int - - - - - - \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/metainfo.xml b/ambari_plugin/common-services/DOLPHIN/2.0.0/metainfo.xml deleted file mode 100644 index b3c14e33cb..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/metainfo.xml +++ /dev/null @@ -1,137 +0,0 @@ - - - - 2.0 - - - DOLPHIN - Dolphin Scheduler - 分布式易扩展的可视化DAG工作流任务调度系统 - 2.0.0 - - - DOLPHIN_MASTER - DS Master - MASTER - 1+ - - - PYTHON - 600 - - - - - DOLPHIN_LOGGER - DS Logger - SLAVE - 1+ - - - PYTHON - 600 - - - - - DOLPHIN_WORKER - DS Worker - SLAVE - 1+ - - - DOLPHIN/DOLPHIN_LOGGER - host - - true - - - - - - PYTHON - 600 - - - - - DOLPHIN_ALERT - DS Alert - SLAVE - 1 - - - PYTHON - 600 - - - - - DOLPHIN_API - DS_Api - SLAVE - 1 - - - PYTHON - 600 - - - - - - ZOOKEEPER - - - - - any - - - apache-dolphinscheduler-incubating-1.2.1* - - - - - - - dolphin-alert - dolphin-app-api - dolphin-app-dao - dolphin-common - dolphin-env - dolphin-quartz - - - - - theme.json - true - - - - quicklinks - - - quicklinks.json - true - - - - - diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py deleted file mode 100644 index 87cc7b453b..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/alerts/alert_dolphin_scheduler_status.py +++ /dev/null @@ -1,124 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import socket -import urllib2 -import os -import logging -import ambari_simplejson as json -from resource_management.libraries.script.script import Script -import sys -reload(sys) -sys.setdefaultencoding('utf-8') - -logger = logging.getLogger('ambari_alerts') - -config = Script.get_config() - - -def get_tokens(): - """ - Returns a tuple of tokens in the format {{site/property}} that will be used - to build the dictionary passed into execute - - :rtype tuple - """ - -def get_info(url, connection_timeout): - response = None - - try: - response = urllib2.urlopen(url, timeout=connection_timeout) - json_data = response.read() - return json_data - finally: - if response is not None: - try: - response.close() - except: - pass - - -def execute(configurations={}, parameters={}, host_name=None): - """ - Returns a tuple containing the result code and a pre-formatted result label - - Keyword arguments: - configurations : a mapping of configuration key to value - parameters : a mapping of script parameter key to value - host_name : the name of this host where the alert is running - - :type configurations dict - :type parameters dict - :type host_name str - """ - - alert_name = parameters['alertName'] - - dolphin_pidfile_dir = "/opt/soft/run/dolphinscheduler" - - pid = "0" - - - from resource_management.core import sudo - - is_running = True - pid_file_path = "" - if alert_name == 'DOLPHIN_MASTER': - pid_file_path = dolphin_pidfile_dir + "/master-server.pid" - elif alert_name == 'DOLPHIN_WORKER': - pid_file_path = dolphin_pidfile_dir + "/worker-server.pid" - elif alert_name == 'DOLPHIN_ALERT': - pid_file_path = dolphin_pidfile_dir + "/alert-server.pid" - elif alert_name == 'DOLPHIN_LOGGER': - pid_file_path = dolphin_pidfile_dir + "/logger-server.pid" - elif alert_name == 'DOLPHIN_API': - pid_file_path = dolphin_pidfile_dir + "/api-server.pid" - - if not pid_file_path or not os.path.isfile(pid_file_path): - is_running = False - - try: - pid = int(sudo.read_file(pid_file_path)) - except: - is_running = False - - try: - # Kill will not actually kill the process - # From the doc: - # If sig is 0, then no signal is sent, but error checking is still - # performed; this can be used to check for the existence of a - # process ID or process group ID. - sudo.kill(pid, 0) - except OSError: - is_running = False - - if host_name is None: - host_name = socket.getfqdn() - - if not is_running: - result_code = "CRITICAL" - else: - result_code = "OK" - - label = "The comment {0} of DOLPHIN_SCHEDULER on {1} is {2}".format(alert_name, host_name, result_code) - - return ((result_code, [label])) - -if __name__ == "__main__": - pass diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_alert_service.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_alert_service.py deleted file mode 100644 index 62255a3432..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_alert_service.py +++ /dev/null @@ -1,61 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import time -from resource_management import * - -from dolphin_env import dolphin_env - - -class DolphinAlertService(Script): - def install(self, env): - import params - env.set_params(params) - self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) - - def configure(self, env): - import params - params.pika_slave = True - env.set_params(params) - - dolphin_env() - - def start(self, env): - import params - env.set_params(params) - self.configure(env) - no_op_test = format("ls {dolphin_pidfile_dir}/alert-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/alert-server.pid` | grep `cat {dolphin_pidfile_dir}/alert-server.pid` >/dev/null 2>&1") - - start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start alert-server") - Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) - - def stop(self, env): - import params - env.set_params(params) - stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop alert-server") - Execute(stop_cmd, user=params.dolphin_user) - time.sleep(5) - - def status(self, env): - import status_params - env.set_params(status_params) - check_process_status(status_params.dolphin_run_dir + "alert-server.pid") - - -if __name__ == "__main__": - DolphinAlertService().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_api_service.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_api_service.py deleted file mode 100644 index bdc18fb602..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_api_service.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import time -from resource_management import * - -from dolphin_env import dolphin_env - - -class DolphinApiService(Script): - def install(self, env): - import params - env.set_params(params) - self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) - - def configure(self, env): - import params - params.pika_slave = True - env.set_params(params) - - dolphin_env() - - def start(self, env): - import params - env.set_params(params) - self.configure(env) - - #init - init_cmd=format("sh " + params.dolphin_home + "/script/create-dolphinscheduler.sh") - Execute(init_cmd, user=params.dolphin_user) - - #upgrade - upgrade_cmd=format("sh " + params.dolphin_home + "/script/upgrade-dolphinscheduler.sh") - Execute(upgrade_cmd, user=params.dolphin_user) - - no_op_test = format("ls {dolphin_pidfile_dir}/api-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/api-server.pid` | grep `cat {dolphin_pidfile_dir}/api-server.pid` >/dev/null 2>&1") - - start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start api-server") - Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) - - def stop(self, env): - import params - env.set_params(params) - stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop api-server") - Execute(stop_cmd, user=params.dolphin_user) - time.sleep(5) - - def status(self, env): - import status_params - env.set_params(status_params) - check_process_status(status_params.dolphin_run_dir + "api-server.pid") - - -if __name__ == "__main__": - DolphinApiService().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_env.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_env.py deleted file mode 100644 index 1661d76c75..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_env.py +++ /dev/null @@ -1,123 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management import * - - -def dolphin_env(): - import params - - Directory(params.dolphin_pidfile_dir, - mode=0777, - owner=params.dolphin_user, - group=params.dolphin_group, - create_parents=True - ) - Directory(params.dolphin_log_dir, - mode=0777, - owner=params.dolphin_user, - group=params.dolphin_group, - create_parents=True - ) - Directory(params.dolphin_conf_dir, - mode=0777, - owner=params.dolphin_user, - group=params.dolphin_group, - create_parents=True - ) - - Directory(params.dolphin_common_map['data.basedir.path'], - mode=0777, - owner=params.dolphin_user, - group=params.dolphin_group, - create_parents=True - ) - - - File(format(params.dolphin_env_path), - mode=0777, - content=InlineTemplate(params.dolphin_env_content), - owner=params.dolphin_user, - group=params.dolphin_group - ) - - - File(format(params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh"), - mode=0755, - content=Template("dolphin-daemon.sh.j2"), - owner=params.dolphin_user, - group=params.dolphin_group - ) - - File(format(params.dolphin_conf_dir + "/master.properties"), - mode=0755, - content=Template("master.properties.j2"), - owner=params.dolphin_user, - group=params.dolphin_group - ) - - File(format(params.dolphin_conf_dir + "/worker.properties"), - mode=0755, - content=Template("worker.properties.j2"), - owner=params.dolphin_user, - group=params.dolphin_group - ) - - - File(format(params.dolphin_conf_dir + "/alert.properties"), - mode=0755, - content=Template("alert.properties.j2"), - owner=params.dolphin_user, - group=params.dolphin_group - ) - - File(format(params.dolphin_conf_dir + "/datasource.properties"), - mode=0755, - content=Template("datasource.properties.j2"), - owner=params.dolphin_user, - group=params.dolphin_group - ) - - File(format(params.dolphin_conf_dir + "/application-api.properties"), - mode=0755, - content=Template("application-api.properties.j2"), - owner=params.dolphin_user, - group=params.dolphin_group - ) - - File(format(params.dolphin_conf_dir + "/common.properties"), - mode=0755, - content=Template("common.properties.j2"), - owner=params.dolphin_user, - group=params.dolphin_group - ) - - File(format(params.dolphin_conf_dir + "/quartz.properties"), - mode=0755, - content=Template("quartz.properties.j2"), - owner=params.dolphin_user, - group=params.dolphin_group - ) - - File(format(params.dolphin_conf_dir + "/zookeeper.properties"), - mode=0755, - content=Template("zookeeper.properties.j2"), - owner=params.dolphin_user, - group=params.dolphin_group - ) diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_logger_service.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_logger_service.py deleted file mode 100644 index f1c19bd66f..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_logger_service.py +++ /dev/null @@ -1,61 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import time -from resource_management import * - -from dolphin_env import dolphin_env - - -class DolphinLoggerService(Script): - def install(self, env): - import params - env.set_params(params) - self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) - - def configure(self, env): - import params - params.pika_slave = True - env.set_params(params) - - dolphin_env() - - def start(self, env): - import params - env.set_params(params) - self.configure(env) - no_op_test = format("ls {dolphin_pidfile_dir}/logger-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/logger-server.pid` | grep `cat {dolphin_pidfile_dir}/logger-server.pid` >/dev/null 2>&1") - - start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start logger-server") - Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) - - def stop(self, env): - import params - env.set_params(params) - stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop logger-server") - Execute(stop_cmd, user=params.dolphin_user) - time.sleep(5) - - def status(self, env): - import status_params - env.set_params(status_params) - check_process_status(status_params.dolphin_run_dir + "logger-server.pid") - - -if __name__ == "__main__": - DolphinLoggerService().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_master_service.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_master_service.py deleted file mode 100644 index 6ee7ecfcf3..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_master_service.py +++ /dev/null @@ -1,61 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import time -from resource_management import * - -from dolphin_env import dolphin_env - - -class DolphinMasterService(Script): - def install(self, env): - import params - env.set_params(params) - self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) - - def configure(self, env): - import params - params.pika_slave = True - env.set_params(params) - - dolphin_env() - - def start(self, env): - import params - env.set_params(params) - self.configure(env) - no_op_test = format("ls {dolphin_pidfile_dir}/master-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/master-server.pid` | grep `cat {dolphin_pidfile_dir}/master-server.pid` >/dev/null 2>&1") - start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start master-server") - Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) - - def stop(self, env): - import params - env.set_params(params) - stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop master-server") - Execute(stop_cmd, user=params.dolphin_user) - time.sleep(5) - - def status(self, env): - import status_params - env.set_params(status_params) - check_process_status(status_params.dolphin_run_dir + "master-server.pid") - - -if __name__ == "__main__": - DolphinMasterService().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_worker_service.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_worker_service.py deleted file mode 100644 index 2d145ee730..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/dolphin_worker_service.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import time -from resource_management import * - -from dolphin_env import dolphin_env - - -class DolphinWorkerService(Script): - def install(self, env): - import params - env.set_params(params) - self.install_packages(env) - Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True) - - def configure(self, env): - import params - params.pika_slave = True - env.set_params(params) - - dolphin_env() - - def start(self, env): - import params - env.set_params(params) - self.configure(env) - no_op_test = format("ls {dolphin_pidfile_dir}/worker-server.pid >/dev/null 2>&1 && ps `cat {dolphin_pidfile_dir}/worker-server.pid` | grep `cat {dolphin_pidfile_dir}/worker-server.pid` >/dev/null 2>&1") - start_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh start worker-server") - Execute(start_cmd, user=params.dolphin_user, not_if=no_op_test) - - def stop(self, env): - import params - env.set_params(params) - stop_cmd = format("sh " + params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh stop worker-server") - Execute(stop_cmd, user=params.dolphin_user) - time.sleep(5) - - def status(self, env): - import status_params - env.set_params(status_params) - check_process_status(status_params.dolphin_run_dir + "worker-server.pid") - - -if __name__ == "__main__": - DolphinWorkerService().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/params.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/params.py deleted file mode 100644 index b09b2589f4..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/params.py +++ /dev/null @@ -1,154 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - - -import sys -from resource_management import * -from resource_management.core.logger import Logger -from resource_management.libraries.functions import default - -Logger.initialize_logger() -reload(sys) -sys.setdefaultencoding('utf-8') - -# server configurations -config = Script.get_config() - -# conf_dir = "/etc/" -dolphin_home = "/opt/soft/dolphinscheduler" -dolphin_conf_dir = dolphin_home + "/conf" -dolphin_log_dir = dolphin_home + "/logs" -dolphin_bin_dir = dolphin_home + "/bin" -dolphin_lib_jars = dolphin_home + "/lib/*" -dolphin_pidfile_dir = "/opt/soft/run/dolphinscheduler" - -rmHosts = default("/clusterHostInfo/rm_host", []) - -# dolphin-env -dolphin_env_map = {} -dolphin_env_map.update(config['configurations']['dolphin-env']) - -# which user to install and admin dolphin scheduler -dolphin_user = dolphin_env_map['dolphin.user'] -dolphin_group = dolphin_env_map['dolphin.group'] - -# .dolphinscheduler_env.sh -dolphin_env_path = dolphin_conf_dir + '/env/dolphinscheduler_env.sh' -dolphin_env_content = dolphin_env_map['dolphinscheduler-env-content'] - -# database config -dolphin_database_config = {} -dolphin_database_config['dolphin_database_type'] = dolphin_env_map['dolphin.database.type'] -dolphin_database_config['dolphin_database_username'] = dolphin_env_map['dolphin.database.username'] -dolphin_database_config['dolphin_database_password'] = dolphin_env_map['dolphin.database.password'] -if 'mysql' == dolphin_database_config['dolphin_database_type']: - dolphin_database_config['dolphin_database_driver'] = 'com.mysql.jdbc.Driver' - dolphin_database_config['driverDelegateClass'] = 'org.quartz.impl.jdbcjobstore.StdJDBCDelegate' - dolphin_database_config['dolphin_database_url'] = 'jdbc:mysql://' + dolphin_env_map['dolphin.database.host'] \ - + ':' + dolphin_env_map['dolphin.database.port'] \ - + '/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8' -else: - dolphin_database_config['dolphin_database_driver'] = 'org.postgresql.Driver' - dolphin_database_config['driverDelegateClass'] = 'org.quartz.impl.jdbcjobstore.PostgreSQLDelegate' - dolphin_database_config['dolphin_database_url'] = 'jdbc:postgresql://' + dolphin_env_map['dolphin.database.host'] \ - + ':' + dolphin_env_map['dolphin.database.port'] \ - + '/dolphinscheduler' - - - - - -# application-alert.properties -dolphin_alert_map = {} -wechat_push_url = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token' -wechat_token_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret' -wechat_team_send_msg = '{\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"}' -wechat_user_send_msg = '{\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}}' - -dolphin_alert_config_map = config['configurations']['dolphin-alert'] - -if dolphin_alert_config_map['enterprise.wechat.enable']: - dolphin_alert_map['enterprise.wechat.push.ur'] = wechat_push_url - dolphin_alert_map['enterprise.wechat.token.url'] = wechat_token_url - dolphin_alert_map['enterprise.wechat.team.send.msg'] = wechat_team_send_msg - dolphin_alert_map['enterprise.wechat.user.send.msg'] = wechat_user_send_msg - -dolphin_alert_map.update(dolphin_alert_config_map) - - - -# application-api.properties -dolphin_app_api_map = {} -dolphin_app_api_map.update(config['configurations']['dolphin-application-api']) - - -# common.properties -dolphin_common_map = {} - -if 'yarn-site' in config['configurations'] and \ - 'yarn.resourcemanager.webapp.address' in config['configurations']['yarn-site']: - yarn_resourcemanager_webapp_address = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'] - yarn_application_status_address = 'http://' + yarn_resourcemanager_webapp_address + '/ws/v1/cluster/apps/%s' - dolphin_common_map['yarn.application.status.address'] = yarn_application_status_address - -rmHosts = default("/clusterHostInfo/rm_host", []) -if len(rmHosts) > 1: - dolphin_common_map['yarn.resourcemanager.ha.rm.ids'] = ','.join(rmHosts) -else: - dolphin_common_map['yarn.resourcemanager.ha.rm.ids'] = '' - -dolphin_common_map_tmp = config['configurations']['dolphin-common'] -data_basedir_path = dolphin_common_map_tmp['data.basedir.path'] -process_exec_basepath = data_basedir_path + '/exec' -data_download_basedir_path = data_basedir_path + '/download' -dolphin_common_map['process.exec.basepath'] = process_exec_basepath -dolphin_common_map['data.download.basedir.path'] = data_download_basedir_path -dolphin_common_map['dolphinscheduler.env.path'] = dolphin_env_path -dolphin_common_map.update(config['configurations']['dolphin-common']) - -# datasource.properties -dolphin_datasource_map = {} -dolphin_datasource_map['spring.datasource.type'] = 'com.alibaba.druid.pool.DruidDataSource' -dolphin_datasource_map['spring.datasource.driver-class-name'] = dolphin_database_config['dolphin_database_driver'] -dolphin_datasource_map['spring.datasource.url'] = dolphin_database_config['dolphin_database_url'] -dolphin_datasource_map['spring.datasource.username'] = dolphin_database_config['dolphin_database_username'] -dolphin_datasource_map['spring.datasource.password'] = dolphin_database_config['dolphin_database_password'] -dolphin_datasource_map.update(config['configurations']['dolphin-datasource']) - -# master.properties -dolphin_master_map = config['configurations']['dolphin-master'] - -# quartz.properties -dolphin_quartz_map = {} -dolphin_quartz_map['org.quartz.jobStore.driverDelegateClass'] = dolphin_database_config['driverDelegateClass'] -dolphin_quartz_map.update(config['configurations']['dolphin-quartz']) - -# worker.properties -dolphin_worker_map = config['configurations']['dolphin-worker'] - -# zookeeper.properties -dolphin_zookeeper_map={} -zookeeperHosts = default("/clusterHostInfo/zookeeper_hosts", []) -if len(zookeeperHosts) > 0 and "clientPort" in config['configurations']['zoo.cfg']: - clientPort = config['configurations']['zoo.cfg']['clientPort'] - zookeeperPort = ":" + clientPort + "," - dolphin_zookeeper_map['zookeeper.quorum'] = zookeeperPort.join(zookeeperHosts) + ":" + clientPort -dolphin_zookeeper_map.update(config['configurations']['dolphin-zookeeper']) - - - diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/service_check.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/service_check.py deleted file mode 100644 index 0e12f69932..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/service_check.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management import * -from resource_management.libraries.functions import get_unique_id_and_date - -class ServiceCheck(Script): - def service_check(self, env): - import params - #env.set_params(params) - - # Execute(format("which pika_server")) - -if __name__ == "__main__": - ServiceCheck().execute() diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/status_params.py b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/status_params.py deleted file mode 100644 index 24b2c8b1bc..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/scripts/status_params.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -from resource_management import * - -config = Script.get_config() - -dolphin_run_dir = "/opt/soft/run/dolphinscheduler/" diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/application-api.properties.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/application-api.properties.j2 deleted file mode 100644 index 70118003b9..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/application-api.properties.j2 +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -{% for key, value in dolphin_app_api_map.iteritems() -%} - {{key}}={{value}} -{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/datasource.properties.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/datasource.properties.j2 deleted file mode 100644 index 40aed83543..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/datasource.properties.j2 +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -{% for key, value in dolphin_datasource_map.iteritems() -%} - {{key}}={{value}} -{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/dolphin-daemon.sh.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/dolphin-daemon.sh.j2 deleted file mode 100644 index 0802b74750..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/dolphin-daemon.sh.j2 +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/sh -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -usage="Usage: dolphinscheduler-daemon.sh (start|stop) " - -# if no args specified, show usage -if [ $# -le 1 ]; then - echo $usage - exit 1 -fi - -startStop=$1 -shift -command=$1 -shift - -echo "Begin $startStop $command......" - -BIN_DIR=`dirname $0` -BIN_DIR=`cd "$BIN_DIR"; pwd` -DOLPHINSCHEDULER_HOME=$BIN_DIR/.. - -export HOSTNAME=`hostname` - -DOLPHINSCHEDULER_LIB_JARS={{dolphin_lib_jars}} - -DOLPHINSCHEDULER_OPTS="-server -Xmx16g -Xms4g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" -STOP_TIMEOUT=5 - -log={{dolphin_log_dir}}/dolphinscheduler-$command-$HOSTNAME.out -pid={{dolphin_pidfile_dir}}/$command.pid - -cd $DOLPHINSCHEDULER_HOME - -if [ "$command" = "api-server" ]; then - LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-api.xml -Dspring.profiles.active=api" - CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer -elif [ "$command" = "master-server" ]; then - LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-master.xml -Ddruid.mysql.usePingMethod=false" - CLASS=org.apache.dolphinscheduler.server.master.MasterServer -elif [ "$command" = "worker-server" ]; then - LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-worker.xml -Ddruid.mysql.usePingMethod=false" - CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer -elif [ "$command" = "alert-server" ]; then - LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-alert.xml" - CLASS=org.apache.dolphinscheduler.alert.AlertServer -elif [ "$command" = "logger-server" ]; then - CLASS=org.apache.dolphinscheduler.server.log.LoggerServer -else - echo "Error: No command named \`$command' was found." - exit 1 -fi - -case $startStop in - (start) - - if [ -f $pid ]; then - if kill -0 `cat $pid` > /dev/null 2>&1; then - echo $command running as process `cat $pid`. Stop it first. - exit 1 - fi - fi - - echo starting $command, logging to $log - - exec_command="$LOG_FILE $DOLPHINSCHEDULER_OPTS -classpath {{dolphin_conf_dir}}:{{dolphin_lib_jars}} $CLASS" - - echo "nohup java $exec_command > $log 2>&1 < /dev/null &" - nohup java $exec_command > $log 2>&1 < /dev/null & - echo $! > $pid - ;; - - (stop) - - if [ -f $pid ]; then - TARGET_PID=`cat $pid` - if kill -0 $TARGET_PID > /dev/null 2>&1; then - echo stopping $command - kill $TARGET_PID - sleep $STOP_TIMEOUT - if kill -0 $TARGET_PID > /dev/null 2>&1; then - echo "$command did not stop gracefully after $STOP_TIMEOUT seconds: killing with kill -9" - kill -9 $TARGET_PID - fi - else - echo no $command to stop - fi - rm -f $pid - else - echo no $command to stop - fi - ;; - - (*) - echo $usage - exit 1 - ;; - -esac - -echo "End $startStop $command." \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/master.properties.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/master.properties.j2 deleted file mode 100644 index d9b85e14cf..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/master.properties.j2 +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -{% for key, value in dolphin_master_map.iteritems() -%} - {{key}}={{value}} -{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/quartz.properties.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/quartz.properties.j2 deleted file mode 100644 index e027a263b5..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/quartz.properties.j2 +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -{% for key, value in dolphin_quartz_map.iteritems() -%} - {{key}}={{value}} -{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/worker.properties.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/worker.properties.j2 deleted file mode 100644 index a008b74084..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/worker.properties.j2 +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -{% for key, value in dolphin_worker_map.iteritems() -%} - {{key}}={{value}} -{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/zookeeper.properties.j2 b/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/zookeeper.properties.j2 deleted file mode 100644 index 9eb14eaef3..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/zookeeper.properties.j2 +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -{% for key, value in dolphin_zookeeper_map.iteritems() -%} - {{key}}={{value}} -{% endfor %} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/quicklinks/quicklinks.json b/ambari_plugin/common-services/DOLPHIN/2.0.0/quicklinks/quicklinks.json deleted file mode 100755 index 8753004fef..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/quicklinks/quicklinks.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "name": "default", - "description": "default quick links configuration", - "configuration": { - "protocol": - { - "type":"http" - }, - - "links": [ - { - "name": "dolphin-application-ui", - "label": "DolphinApplication UI", - "requires_user_name": "false", - "component_name": "DOLPHIN_API", - "url": "%@://%@:%@/dolphinscheduler/ui/view/login/index.html", - "port":{ - "http_property": "server.port", - "http_default_port": "12345", - "regex": "^(\\d+)$", - "site": "dolphin-application-api" - } - } - ] - } -} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/themes/theme.json b/ambari_plugin/common-services/DOLPHIN/2.0.0/themes/theme.json deleted file mode 100644 index 953e2323f8..0000000000 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/themes/theme.json +++ /dev/null @@ -1,661 +0,0 @@ -{ - "name": "default", - "description": "Default theme for Dolphin Scheduler service", - "configuration": { - "layouts": [ - { - "name": "default", - "tabs": [ - { - "name": "settings", - "display-name": "Settings", - "layout": { - "tab-rows": "3", - "tab-columns": "3", - "sections": [ - { - "name": "dolphin-env-config", - "display-name": "Dolphin Env Config", - "row-index": "0", - "column-index": "0", - "row-span": "1", - "column-span": "2", - "section-rows": "1", - "section-columns": "2", - "subsections": [ - { - "name": "env-row1-col1", - "display-name": "Deploy User Info", - "row-index": "0", - "column-index": "0", - "row-span": "1", - "column-span": "1" - }, - { - "name": "env-row1-col2", - "display-name": "System Env Optimization", - "row-index": "0", - "column-index": "1", - "row-span": "1", - "column-span": "1" - } - ] - }, - { - "name": "dolphin-database-config", - "display-name": "Database Config", - "row-index": "1", - "column-index": "0", - "row-span": "1", - "column-span": "2", - "section-rows": "1", - "section-columns": "3", - "subsections": [ - { - "name": "database-row1-col1", - "row-index": "0", - "column-index": "0", - "row-span": "1", - "column-span": "1" - }, - { - "name": "database-row1-col2", - "row-index": "0", - "column-index": "1", - "row-span": "1", - "column-span": "1" - }, - { - "name": "database-row1-col3", - "row-index": "0", - "column-index": "2", - "row-span": "1", - "column-span": "1" - } - ] - }, - { - "name": "dynamic-config", - "row-index": "2", - "column-index": "0", - "row-span": "1", - "column-span": "2", - "section-rows": "1", - "section-columns": "3", - "subsections": [ - { - "name": "dynamic-row1-col1", - "display-name": "Resource FS Config", - "row-index": "0", - "column-index": "0", - "row-span": "1", - "column-span": "1" - }, - { - "name": "dynamic-row1-col2", - "display-name": "Kerberos Info", - "row-index": "0", - "column-index": "1", - "row-span": "1", - "column-span": "1" - }, - { - "name": "dynamic-row1-col3", - "display-name": "Wechat Info", - "row-index": "0", - "column-index": "1", - "row-span": "1", - "column-span": "1" - } - ] - } - ] - } - } - ] - } - ], - "placement": { - "configuration-layout": "default", - "configs": [ - { - "config": "dolphin-env/dolphin.database.type", - "subsection-name": "database-row1-col1" - }, - { - "config": "dolphin-env/dolphin.database.host", - "subsection-name": "database-row1-col2" - }, - { - "config": "dolphin-env/dolphin.database.port", - "subsection-name": "database-row1-col2" - }, - { - "config": "dolphin-env/dolphin.database.username", - "subsection-name": "database-row1-col3" - }, - { - "config": "dolphin-env/dolphin.database.password", - "subsection-name": "database-row1-col3" - }, - { - "config": "dolphin-env/dolphin.user", - "subsection-name": "env-row1-col1" - }, - { - "config": "dolphin-env/dolphin.group", - "subsection-name": "env-row1-col1" - }, - { - "config": "dolphin-env/dolphinscheduler-env-content", - "subsection-name": "env-row1-col2" - }, - { - "config": "dolphin-common/resource.storage.type", - "subsection-name": "dynamic-row1-col1" - }, - { - "config": "dolphin-common/resource.upload.path", - "subsection-name": "dynamic-row1-col1", - "depends-on": [ - { - "configs":[ - "dolphin-common/resource.storage.type" - ], - "if": "${dolphin-common/resource.storage.type} === HDFS || ${dolphin-common/resource.storage.type} === S3", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-common/hdfs.root.user", - "subsection-name": "dynamic-row1-col1", - "depends-on": [ - { - "configs":[ - "dolphin-common/resource.storage.type" - ], - "if": "${dolphin-common/resource.storage.type} === HDFS", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-common/data.store2hdfs.basepath", - "subsection-name": "dynamic-row1-col1", - "depends-on": [ - { - "configs":[ - "dolphin-common/resource.storage.type" - ], - "if": "${dolphin-common/resource.storage.type} === HDFS", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-common/fs.defaultFS", - "subsection-name": "dynamic-row1-col1", - "depends-on": [ - { - "configs":[ - "dolphin-common/resource.storage.type" - ], - "if": "${dolphin-common/resource.storage.type} === HDFS", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-common/fs.s3a.endpoint", - "subsection-name": "dynamic-row1-col1", - "depends-on": [ - { - "configs":[ - "dolphin-common/resource.storage.type" - ], - "if": "${dolphin-common/resource.storage.type} === S3", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-common/fs.s3a.access.key", - "subsection-name": "dynamic-row1-col1", - "depends-on": [ - { - "configs":[ - "dolphin-common/resource.storage.type" - ], - "if": "${dolphin-common/resource.storage.type} === S3", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-common/fs.s3a.secret.key", - "subsection-name": "dynamic-row1-col1", - "depends-on": [ - { - "configs":[ - "dolphin-common/resource.storage.type" - ], - "if": "${dolphin-common/resource.storage.type} === S3", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-common/hadoop.security.authentication.startup.state", - "subsection-name": "dynamic-row1-col2" - }, - { - "config": "dolphin-common/java.security.krb5.conf.path", - "subsection-name": "dynamic-row1-col2", - "depends-on": [ - { - "configs":[ - "dolphin-common/hadoop.security.authentication.startup.state" - ], - "if": "${dolphin-common/hadoop.security.authentication.startup.state}", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-common/login.user.keytab.username", - "subsection-name": "dynamic-row1-col2", - "depends-on": [ - { - "configs":[ - "dolphin-common/hadoop.security.authentication.startup.state" - ], - "if": "${dolphin-common/hadoop.security.authentication.startup.state}", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-common/login.user.keytab.path", - "subsection-name": "dynamic-row1-col2", - "depends-on": [ - { - "configs":[ - "dolphin-common/hadoop.security.authentication.startup.state" - ], - "if": "${dolphin-common/hadoop.security.authentication.startup.state}", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-common/kerberos.expire.time", - "subsection-name": "dynamic-row1-col2", - "depends-on": [ - { - "configs":[ - "dolphin-common/hadoop.security.authentication.startup.state" - ], - "if": "${dolphin-common/hadoop.security.authentication.startup.state}", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-alert/enterprise.wechat.enable", - "subsection-name": "dynamic-row1-col3" - }, - { - "config": "dolphin-alert/enterprise.wechat.corp.id", - "subsection-name": "dynamic-row1-col3", - "depends-on": [ - { - "configs":[ - "dolphin-alert/enterprise.wechat.enable" - ], - "if": "${dolphin-alert/enterprise.wechat.enable}", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-alert/enterprise.wechat.secret", - "subsection-name": "dynamic-row1-col3", - "depends-on": [ - { - "configs":[ - "dolphin-alert/enterprise.wechat.enable" - ], - "if": "${dolphin-alert/enterprise.wechat.enable}", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-alert/enterprise.wechat.agent.id", - "subsection-name": "dynamic-row1-col3", - "depends-on": [ - { - "configs":[ - "dolphin-alert/enterprise.wechat.enable" - ], - "if": "${dolphin-alert/enterprise.wechat.enable}", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - }, - { - "config": "dolphin-alert/enterprise.wechat.users", - "subsection-name": "dynamic-row1-col3", - "depends-on": [ - { - "configs":[ - "dolphin-alert/enterprise.wechat.enable" - ], - "if": "${dolphin-alert/enterprise.wechat.enable}", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] - } - ] - }, - "widgets": [ - { - "config": "dolphin-env/dolphin.database.type", - "widget": { - "type": "combo" - } - }, - { - "config": "dolphin-env/dolphin.database.host", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-env/dolphin.database.port", - "widget": { - "type": "text-field", - "units": [ - { - "unit-name": "int" - } - ] - } - }, - { - "config": "dolphin-env/dolphin.database.username", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-env/dolphin.database.password", - "widget": { - "type": "password" - } - }, - { - "config": "dolphin-env/dolphin.user", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-env/dolphin.group", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-env/dolphinscheduler-env-content", - "widget": { - "type": "text-area" - } - }, - { - "config": "dolphin-common/resource.storage.type", - "widget": { - "type": "combo" - } - }, - { - "config": "dolphin-common/resource.upload.path", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-common/hdfs.root.user", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-common/data.store2hdfs.basepath", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-common/fs.defaultFS", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-common/fs.s3a.endpoint", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-common/fs.s3a.access.key", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-common/fs.s3a.secret.key", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-common/hadoop.security.authentication.startup.state", - "widget": { - "type": "toggle" - } - }, - { - "config": "dolphin-common/java.security.krb5.conf.path", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-common/login.user.keytab.username", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-common/login.user.keytab.path", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-common/kerberos.expire.time", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-alert/enterprise.wechat.enable", - "widget": { - "type": "toggle" - } - }, - { - "config": "dolphin-alert/enterprise.wechat.corp.id", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-alert/enterprise.wechat.secret", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-alert/enterprise.wechat.agent.id", - "widget": { - "type": "text-field" - } - }, - { - "config": "dolphin-alert/enterprise.wechat.users", - "widget": { - "type": "text-field" - } - } - ] - } -} diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 7e9c4e57cb..26ffb5b2b5 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -23,6 +23,7 @@ services: - "2181:2181" environment: ZOO_MY_ID: 1 + ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons db: image: postgres container_name: postgres diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java index 58a37c2f41..347336cada 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java @@ -16,8 +16,11 @@ */ package org.apache.dolphinscheduler.alert; +import org.apache.dolphinscheduler.alert.plugin.EmailAlertPlugin; import org.apache.dolphinscheduler.alert.runner.AlertSender; import org.apache.dolphinscheduler.alert.utils.Constants; +import org.apache.dolphinscheduler.alert.utils.PropertyUtils; +import org.apache.dolphinscheduler.common.plugin.FilePluginManager; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.DaoFactory; @@ -41,34 +44,47 @@ public class AlertServer { private static AlertServer instance; - public AlertServer() { + private FilePluginManager alertPluginManager; + + private static final String[] whitePrefixes = new String[]{"org.apache.dolphinscheduler.plugin.utils."}; + private static final String[] excludePrefixes = new String[]{ + "org.apache.dolphinscheduler.plugin.", + "ch.qos.logback.", + "org.slf4j." + }; + + public AlertServer() { + alertPluginManager = + new FilePluginManager(PropertyUtils.getString(Constants.PLUGIN_DIR), whitePrefixes, excludePrefixes); + // add default alert plugins + alertPluginManager.addPlugin(new EmailAlertPlugin()); } - public synchronized static AlertServer getInstance(){ + public synchronized static AlertServer getInstance() { if (null == instance) { instance = new AlertServer(); } return instance; } - public void start(){ + public void start() { logger.info("alert server ready start "); - while (Stopper.isRunning()){ + while (Stopper.isRunning()) { try { Thread.sleep(Constants.ALERT_SCAN_INTERVAL); } catch (InterruptedException e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); Thread.currentThread().interrupt(); } List alerts = alertDao.listWaitExecutionAlert(); - alertSender = new AlertSender(alerts, alertDao); + alertSender = new AlertSender(alerts, alertDao, alertPluginManager); alertSender.run(); } } - public static void main(String[] args){ + public static void main(String[] args) { AlertServer alertServer = AlertServer.getInstance(); alertServer.start(); } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EmailManager.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EmailManager.java index 047ee8bfed..96feb7f09e 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EmailManager.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EmailManager.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.alert.manager; import org.apache.dolphinscheduler.alert.utils.MailUtils; -import org.apache.dolphinscheduler.common.enums.ShowType; import java.util.List; import java.util.Map; @@ -35,7 +34,7 @@ public class EmailManager { * @param showType the showType * @return the send result */ - public Map send(List receviersList,List receviersCcList,String title,String content,ShowType showType){ + public Map send(List receviersList,List receviersCcList,String title,String content,String showType){ return MailUtils.sendMails(receviersList, receviersCcList, title, content, showType); } @@ -48,7 +47,7 @@ public class EmailManager { * @param showType the showType * @return the send result */ - public Map send(List receviersList,String title,String content,ShowType showType){ + public Map send(List receviersList,String title,String content,String showType){ return MailUtils.sendMails(receviersList,title, content, showType); } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java index bb06be6561..43649d6758 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.alert.manager; import org.apache.dolphinscheduler.alert.utils.Constants; import org.apache.dolphinscheduler.alert.utils.EnterpriseWeChatUtils; -import org.apache.dolphinscheduler.dao.entity.Alert; +import org.apache.dolphinscheduler.plugin.model.AlertInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,18 +35,18 @@ public class EnterpriseWeChatManager { private static final Logger logger = LoggerFactory.getLogger(EnterpriseWeChatManager.class); /** * Enterprise We Chat send - * @param alert the alert + * @param alertInfo the alert info * @param token the token * @return the send result */ - public Map send(Alert alert, String token){ + public Map send(AlertInfo alertInfo, String token){ Map retMap = new HashMap<>(); retMap.put(Constants.STATUS, false); String agentId = EnterpriseWeChatUtils.ENTERPRISE_WE_CHAT_AGENT_ID; String users = EnterpriseWeChatUtils.ENTERPRISE_WE_CHAT_USERS; List userList = Arrays.asList(users.split(",")); - logger.info("send message {}",alert); - String msg = EnterpriseWeChatUtils.makeUserSendMsg(userList, agentId,EnterpriseWeChatUtils.markdownByAlert(alert)); + logger.info("send message {}", alertInfo.getAlertData().getTitle()); + String msg = EnterpriseWeChatUtils.makeUserSendMsg(userList, agentId,EnterpriseWeChatUtils.markdownByAlert(alertInfo.getAlertData())); try { EnterpriseWeChatUtils.sendEnterpriseWeChat(Constants.UTF_8, msg, token); } catch (IOException e) { diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java new file mode 100644 index 0000000000..d20306b153 --- /dev/null +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPlugin.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.alert.plugin; + +import org.apache.dolphinscheduler.alert.manager.EmailManager; +import org.apache.dolphinscheduler.alert.manager.EnterpriseWeChatManager; +import org.apache.dolphinscheduler.alert.utils.Constants; +import org.apache.dolphinscheduler.alert.utils.EnterpriseWeChatUtils; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.plugin.api.AlertPlugin; +import org.apache.dolphinscheduler.plugin.model.AlertData; +import org.apache.dolphinscheduler.plugin.model.AlertInfo; +import org.apache.dolphinscheduler.plugin.model.PluginName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +/** + * EmailAlertPlugin + * + * This plugin is a default plugin, and mix up email and enterprise wechat, because adapt with former alert behavior + */ +public class EmailAlertPlugin implements AlertPlugin { + + private static final Logger logger = LoggerFactory.getLogger(EmailAlertPlugin.class); + + private PluginName pluginName; + + private static final EmailManager emailManager = new EmailManager(); + private static final EnterpriseWeChatManager weChatManager = new EnterpriseWeChatManager(); + + public EmailAlertPlugin() { + this.pluginName = new PluginName(); + this.pluginName.setEnglish(Constants.PLUGIN_DEFAULT_EMAIL_EN); + this.pluginName.setChinese(Constants.PLUGIN_DEFAULT_EMAIL_CH); + } + + @Override + public String getId() { + return Constants.PLUGIN_DEFAULT_EMAIL; + } + + @Override + public PluginName getName() { + return pluginName; + } + + @Override + @SuppressWarnings("unchecked") + public Map process(AlertInfo info) { + Map retMaps = new HashMap<>(); + + AlertData alert = info.getAlertData(); + + List receviersList = (List) info.getProp(Constants.PLUGIN_DEFAULT_EMAIL_RECEIVERS); + + // receiving group list + // custom receiver + String receivers = alert.getReceivers(); + if (StringUtils.isNotEmpty(receivers)) { + String[] splits = receivers.split(","); + receviersList.addAll(Arrays.asList(splits)); + } + + List receviersCcList = new ArrayList<>(); + // Custom Copier + String receiversCc = alert.getReceiversCc(); + if (StringUtils.isNotEmpty(receiversCc)) { + String[] splits = receiversCc.split(","); + receviersCcList.addAll(Arrays.asList(splits)); + } + + if (CollectionUtils.isEmpty(receviersList) && CollectionUtils.isEmpty(receviersCcList)) { + logger.warn("alert send error : At least one receiver address required"); + retMaps.put(Constants.STATUS, "false"); + retMaps.put(Constants.MESSAGE, "execution failure,At least one receiver address required."); + return retMaps; + } + + retMaps = emailManager.send(receviersList, receviersCcList, alert.getTitle(), alert.getContent(), + alert.getShowType()); + + //send flag + boolean flag = false; + + if (retMaps == null) { + retMaps = new HashMap<>(); + retMaps.put(Constants.MESSAGE, "alert send error."); + retMaps.put(Constants.STATUS, "false"); + logger.info("alert send error : {}", retMaps.get(Constants.MESSAGE)); + return retMaps; + } + + flag = Boolean.parseBoolean(String.valueOf(retMaps.get(Constants.STATUS))); + + if (flag) { + logger.info("alert send success"); + retMaps.put(Constants.MESSAGE, "email send success."); + if (EnterpriseWeChatUtils.isEnable()) { + logger.info("Enterprise WeChat is enable!"); + try { + String token = EnterpriseWeChatUtils.getToken(); + weChatManager.send(info, token); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + } else { + retMaps.put(Constants.MESSAGE, "alert send error."); + logger.info("alert send error : {}", retMaps.get(Constants.MESSAGE)); + } + + return retMaps; + } + +} diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java index 5feb36b60f..d3ac852b0f 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java @@ -16,139 +16,85 @@ */ package org.apache.dolphinscheduler.alert.runner; -import org.apache.dolphinscheduler.alert.manager.EmailManager; -import org.apache.dolphinscheduler.alert.manager.EnterpriseWeChatManager; import org.apache.dolphinscheduler.alert.utils.Constants; -import org.apache.dolphinscheduler.alert.utils.EnterpriseWeChatUtils; import org.apache.dolphinscheduler.common.enums.AlertStatus; -import org.apache.dolphinscheduler.common.enums.AlertType; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.plugin.PluginManager; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.entity.Alert; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.api.AlertPlugin; +import org.apache.dolphinscheduler.plugin.model.AlertData; +import org.apache.dolphinscheduler.plugin.model.AlertInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; /** * alert sender */ -public class AlertSender{ +public class AlertSender { private static final Logger logger = LoggerFactory.getLogger(AlertSender.class); - private static final EmailManager emailManager= new EmailManager(); - private static final EnterpriseWeChatManager weChatManager= new EnterpriseWeChatManager(); - - private List alertList; private AlertDao alertDao; + private PluginManager pluginManager; + + public AlertSender() { + } - public AlertSender(){} - public AlertSender(List alertList, AlertDao alertDao){ + public AlertSender(List alertList, AlertDao alertDao, PluginManager pluginManager) { super(); this.alertList = alertList; this.alertDao = alertDao; + this.pluginManager = pluginManager; } public void run() { - List users; - Map retMaps = null; - for(Alert alert:alertList){ + for (Alert alert : alertList) { users = alertDao.listUserByAlertgroupId(alert.getAlertGroupId()); // receiving group list List receviersList = new ArrayList<>(); - for(User user:users){ + for (User user : users) { receviersList.add(user.getEmail()); } - // custom receiver - String receivers = alert.getReceivers(); - if (StringUtils.isNotEmpty(receivers)){ - String[] splits = receivers.split(","); - receviersList.addAll(Arrays.asList(splits)); - } - // copy list - List receviersCcList = new ArrayList<>(); - - - // Custom Copier - String receiversCc = alert.getReceiversCc(); - - if (StringUtils.isNotEmpty(receiversCc)){ - String[] splits = receiversCc.split(","); - receviersCcList.addAll(Arrays.asList(splits)); - } - - if (CollectionUtils.isEmpty(receviersList) && CollectionUtils.isEmpty(receviersCcList)) { - logger.warn("alert send error : At least one receiver address required"); - alertDao.updateAlert(AlertStatus.EXECUTION_FAILURE, "execution failure,At least one receiver address required.", alert.getId()); - continue; - } - - if (alert.getAlertType() == AlertType.EMAIL){ - retMaps = emailManager.send(receviersList,receviersCcList, alert.getTitle(), alert.getContent(),alert.getShowType()); - - alert.setInfo(retMaps); - }else if (alert.getAlertType() == AlertType.SMS){ - retMaps = emailManager.send(getReciversForSMS(users), alert.getTitle(), alert.getContent(),alert.getShowType()); - alert.setInfo(retMaps); + AlertData alertData = new AlertData(); + alertData.setId(alert.getId()) + .setAlertGroupId(alert.getAlertGroupId()) + .setContent(alert.getContent()) + .setLog(alert.getLog()) + .setReceivers(alert.getReceivers()) + .setReceiversCc(alert.getReceiversCc()) + .setShowType(alert.getShowType().getDescp()) + .setTitle(alert.getTitle()); + + AlertInfo alertInfo = new AlertInfo(); + alertInfo.setAlertData(alertData); + + alertInfo.addProp("receivers", receviersList); + + AlertPlugin emailPlugin = pluginManager.findOne(Constants.PLUGIN_DEFAULT_EMAIL); + retMaps = emailPlugin.process(alertInfo); + + if (retMaps == null) { + alertDao.updateAlert(AlertStatus.EXECUTION_FAILURE, "alert send error", alert.getId()); + logger.info("alert send error : return value is null"); + } else if (!Boolean.parseBoolean(String.valueOf(retMaps.get(Constants.STATUS)))) { + alertDao.updateAlert(AlertStatus.EXECUTION_FAILURE, String.valueOf(retMaps.get(Constants.MESSAGE)), alert.getId()); + logger.info("alert send error : {}", retMaps.get(Constants.MESSAGE)); } else { - logger.error("AlertType is not defined. code: {}, descp: {}", - alert.getAlertType().getCode(), - alert.getAlertType().getDescp()); - return; - } - - //send flag - boolean flag = false; - - if (null != retMaps) { - flag = Boolean.parseBoolean(String.valueOf(retMaps.get(Constants.STATUS))); - } - - if (flag) { - alertDao.updateAlert(AlertStatus.EXECUTION_SUCCESS, "execution success", alert.getId()); + alertDao.updateAlert(AlertStatus.EXECUTION_SUCCESS, (String) retMaps.get(Constants.MESSAGE), alert.getId()); logger.info("alert send success"); - if (EnterpriseWeChatUtils.isEnable()) { - logger.info("Enterprise WeChat is enable!"); - try { - String token = EnterpriseWeChatUtils.getToken(); - weChatManager.send(alert, token); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - } else { - if (null != retMaps) { - alertDao.updateAlert(AlertStatus.EXECUTION_FAILURE, String.valueOf(retMaps.get(Constants.MESSAGE)), alert.getId()); - logger.info("alert send error : {}", retMaps.get(Constants.MESSAGE)); - } } } } - - /** - * get a list of SMS users - * @param users - * @return - */ - private List getReciversForSMS(List users){ - List list = new ArrayList<>(); - for (User user : users){ - list.add(user.getPhone()); - } - return list; - } } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java index 28be8aa195..8fa38c62fc 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java @@ -156,4 +156,21 @@ public class Constants { public static final String ENTERPRISE_WECHAT_AGENT_ID = "enterprise.wechat.agent.id"; public static final String ENTERPRISE_WECHAT_USERS = "enterprise.wechat.users"; + + /** + * plugin config + */ + public static final String PLUGIN_DIR = "plugin.dir"; + + public static final String PLUGIN_DEFAULT_EMAIL = "email"; + + public static final String PLUGIN_DEFAULT_EMAIL_CH = "邮件"; + + public static final String PLUGIN_DEFAULT_EMAIL_EN = "email"; + + public static final String PLUGIN_DEFAULT_EMAIL_RECEIVERS = "receivers"; + + public static final String PLUGIN_DEFAULT_EMAIL_RECEIVERCCS = "receiverCcs"; + + public static final String RETMAP_MSG = "msg"; } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java index 170c0dd37e..d199d154aa 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java @@ -18,10 +18,10 @@ package org.apache.dolphinscheduler.alert.utils; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.dao.entity.Alert; import com.alibaba.fastjson.JSON; import com.google.common.reflect.TypeToken; +import org.apache.dolphinscheduler.plugin.model.AlertData; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; @@ -66,15 +66,17 @@ public class EnterpriseWeChatUtils { * get Enterprise WeChat is enable * @return isEnable */ - public static Boolean isEnable(){ - Boolean isEnable = false; + public static boolean isEnable(){ + Boolean isEnable = null; try { isEnable = PropertyUtils.getBoolean(Constants.ENTERPRISE_WECHAT_ENABLE); } catch (Exception e) { logger.error(e.getMessage(),e); } + if (isEnable == null) { + return false; + } return isEnable; - } /** @@ -253,14 +255,13 @@ public class EnterpriseWeChatUtils { /** * Determine the mardown style based on the show type of the alert - * @param alert the alert * @return the markdown alert table/text */ - public static String markdownByAlert(Alert alert){ + public static String markdownByAlert(AlertData alert){ String result = ""; - if (alert.getShowType() == ShowType.TABLE) { + if (alert.getShowType().equals(ShowType.TABLE.getDescp())) { result = markdownTable(alert.getTitle(),alert.getContent()); - }else if(alert.getShowType() == ShowType.TEXT){ + }else if(alert.getShowType().equals(ShowType.TEXT.getDescp())){ result = markdownText(alert.getTitle(),alert.getContent()); } return result; diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java index ef364cb1c2..bb565f5133 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java @@ -74,7 +74,7 @@ public class MailUtils { * @param showType the show type * @return the result map */ - public static Map sendMails(Collection receivers, String title, String content,ShowType showType) { + public static Map sendMails(Collection receivers, String title, String content,String showType) { return sendMails(receivers, null, title, content, showType); } @@ -87,7 +87,7 @@ public class MailUtils { * @param showType the show type * @return the send result */ - public static Map sendMails(Collection receivers, Collection receiversCc, String title, String content, ShowType showType) { + public static Map sendMails(Collection receivers, Collection receiversCc, String title, String content, String showType) { Map retMap = new HashMap<>(); retMap.put(Constants.STATUS, false); @@ -98,7 +98,7 @@ public class MailUtils { receivers.removeIf(StringUtils::isEmpty); - if (showType == ShowType.TABLE || showType == ShowType.TEXT){ + if (showType.equals(ShowType.TABLE.getDescp()) || showType.equals(ShowType.TEXT.getDescp())) { // send email HtmlEmail email = new HtmlEmail(); @@ -125,10 +125,10 @@ public class MailUtils { } catch (Exception e) { handleException(receivers, retMap, e); } - }else if (showType == ShowType.ATTACHMENT || showType == ShowType.TABLEATTACHMENT){ + }else if (showType.equals(ShowType.ATTACHMENT.getDescp()) || showType.equals(ShowType.TABLEATTACHMENT.getDescp())) { try { - String partContent = (showType == ShowType.ATTACHMENT ? "Please see the attachment " + title + Constants.EXCEL_SUFFIX_XLS : htmlTable(content,false)); + String partContent = (showType.equals(ShowType.ATTACHMENT.getDescp()) ? "Please see the attachment " + title + Constants.EXCEL_SUFFIX_XLS : htmlTable(content,false)); attachment(receivers,receiversCc,title,content,partContent); @@ -290,7 +290,7 @@ public class MailUtils { * @return the result map * @throws EmailException */ - private static Map getStringObjectMap(String title, String content, ShowType showType, Map retMap, HtmlEmail email) throws EmailException { + private static Map getStringObjectMap(String title, String content, String showType, Map retMap, HtmlEmail email) throws EmailException { /** * the subject of the message to be sent @@ -299,9 +299,9 @@ public class MailUtils { /** * to send information, you can use HTML tags in mail content because of the use of HtmlEmail */ - if (showType == ShowType.TABLE) { + if (showType.equals(ShowType.TABLE.getDescp())) { email.setMsg(htmlTable(content)); - } else if (showType == ShowType.TEXT) { + } else if (showType.equals(ShowType.TEXT.getDescp())) { email.setMsg(htmlText(content)); } diff --git a/dolphinscheduler-alert/src/main/resources/alert.properties b/dolphinscheduler-alert/src/main/resources/alert.properties index 3e83c01235..19b55fec97 100644 --- a/dolphinscheduler-alert/src/main/resources/alert.properties +++ b/dolphinscheduler-alert/src/main/resources/alert.properties @@ -45,5 +45,5 @@ enterprise.wechat.enable=false #enterprise.wechat.team.send.msg={\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"} #enterprise.wechat.user.send.msg={\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}} - +plugin.dir=/Users/xx/your/path/to/plugin/dir diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java new file mode 100644 index 0000000000..52c2fc050e --- /dev/null +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.alert.plugin; + +import org.apache.dolphinscheduler.alert.utils.Constants; +import org.apache.dolphinscheduler.common.enums.ShowType; +import org.apache.dolphinscheduler.plugin.api.AlertPlugin; +import org.apache.dolphinscheduler.plugin.model.AlertData; +import org.apache.dolphinscheduler.plugin.model.AlertInfo; +import org.apache.dolphinscheduler.plugin.model.PluginName; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.*; + +public class EmailAlertPluginTest { + + private static final Logger logger = LoggerFactory.getLogger(EmailAlertPluginTest.class); + + private AlertPlugin plugin; + + @Before + public void before() { + plugin = new EmailAlertPlugin(); + } + + @Test + public void getId() { + String id = plugin.getId(); + assertEquals(Constants.PLUGIN_DEFAULT_EMAIL, id); + } + + @Test + public void getName() { + PluginName pluginName = plugin.getName(); + assertEquals(Constants.PLUGIN_DEFAULT_EMAIL_CH, pluginName.getChinese()); + assertEquals(Constants.PLUGIN_DEFAULT_EMAIL_EN, pluginName.getEnglish()); + } + + @Test + public void process() { + AlertInfo alertInfo = new AlertInfo(); + AlertData alertData = new AlertData(); + alertData.setId(1) + .setAlertGroupId(1) + .setContent("[\"alarm time:2018-02-05\", \"service name:MYSQL_ALTER\", \"alarm name:MYSQL_ALTER_DUMP\", " + + "\"get the alarm exception.!,interface error,exception information:timed out\", \"request address:http://blog.csdn.net/dreamInTheWorld/article/details/78539286\"]") + .setLog("test log") + .setReceivers("xx@xx.com") + .setReceiversCc("xx@xx.com") + .setShowType(ShowType.TEXT.getDescp()) + .setTitle("test title"); + + alertInfo.setAlertData(alertData); + List list = new ArrayList(){{ add("xx@xx.com"); }}; + alertInfo.addProp("receivers", list); + Map ret = plugin.process(alertInfo); + assertFalse(Boolean.parseBoolean(String.valueOf(ret.get(Constants.STATUS)))); + } +} \ No newline at end of file diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java index d0f3538c1b..2b405cc436 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java @@ -20,6 +20,7 @@ import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.dao.entity.Alert; +import org.apache.dolphinscheduler.plugin.model.AlertData; import org.junit.Assert; import org.junit.Before; import org.junit.Ignore; @@ -120,14 +121,22 @@ public class EnterpriseWeChatUtilsTest { @Test public void testMarkdownByAlertForText(){ Alert alertForText = createAlertForText(); - String result = EnterpriseWeChatUtils.markdownByAlert(alertForText); + AlertData alertData = new AlertData(); + alertData.setTitle(alertForText.getTitle()) + .setShowType(alertForText.getShowType().getDescp()) + .setContent(alertForText.getContent()); + String result = EnterpriseWeChatUtils.markdownByAlert(alertData); Assert.assertNotNull(result); } @Test public void testMarkdownByAlertForTable(){ Alert alertForText = createAlertForTable(); - String result = EnterpriseWeChatUtils.markdownByAlert(alertForText); + AlertData alertData = new AlertData(); + alertData.setTitle(alertForText.getTitle()) + .setShowType(alertForText.getShowType().getDescp()) + .setContent(alertForText.getContent()); + String result = EnterpriseWeChatUtils.markdownByAlert(alertData); Assert.assertNotNull(result); } diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java index 1820a1ef89..11322da0e3 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java @@ -58,7 +58,7 @@ public class MailUtilsTest { alert.setAlertType(AlertType.EMAIL); alert.setAlertGroupId(4); - MailUtils.sendMails(Arrays.asList(receivers),Arrays.asList(receiversCc),alert.getTitle(),alert.getContent(), ShowType.TEXT); + MailUtils.sendMails(Arrays.asList(receivers),Arrays.asList(receiversCc),alert.getTitle(),alert.getContent(), ShowType.TEXT.getDescp()); } @@ -70,7 +70,7 @@ public class MailUtilsTest { String[] mails = new String[]{"xx@xx.com"}; for(Alert alert : alerts){ - MailUtils.sendMails(Arrays.asList(mails),"gaojing", alert.getContent(), alert.getShowType()); + MailUtils.sendMails(Arrays.asList(mails),"gaojing", alert.getContent(), ShowType.TABLE.getDescp()); } } @@ -111,7 +111,7 @@ public class MailUtilsTest { alert.setContent(content); alert.setAlertType(AlertType.EMAIL); alert.setAlertGroupId(1); - MailUtils.sendMails(Arrays.asList(mails),"gaojing", alert.getContent(), ShowType.TABLE); + MailUtils.sendMails(Arrays.asList(mails),"gaojing", alert.getContent(), ShowType.TABLE.getDescp()); } /** @@ -170,7 +170,7 @@ public class MailUtilsTest { alert.setContent(content); alert.setAlertType(AlertType.EMAIL); alert.setAlertGroupId(1); - MailUtils.sendMails(Arrays.asList(mails),"gaojing",alert.getContent(),ShowType.ATTACHMENT); + MailUtils.sendMails(Arrays.asList(mails),"gaojing",alert.getContent(),ShowType.ATTACHMENT.getDescp()); } @Test @@ -183,7 +183,7 @@ public class MailUtilsTest { alert.setContent(content); alert.setAlertType(AlertType.EMAIL); alert.setAlertGroupId(1); - MailUtils.sendMails(Arrays.asList(mails),"gaojing",alert.getContent(),ShowType.TABLEATTACHMENT); + MailUtils.sendMails(Arrays.asList(mails),"gaojing",alert.getContent(),ShowType.TABLEATTACHMENT.getDescp()); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java index 4f3dafdf27..6b539d01b1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java @@ -449,30 +449,30 @@ public class ProcessDefinitionController extends BaseController { } /** - * export process definition by id + * batch export process definition by ids * * @param loginUser login user * @param projectName project name - * @param processDefinitionId process definition id + * @param processDefinitionIds process definition ids * @param response response */ - @ApiOperation(value = "exportProcessDefinitionById", notes= "EXPORT_PROCESS_DEFINITION_BY_ID_NOTES") + @ApiOperation(value = "batchExportProcessDefinitionByIds", notes= "BATCH_EXPORT_PROCESS_DEFINITION_BY_IDS_NOTES") @ApiImplicitParams({ - @ApiImplicitParam(name = "processDefinitionId", value = "PROCESS_DEFINITION_ID", required = true, dataType = "Int", example = "100") + @ApiImplicitParam(name = "processDefinitionIds", value = "PROCESS_DEFINITION_ID", required = true, dataType = "String") }) @GetMapping(value = "/export") @ResponseBody - public void exportProcessDefinitionById(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @PathVariable String projectName, - @RequestParam("processDefinitionId") Integer processDefinitionId, - HttpServletResponse response) { + public void batchExportProcessDefinitionByIds(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, + @RequestParam("processDefinitionIds") String processDefinitionIds, + HttpServletResponse response) { try { - logger.info("export process definition by id, login user:{}, project name:{}, process definition id:{}", - loginUser.getUserName(), projectName, processDefinitionId); - processDefinitionService.exportProcessDefinitionById(loginUser, projectName, processDefinitionId, response); + logger.info("batch export process definition by ids, login user:{}, project name:{}, process definition ids:{}", + loginUser.getUserName(), projectName, processDefinitionIds); + processDefinitionService.batchExportProcessDefinitionByIds(loginUser, projectName, processDefinitionIds, response); } catch (Exception e) { - logger.error(Status.EXPORT_PROCESS_DEFINE_BY_ID_ERROR.getMsg(), e); + logger.error(Status.BATCH_EXPORT_PROCESS_DEFINE_BY_IDS_ERROR.getMsg(), e); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java index 8c52dd4d50..b3d10da288 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java @@ -214,8 +214,8 @@ public enum Status { EXECUTE_PROCESS_INSTANCE_ERROR(50015,"execute process instance error", "操作工作流实例错误"), CHECK_PROCESS_DEFINITION_ERROR(50016,"check process definition error", "检查工作流实例错误"), QUERY_RECIPIENTS_AND_COPYERS_BY_PROCESS_DEFINITION_ERROR(50017,"query recipients and copyers by process definition error", "查询收件人和抄送人错误"), - DATA_IS_NOT_VALID(50017,"data %s not valid", "数据[%s]无效"), - DATA_IS_NULL(50018,"data %s is null", "数据[%s]不能为空"), + DATA_IS_NOT_VALID(50017,"data {0} not valid", "数据[{0}]无效"), + DATA_IS_NULL(50018,"data {0} is null", "数据[{0}]不能为空"), PROCESS_NODE_HAS_CYCLE(50019,"process node has cycle", "流程节点间存在循环依赖"), PROCESS_NODE_S_PARAMETER_INVALID(50020,"process node %s parameter invalid", "流程节点[%s]参数无效"), PROCESS_DEFINE_STATE_ONLINE(50021, "process definition {0} is already on line", "工作流定义[{0}]已上线"), @@ -226,6 +226,7 @@ public enum Status { BATCH_DELETE_PROCESS_DEFINE_BY_IDS_ERROR(50026,"batch delete process definition by ids {0} error", "批量删除工作流定义[{0}]错误"), TENANT_NOT_SUITABLE(50027,"there is not any tenant suitable, please choose a tenant available.", "没有合适的租户,请选择可用的租户"), EXPORT_PROCESS_DEFINE_BY_ID_ERROR(50028,"export process definition by id error", "导出工作流定义错误"), + BATCH_EXPORT_PROCESS_DEFINE_BY_IDS_ERROR(50028,"batch export process definition by ids error", "批量导出工作流定义错误"), IMPORT_PROCESS_DEFINE_ERROR(50029,"import process definition error", "导入工作流定义错误"), HDFS_NOT_STARTUP(60001,"hdfs not startup", "hdfs未启用"), diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index 1aab0953da..611d3ea7f9 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -124,14 +124,12 @@ public class ExecutorService extends BaseService{ return result; } - // check master server exists - List masterServers = monitorService.getServerListFromZK(true); - - - if (masterServers.size() == 0) { - putMsg(result, Status.MASTER_NOT_EXISTS); + // check master exists + if (!checkMasterExists(result)) { return result; } + + /** * create command */ @@ -152,6 +150,22 @@ public class ExecutorService extends BaseService{ return result; } + /** + * check whether master exists + * @param result result + * @return master exists return true , otherwise return false + */ + private boolean checkMasterExists(Map result) { + // check master server exists + List masterServers = monitorService.getServerListFromZK(true); + + // no master + if (masterServers.size() == 0) { + putMsg(result, Status.MASTER_NOT_EXISTS); + return false; + } + return true; + } /** @@ -195,6 +209,12 @@ public class ExecutorService extends BaseService{ return checkResult; } + // check master exists + if (!checkMasterExists(result)) { + return result; + } + + ProcessInstance processInstance = processService.findProcessInstanceDetailById(processInstanceId); if (processInstance == null) { putMsg(result, Status.PROCESS_INSTANCE_NOT_EXIST, processInstanceId); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index 14cadbf189..4fdafa46fa 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -563,14 +563,18 @@ public class ProcessDefinitionService extends BaseDAGService { } /** - * export process definition by id - * - * @param loginUser login user - * @param projectName project name - * @param processDefinitionId process definition id - * @param response response + * batch export process definition by ids + * @param loginUser + * @param projectName + * @param processDefinitionIds + * @param response */ - public void exportProcessDefinitionById(User loginUser, String projectName, Integer processDefinitionId, HttpServletResponse response) { + public void batchExportProcessDefinitionByIds(User loginUser, String projectName, String processDefinitionIds, HttpServletResponse response){ + + if(StringUtils.isEmpty(processDefinitionIds)){ + return; + } + //export project info Project project = projectMapper.queryByName(projectName); @@ -578,39 +582,68 @@ public class ProcessDefinitionService extends BaseDAGService { Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); Status resultStatus = (Status) checkResult.get(Constants.STATUS); - if (resultStatus == Status.SUCCESS) { + if(resultStatus != Status.SUCCESS){ + return; + } + + List processDefinitionList = + getProcessDefinitionList(processDefinitionIds); + + if(CollectionUtils.isNotEmpty(processDefinitionList)){ + downloadProcessDefinitionFile(response, processDefinitionList); + } + } + + /** + * get process definition list by ids + * @param processDefinitionIds + * @return + */ + private List getProcessDefinitionList(String processDefinitionIds){ + List processDefinitionList = new ArrayList<>(); + String[] processDefinitionIdArray = processDefinitionIds.split(","); + for (String strProcessDefinitionId : processDefinitionIdArray) { //get workflow info + int processDefinitionId = Integer.parseInt(strProcessDefinitionId); ProcessDefinition processDefinition = processDefineMapper.queryByDefineId(processDefinitionId); - if (null != processDefinition) { - String exportProcessJson = exportProcessMetaDataStr(processDefinitionId, processDefinition); - response.setContentType(MediaType.APPLICATION_JSON_UTF8_VALUE); - response.setHeader("Content-Disposition", "attachment;filename="+processDefinition.getName()+".json"); - BufferedOutputStream buff = null; - ServletOutputStream out = null; + processDefinitionList.add(exportProcessMetaData(processDefinitionId, processDefinition)); + } + } + + return processDefinitionList; + } + + /** + * download the process definition file + * @param response + * @param processDefinitionList + */ + private void downloadProcessDefinitionFile(HttpServletResponse response, List processDefinitionList) { + response.setContentType(MediaType.APPLICATION_JSON_UTF8_VALUE); + BufferedOutputStream buff = null; + ServletOutputStream out = null; + try { + out = response.getOutputStream(); + buff = new BufferedOutputStream(out); + buff.write(JSON.toJSONString(processDefinitionList).getBytes(StandardCharsets.UTF_8)); + buff.flush(); + buff.close(); + } catch (IOException e) { + logger.warn("export process fail", e); + }finally { + if (null != buff) { try { - out = response.getOutputStream(); - buff = new BufferedOutputStream(out); - buff.write(exportProcessJson.getBytes(StandardCharsets.UTF_8)); - buff.flush(); buff.close(); - } catch (IOException e) { - logger.warn("export process fail", e); - }finally { - if (null != buff) { - try { - buff.close(); - } catch (Exception e) { - logger.warn("export process buffer not close", e); - } - } - if (null != out) { - try { - out.close(); - } catch (Exception e) { - logger.warn("export process output stream not close", e); - } - } + } catch (Exception e) { + logger.warn("export process buffer not close", e); + } + } + if (null != out) { + try { + out.close(); + } catch (Exception e) { + logger.warn("export process output stream not close", e); } } } @@ -623,6 +656,17 @@ public class ProcessDefinitionService extends BaseDAGService { * @return export process metadata string */ public String exportProcessMetaDataStr(Integer processDefinitionId, ProcessDefinition processDefinition) { + //create workflow json file + return JSONUtils.toJsonString(exportProcessMetaData(processDefinitionId,processDefinition)); + } + + /** + * get export process metadata string + * @param processDefinitionId process definition id + * @param processDefinition process definition + * @return export process metadata string + */ + public ProcessMeta exportProcessMetaData(Integer processDefinitionId, ProcessDefinition processDefinition) { //correct task param which has data source or dependent param String correctProcessDefinitionJson = addExportTaskNodeSpecialParam(processDefinition.getProcessDefinitionJson()); processDefinition.setProcessDefinitionJson(correctProcessDefinitionJson); @@ -639,14 +683,6 @@ public class ProcessDefinitionService extends BaseDAGService { List schedules = scheduleMapper.queryByProcessDefinitionId(processDefinitionId); if (!schedules.isEmpty()) { Schedule schedule = schedules.get(0); - /*WorkerGroup workerGroup = workerGroupMapper.selectById(schedule.getWorkerGroupId()); - - if (null == workerGroup && schedule.getWorkerGroupId() == -1) { - workerGroup = new WorkerGroup(); - workerGroup.setId(-1); - workerGroup.setName(""); - }*/ - exportProcessMeta.setScheduleWarningType(schedule.getWarningType().toString()); exportProcessMeta.setScheduleWarningGroupId(schedule.getWarningGroupId()); exportProcessMeta.setScheduleStartTime(DateUtils.dateToString(schedule.getStartTime())); @@ -658,7 +694,7 @@ public class ProcessDefinitionService extends BaseDAGService { exportProcessMeta.setScheduleWorkerGroupName(schedule.getWorkerGroup()); } //create workflow json file - return JSONUtils.toJsonString(exportProcessMeta); + return exportProcessMeta; } /** @@ -705,24 +741,36 @@ public class ProcessDefinitionService extends BaseDAGService { public Map importProcessDefinition(User loginUser, MultipartFile file, String currentProjectName) { Map result = new HashMap<>(5); String processMetaJson = FileUtils.file2String(file); - ProcessMeta processMeta = JSONUtils.parseObject(processMetaJson, ProcessMeta.class); + List processMetaList = JSON.parseArray(processMetaJson,ProcessMeta.class); //check file content - if (null == processMeta) { + if (CollectionUtils.isEmpty(processMetaList)) { putMsg(result, Status.DATA_IS_NULL, "fileContent"); return result; } - if (StringUtils.isEmpty(processMeta.getProjectName())) { - putMsg(result, Status.DATA_IS_NULL, "projectName"); - return result; - } - if (StringUtils.isEmpty(processMeta.getProcessDefinitionName())) { - putMsg(result, Status.DATA_IS_NULL, "processDefinitionName"); - return result; + + for(ProcessMeta processMeta:processMetaList){ + + if (!checkAndImportProcessDefinition(loginUser, currentProjectName, result, processMeta)){ + return result; + } } - if (StringUtils.isEmpty(processMeta.getProcessDefinitionJson())) { - putMsg(result, Status.DATA_IS_NULL, "processDefinitionJson"); - return result; + + return result; + } + + /** + * check and import process definition + * @param loginUser + * @param currentProjectName + * @param result + * @param processMeta + * @return + */ + private boolean checkAndImportProcessDefinition(User loginUser, String currentProjectName, Map result, ProcessMeta processMeta) { + + if(!checkImportanceParams(processMeta,result)){ + return false; } //deal with process name @@ -734,31 +782,84 @@ public class ProcessDefinitionService extends BaseDAGService { processDefinitionName, 1); } - //add special task param - String importProcessParam = addImportTaskNodeParam(loginUser, processMeta.getProcessDefinitionJson(), targetProject); + // get create process result + Map createProcessResult = + getCreateProcessResult(loginUser, + currentProjectName, + result, + processMeta, + processDefinitionName, + addImportTaskNodeParam(loginUser, processMeta.getProcessDefinitionJson(), targetProject)); + + if(createProcessResult == null){ + return false; + } + + //create process definition + Integer processDefinitionId = + Objects.isNull(createProcessResult.get("processDefinitionId"))? + null:Integer.parseInt(createProcessResult.get("processDefinitionId").toString()); + + //scheduler param + return getImportProcessScheduleResult(loginUser, + currentProjectName, + result, + processMeta, + processDefinitionName, + processDefinitionId); + + } - Map createProcessResult; + /** + * get create process result + * @param loginUser + * @param currentProjectName + * @param result + * @param processMeta + * @param processDefinitionName + * @param importProcessParam + * @return + */ + private Map getCreateProcessResult(User loginUser, + String currentProjectName, + Map result, + ProcessMeta processMeta, + String processDefinitionName, + String importProcessParam){ + Map createProcessResult = null; try { createProcessResult = createProcessDefinition(loginUser ,currentProjectName, - processDefinitionName, + processDefinitionName+"_import_"+System.currentTimeMillis(), importProcessParam, processMeta.getProcessDefinitionDescription(), processMeta.getProcessDefinitionLocations(), processMeta.getProcessDefinitionConnects()); + putMsg(result, Status.SUCCESS); } catch (JsonProcessingException e) { logger.error("import process meta json data: {}", e.getMessage(), e); putMsg(result, Status.IMPORT_PROCESS_DEFINE_ERROR); - return result; } - putMsg(result, Status.SUCCESS); - //create process definition - Integer processDefinitionId = null; - if (null != createProcessResult && Objects.nonNull(createProcessResult.get("processDefinitionId"))) { - processDefinitionId = Integer.parseInt(createProcessResult.get("processDefinitionId").toString()); - } - //scheduler param + return createProcessResult; + } + + /** + * get import process schedule result + * @param loginUser + * @param currentProjectName + * @param result + * @param processMeta + * @param processDefinitionName + * @param processDefinitionId + * @return + */ + private boolean getImportProcessScheduleResult(User loginUser, + String currentProjectName, + Map result, + ProcessMeta processMeta, + String processDefinitionName, + Integer processDefinitionId) { if (null != processMeta.getScheduleCrontab() && null != processDefinitionId) { int scheduleInsert = importProcessSchedule(loginUser, currentProjectName, @@ -768,11 +869,33 @@ public class ProcessDefinitionService extends BaseDAGService { if (0 == scheduleInsert) { putMsg(result, Status.IMPORT_PROCESS_DEFINE_ERROR); - return result; + return false; } } + return true; + } - return result; + /** + * check importance params + * @param processMeta + * @param result + * @return + */ + private boolean checkImportanceParams(ProcessMeta processMeta,Map result){ + if (StringUtils.isEmpty(processMeta.getProjectName())) { + putMsg(result, Status.DATA_IS_NULL, "projectName"); + return false; + } + if (StringUtils.isEmpty(processMeta.getProcessDefinitionName())) { + putMsg(result, Status.DATA_IS_NULL, "processDefinitionName"); + return false; + } + if (StringUtils.isEmpty(processMeta.getProcessDefinitionJson())) { + putMsg(result, Status.DATA_IS_NULL, "processDefinitionJson"); + return false; + } + + return true; } /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java index a33754397d..55bb3d336c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java @@ -33,7 +33,7 @@ public class ZooKeeperState { private final String host; private final int port; - private int minLatency = -1, avgLatency = -1, maxLatency = -1; + private float minLatency = -1, avgLatency = -1, maxLatency = -1; private long received = -1; private long sent = -1; private int outStanding = -1; @@ -60,9 +60,9 @@ public class ZooKeeperState { String line = scannerForStat.nextLine(); if (line.startsWith("Latency min/avg/max:")) { String[] latencys = getStringValueFromLine(line).split("/"); - minLatency = Integer.parseInt(latencys[0]); - avgLatency = Integer.parseInt(latencys[1]); - maxLatency = Integer.parseInt(latencys[2]); + minLatency = Float.parseFloat(latencys[0]); + avgLatency = Float.parseFloat(latencys[1]); + maxLatency = Float.parseFloat(latencys[2]); } else if (line.startsWith("Received:")) { received = Long.parseLong(getStringValueFromLine(line)); } else if (line.startsWith("Sent:")) { @@ -165,15 +165,15 @@ public class ZooKeeperState { return port; } - public int getMinLatency() { + public float getMinLatency() { return minLatency; } - public int getAvgLatency() { + public float getAvgLatency() { return avgLatency; } - public int getMaxLatency() { + public float getMaxLatency() { return maxLatency; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitor.java index f91d3bc68c..7ba704916c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitor.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitor.java @@ -88,9 +88,9 @@ public class ZookeeperMonitor extends AbstractZKClient { long sent = state.getSent(); long received = state.getReceived(); String mode = state.getMode(); - int minLatency = state.getMinLatency(); - int avgLatency = state.getAvgLatency(); - int maxLatency = state.getMaxLatency(); + float minLatency = state.getMinLatency(); + float avgLatency = state.getAvgLatency(); + float maxLatency = state.getMaxLatency(); int nodeCount = state.getNodeCount(); int status = ok ? 1 : 0; Date date = new Date(); diff --git a/dolphinscheduler-api/src/main/resources/i18n/messages.properties b/dolphinscheduler-api/src/main/resources/i18n/messages.properties index 369e5e3c72..c8e48ad865 100644 --- a/dolphinscheduler-api/src/main/resources/i18n/messages.properties +++ b/dolphinscheduler-api/src/main/resources/i18n/messages.properties @@ -252,3 +252,5 @@ UNAUTHORIZED_DATA_SOURCE_NOTES=unauthorized data source AUTHORIZED_DATA_SOURCE_NOTES=authorized data source DELETE_SCHEDULER_BY_ID_NOTES=delete scheduler by id QUERY_ALERT_GROUP_LIST_PAGING_NOTES=query alert group list paging +EXPORT_PROCESS_DEFINITION_BY_ID_NOTES=export process definition by id +BATCH_EXPORT_PROCESS_DEFINITION_BY_IDS_NOTES= batch export process definition by ids \ No newline at end of file diff --git a/dolphinscheduler-api/src/main/resources/i18n/messages_en_US.properties b/dolphinscheduler-api/src/main/resources/i18n/messages_en_US.properties index 92df742613..0669e8d8cf 100644 --- a/dolphinscheduler-api/src/main/resources/i18n/messages_en_US.properties +++ b/dolphinscheduler-api/src/main/resources/i18n/messages_en_US.properties @@ -252,3 +252,5 @@ UNAUTHORIZED_DATA_SOURCE_NOTES=unauthorized data source AUTHORIZED_DATA_SOURCE_NOTES=authorized data source DELETE_SCHEDULER_BY_ID_NOTES=delete scheduler by id QUERY_ALERT_GROUP_LIST_PAGING_NOTES=query alert group list paging +EXPORT_PROCESS_DEFINITION_BY_ID_NOTES=export process definition by id +BATCH_EXPORT_PROCESS_DEFINITION_BY_IDS_NOTES= batch export process definition by ids diff --git a/dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties b/dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties index 3b427912b5..9053b0924c 100644 --- a/dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties +++ b/dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties @@ -250,3 +250,6 @@ UNAUTHORIZED_DATA_SOURCE_NOTES=未授权的数据源 AUTHORIZED_DATA_SOURCE_NOTES=授权的数据源 DELETE_SCHEDULER_BY_ID_NOTES=根据定时id删除定时数据 QUERY_ALERT_GROUP_LIST_PAGING_NOTES=分页查询告警组列表 +EXPORT_PROCESS_DEFINITION_BY_ID_NOTES=通过工作流ID导出工作流定义 +BATCH_EXPORT_PROCESS_DEFINITION_BY_IDS_NOTES=批量导出工作流定义 + diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java index a69df9744e..8c0d04c6c6 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java @@ -18,12 +18,14 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.ProcessDefinitionService; +import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.User; import org.junit.*; import org.junit.runner.RunWith; @@ -33,6 +35,8 @@ import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.mock.web.MockHttpServletResponse; +import javax.servlet.http.HttpServletResponse; import java.text.MessageFormat; import java.util.ArrayList; import java.util.HashMap; @@ -111,7 +115,7 @@ public class ProcessDefinitionControllerTest{ } @Test - public void UpdateProcessDefinition() throws Exception { + public void updateProcessDefinition() throws Exception { String json = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-36196\",\"name\":\"ssh_test1\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"aa=\\\"1234\\\"\\necho ${aa}\"},\"desc\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":-1,\"timeout\":0}"; String locations = "{\"tasks-36196\":{\"name\":\"ssh_test1\",\"targetarr\":\"\",\"x\":141,\"y\":70}}"; @@ -287,4 +291,61 @@ public class ProcessDefinitionControllerTest{ Assert.assertEquals(Status.SUCCESS.getCode(),response.getCode().intValue()); } + + @Test + public void testQueryProcessDefinitionAllByProjectId() throws Exception{ + int projectId = 1; + Map result = new HashMap<>(); + putMsg(result,Status.SUCCESS); + + Mockito.when(processDefinitionService.queryProcessDefinitionAllByProjectId(projectId)).thenReturn(result); + Result response = processDefinitionController.queryProcessDefinitionAllByProjectId(user,projectId); + + Assert.assertEquals(Status.SUCCESS.getCode(),response.getCode().intValue()); + } + + @Test + public void testViewTree() throws Exception{ + String projectName = "test"; + int processId = 1; + int limit = 2; + Map result = new HashMap<>(); + putMsg(result,Status.SUCCESS); + + Mockito.when(processDefinitionService.viewTree(processId,limit)).thenReturn(result); + Result response = processDefinitionController.viewTree(user,projectName,processId,limit); + + Assert.assertEquals(Status.SUCCESS.getCode(),response.getCode().intValue()); + } + + @Test + public void testQueryProcessDefinitionListPaging() throws Exception{ + String projectName = "test"; + int pageNo = 1; + int pageSize = 10; + String searchVal = ""; + int userId = 1; + + Map result = new HashMap<>(); + putMsg(result,Status.SUCCESS); + result.put(Constants.DATA_LIST,new PageInfo(1,10)); + + Mockito.when(processDefinitionService.queryProcessDefinitionListPaging(user,projectName, searchVal, pageNo, pageSize, userId)).thenReturn(result); + Result response = processDefinitionController.queryProcessDefinitionListPaging(user,projectName,pageNo,searchVal,userId,pageSize); + + Assert.assertEquals(Status.SUCCESS.getCode(),response.getCode().intValue()); + } + + @Test + public void testBatchExportProcessDefinitionByIds() throws Exception{ + + String processDefinitionIds = "1,2"; + String projectName = "test"; + HttpServletResponse response = new MockHttpServletResponse(); + ProcessDefinitionService service = new ProcessDefinitionService(); + ProcessDefinitionService spy = Mockito.spy(service); + Mockito.doNothing().when(spy).batchExportProcessDefinitionByIds(user, projectName, processDefinitionIds, response); + processDefinitionController.batchExportProcessDefinitionByIds(user, projectName, processDefinitionIds, response); + } + } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandlerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandlerTest.java index 95cd96d08e..c00b1b0613 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandlerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandlerTest.java @@ -46,7 +46,7 @@ public class ApiExceptionHandlerTest { public void exceptionHandlerRuntime() throws NoSuchMethodException { ApiExceptionHandler handler = new ApiExceptionHandler(); ProcessDefinitionController controller = new ProcessDefinitionController(); - Method method = controller.getClass().getMethod("exportProcessDefinitionById", User.class, String.class, Integer.class, HttpServletResponse.class); + Method method = controller.getClass().getMethod("batchExportProcessDefinitionByIds", User.class, String.class, String.class, HttpServletResponse.class); HandlerMethod hm = new HandlerMethod(controller, method); Result result = handler.exceptionHandler(new RuntimeException("test exception"), hm); Assert.assertEquals(Status.INTERNAL_SERVER_ERROR_ARGS.getCode(),result.getCode().intValue()); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java index 5a03cdb268..edf4ef7b97 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java @@ -667,32 +667,60 @@ public class ProcessDefinitionServiceTest { } + @Test + public void testCreateProcess() throws IOException{ + + String json = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-36196\",\"name\":\"ssh_test1\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"aa=\\\"1234\\\"\\necho ${aa}\"},\"desc\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":-1,\"timeout\":0}"; + String locations = "{\"tasks-36196\":{\"name\":\"ssh_test1\",\"targetarr\":\"\",\"x\":141,\"y\":70}}"; + + String projectName = "test"; + String name = "dag_test"; + String description = "desc test"; + String connects = "[]"; + Map result = new HashMap<>(5); + putMsg(result, Status.SUCCESS); + result.put("processDefinitionId",1); + + Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName)); + User loginUser = new User(); + loginUser.setId(1); + loginUser.setUserType(UserType.ADMIN_USER); + Project project = getProject(projectName); + + //project not found + Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result); + Mockito.when(processDefineMapper.insert(getProcessDefinition())).thenReturn(1); + Map result1 = processDefinitionService.createProcessDefinition(loginUser,projectName,name,json,description,locations,connects); + + Assert.assertEquals(Status.SUCCESS,result1.get(Constants.STATUS)); + } + @Test public void testImportProcessDefinitionById() throws IOException { - String processJson = "{\"projectName\":\"testProject\",\"processDefinitionName\":\"shell-4\"," + - "\"processDefinitionJson\":\"{\\\"tenantId\\\":1,\\\"globalParams\\\":[]," + - "\\\"tasks\\\":[{\\\"workerGroupId\\\":-1,\\\"description\\\":\\\"\\\",\\\"runFlag\\\":\\\"NORMAL\\\"," + - "\\\"type\\\":\\\"SHELL\\\",\\\"params\\\":{\\\"rawScript\\\":\\\"#!/bin/bash\\\\necho \\\\\\\"shell-4\\\\\\\"\\\"," + - "\\\"localParams\\\":[],\\\"resourceList\\\":[]},\\\"timeout\\\":{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}," + - "\\\"maxRetryTimes\\\":\\\"0\\\",\\\"taskInstancePriority\\\":\\\"MEDIUM\\\",\\\"name\\\":\\\"shell-4\\\"," + - "\\\"dependence\\\":{},\\\"retryInterval\\\":\\\"1\\\",\\\"preTasks\\\":[],\\\"id\\\":\\\"tasks-84090\\\"}," + - "{\\\"taskInstancePriority\\\":\\\"MEDIUM\\\",\\\"name\\\":\\\"shell-5\\\",\\\"workerGroupId\\\":-1," + - "\\\"description\\\":\\\"\\\",\\\"dependence\\\":{},\\\"preTasks\\\":[\\\"shell-4\\\"],\\\"id\\\":\\\"tasks-87364\\\"," + - "\\\"runFlag\\\":\\\"NORMAL\\\",\\\"type\\\":\\\"SUB_PROCESS\\\",\\\"params\\\":{\\\"processDefinitionId\\\":46}," + - "\\\"timeout\\\":{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}}],\\\"timeout\\\":0}\"," + - "\"processDefinitionDescription\":\"\",\"processDefinitionLocations\":\"{\\\"tasks-84090\\\":{\\\"name\\\":\\\"shell-4\\\"," + - "\\\"targetarr\\\":\\\"\\\",\\\"x\\\":128,\\\"y\\\":114},\\\"tasks-87364\\\":{\\\"name\\\":\\\"shell-5\\\"," + - "\\\"targetarr\\\":\\\"tasks-84090\\\",\\\"x\\\":266,\\\"y\\\":115}}\"," + - "\"processDefinitionConnects\":\"[{\\\"endPointSourceId\\\":\\\"tasks-84090\\\"," + - "\\\"endPointTargetId\\\":\\\"tasks-87364\\\"}]\"}"; - - String subProcessJson = "{\"globalParams\":[]," + - "\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-52423\",\"name\":\"shell-5\"," + - "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo \\\"shell-5\\\"\"},\"description\":\"\"," + - "\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + - "\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1," + - "\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}"; + String json = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-36196\",\"name\":\"ssh_test1\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"aa=\\\"1234\\\"\\necho ${aa}\"},\"desc\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":-1,\"timeout\":0}"; + String locations = "{\"tasks-36196\":{\"name\":\"ssh_test1\",\"targetarr\":\"\",\"x\":141,\"y\":70}}"; + + String projectName = "test"; + String name = "dag_test"; + String description = "desc test"; + String connects = "[]"; + Map result = new HashMap<>(5); + putMsg(result, Status.SUCCESS); + result.put("processDefinitionId",1); + + Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName)); + User loginUser = new User(); + loginUser.setId(1); + loginUser.setUserType(UserType.ADMIN_USER); + Project project = getProject(projectName); + + //project not found + Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result); + Mockito.when(processDefineMapper.insert(getProcessDefinition())).thenReturn(1); + Map result1 = processDefinitionService.createProcessDefinition(loginUser,projectName,name,json,description,locations,connects); + + String processJson = "[{\"processDefinitionConnects\":\"[]\",\"processDefinitionJson\":\"{\\\"tenantId\\\":-1,\\\"globalParams\\\":[],\\\"tasks\\\":[{\\\"workerGroupId\\\":-1,\\\"runFlag\\\":\\\"NORMAL\\\",\\\"type\\\":\\\"SHELL\\\",\\\"params\\\":{\\\"rawScript\\\":\\\"aa=\\\\\\\"1234\\\\\\\"\\\\necho ${aa}\\\",\\\"localParams\\\":[],\\\"resourceList\\\":[]},\\\"timeout\\\":{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"},\\\"maxRetryTimes\\\":\\\"0\\\",\\\"taskInstancePriority\\\":\\\"MEDIUM\\\",\\\"name\\\":\\\"ssh_test1\\\",\\\"dependence\\\":{},\\\"retryInterval\\\":\\\"1\\\",\\\"preTasks\\\":[],\\\"id\\\":\\\"tasks-36196\\\",\\\"desc\\\":\\\"\\\"}],\\\"timeout\\\":0}\",\"processDefinitionLocations\":\"{\\\"tasks-36196\\\":{\\\"name\\\":\\\"ssh_test1\\\",\\\"targetarr\\\":\\\"\\\",\\\"x\\\":141,\\\"y\\\":70}}\",\"processDefinitionName\":\"dag_test\",\"projectName\":\"test\"}]"; FileUtils.writeStringToFile(new File("/tmp/task.json"),processJson); @@ -703,23 +731,16 @@ public class ProcessDefinitionServiceTest { MultipartFile multipartFile = new MockMultipartFile(file.getName(), file.getName(), ContentType.APPLICATION_OCTET_STREAM.toString(), fileInputStream); - User loginUser = new User(); - loginUser.setId(1); - loginUser.setUserType(UserType.ADMIN_USER); - - String currentProjectName = "testProject"; - Map result = new HashMap<>(5); - putMsg(result, Status.SUCCESS, currentProjectName); + String currentProjectName = "test"; ProcessDefinition shellDefinition2 = new ProcessDefinition(); - shellDefinition2.setId(46); - shellDefinition2.setName("shell-5"); - shellDefinition2.setProjectId(2); - shellDefinition2.setProcessDefinitionJson(subProcessJson); + shellDefinition2.setId(25); + shellDefinition2.setName("B"); + shellDefinition2.setProjectId(1); Mockito.when(projectMapper.queryByName(currentProjectName)).thenReturn(getProject(currentProjectName)); Mockito.when(projectService.checkProjectAndAuth(loginUser, getProject(currentProjectName), currentProjectName)).thenReturn(result); - Mockito.when(processDefineMapper.queryByDefineId(46)).thenReturn(shellDefinition2); + Mockito.when(processDefineMapper.queryByDefineId(25)).thenReturn(shellDefinition2); //import process Map importProcessResult = processDefinitionService.importProcessDefinition(loginUser, multipartFile, currentProjectName); @@ -730,18 +751,17 @@ public class ProcessDefinitionServiceTest { Assert.assertTrue(delete); - String processMetaJson = ""; - improssProcessCheckData(file, loginUser, currentProjectName, processMetaJson); - - processMetaJson = "{\"scheduleWorkerGroupId\":-1}"; - improssProcessCheckData(file, loginUser, currentProjectName, processMetaJson); - - processMetaJson = "{\"scheduleWorkerGroupId\":-1,\"projectName\":\"test\"}"; - improssProcessCheckData(file, loginUser, currentProjectName, processMetaJson); + String processMetaJson = "[]"; + importProcessCheckData(file, loginUser, currentProjectName, processMetaJson); +// + processMetaJson = "[{\"scheduleWorkerGroupId\":-1}]"; + importProcessCheckData(file, loginUser, currentProjectName, processMetaJson); - processMetaJson = "{\"scheduleWorkerGroupId\":-1,\"projectName\":\"test\",\"processDefinitionName\":\"test_definition\"}"; - improssProcessCheckData(file, loginUser, currentProjectName, processMetaJson); + processMetaJson = "[{\"scheduleWorkerGroupId\":-1,\"projectName\":\"test\"}]"; + importProcessCheckData(file, loginUser, currentProjectName, processMetaJson); + processMetaJson = "[{\"scheduleWorkerGroupId\":-1,\"projectName\":\"test\",\"processDefinitionName\":\"test_definition\"}]"; + importProcessCheckData(file, loginUser, currentProjectName, processMetaJson); } @@ -753,7 +773,7 @@ public class ProcessDefinitionServiceTest { * @param processMetaJson process meta json * @throws IOException IO exception */ - private void improssProcessCheckData(File file, User loginUser, String currentProjectName, String processMetaJson) throws IOException { + private void importProcessCheckData(File file, User loginUser, String currentProjectName, String processMetaJson) throws IOException { //check null FileUtils.writeStringToFile(new File("/tmp/task.json"),processMetaJson); diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index ca75a84a62..3954159dfa 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -32,12 +32,15 @@ 3.1.0 + + org.apache.dolphinscheduler + dolphinscheduler-plugin-api + com.alibaba fastjson compile - org.apache.httpcomponents httpclient diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 853ab95d1c..effa4f0f8e 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -954,4 +954,11 @@ public final class Constants { * authorize readable perm */ public static final int AUTHORIZE_READABLE_PERM=4; + + + /** + * plugin configurations + */ + public static final String PLUGIN_JAR_SUFFIX = ".jar"; + } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/FilePluginManager.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/FilePluginManager.java new file mode 100644 index 0000000000..d4dd4bf283 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/FilePluginManager.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.plugin; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.plugin.api.AlertPlugin; +import org.apache.dolphinscheduler.plugin.spi.AlertPluginProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Map; +import java.util.ServiceLoader; +import java.util.concurrent.ConcurrentHashMap; + +/** + * FilePluginManager + */ +public class FilePluginManager implements PluginManager { + + private static final Logger logger = LoggerFactory.getLogger(FilePluginManager.class); + + private Map pluginMap = new ConcurrentHashMap<>(); + + private Map> pluginLoaderMap = new ConcurrentHashMap<>(); + + private Map classLoaderMap = new ConcurrentHashMap<>(); + + private String[] whitePrefixes; + + private String[] excludePrefixes; + + public FilePluginManager(String dirPath, String[] whitePrefixes, String[] excludePrefixes) { + this.whitePrefixes = whitePrefixes; + this.excludePrefixes = excludePrefixes; + try { + load(dirPath); + } catch (MalformedURLException e) { + logger.error("load plugins failed.", e); + } + } + + private void load(String dirPath) throws MalformedURLException { + logger.info("start to load jar files in {}", dirPath); + if (dirPath == null) { + logger.error("not a valid path - {}", dirPath); + return; + } + File[] files = new File(dirPath).listFiles(); + if (files == null) { + logger.error("not a valid path - {}", dirPath); + return; + } + for (File file : files) { + if (file.isDirectory() && !file.getPath().endsWith(Constants.PLUGIN_JAR_SUFFIX)) { + continue; + } + String pluginName = file.getName() + .substring(0, file.getName().length() - Constants.PLUGIN_JAR_SUFFIX.length()); + URL[] urls = new URL[]{ file.toURI().toURL() }; + PluginClassLoader classLoader = + new PluginClassLoader(urls, Thread.currentThread().getContextClassLoader(), whitePrefixes, excludePrefixes); + classLoaderMap.put(pluginName, classLoader); + + ServiceLoader loader = ServiceLoader.load(AlertPluginProvider.class, classLoader); + pluginLoaderMap.put(pluginName, loader); + + loader.forEach(provider -> { + AlertPlugin plugin = provider.createPlugin(); + pluginMap.put(plugin.getId(), plugin); + logger.info("loaded plugin - {}", plugin.getId()); + }); + } + } + + @Override + public AlertPlugin findOne(String name) { + return pluginMap.get(name); + } + + @Override + public Map findAll() { + return pluginMap; + } + + @Override + public void addPlugin(AlertPlugin plugin) { + pluginMap.put(plugin.getId(), plugin); + } + +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/PluginClassLoader.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/PluginClassLoader.java new file mode 100644 index 0000000000..528e83a59c --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/PluginClassLoader.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.plugin; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Enumeration; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; + +/** + * Plugin Class Loader + */ +public class PluginClassLoader extends URLClassLoader { + + private static final Logger logger = LoggerFactory.getLogger(PluginClassLoader.class); + + private static final String JAVA_PACKAGE_PREFIX = "java."; + private static final String JAVAX_PACKAGE_PREFIX = "javax."; + + private final String[] whitePrefixes; + + private final String[] excludePrefixes; + + public PluginClassLoader(URL[] urls, ClassLoader parent, String[] whitePrefix, String[] excludePreifx) { + super(urls, parent); + this.whitePrefixes = whitePrefix; + this.excludePrefixes = excludePreifx; + } + + @Override + public Class loadClass(String name) throws ClassNotFoundException { + logger.trace("Received request to load class '{}'", name); + synchronized (getClassLoadingLock(name)) { + if (name.startsWith(JAVA_PACKAGE_PREFIX) || name.startsWith(JAVAX_PACKAGE_PREFIX)) { + return findSystemClass(name); + } + + boolean isWhitePrefixes = fromWhitePrefix(name); + boolean isExcludePrefixed = fromExcludePrefix(name); + + // if the class is part of the plugin engine use parent class loader + if (!isWhitePrefixes && isExcludePrefixed) { + return getParent().loadClass(name); + } + + // check whether it's already been loaded + Class loadedClass = findLoadedClass(name); + if (loadedClass != null) { + logger.debug("Found loaded class '{}'", name); + return loadedClass; + } + + // nope, try to load locally + try { + loadedClass = findClass(name); + logger.debug("Found class '{}' in plugin classpath", name); + return loadedClass; + } catch (ClassNotFoundException e) { + // try next step + } + + // use the standard ClassLoader (which follows normal parent delegation) + return super.loadClass(name); + } + } + + private boolean fromWhitePrefix(String name) { + if (this.whitePrefixes == null) { + return false; + } + for (String whitePrefix : this.whitePrefixes) { + if (name.startsWith(whitePrefix)) { + return true; + } + } + return false; + } + + private boolean fromExcludePrefix(String name) { + if (this.excludePrefixes == null) { + return false; + } + for (String excludePrefix : this.excludePrefixes) { + if (name.startsWith(excludePrefix)) { + return true; + } + } + return false; + } + + @Override + public Enumeration getResources(String name) throws IOException { + List allRes = new LinkedList<>(); + + Enumeration thisRes = findResources(name); + if (thisRes != null) { + while (thisRes.hasMoreElements()) { + allRes.add(thisRes.nextElement()); + } + } + + Enumeration parentRes = super.findResources(name); + if (parentRes != null) { + while (parentRes.hasMoreElements()) { + allRes.add(parentRes.nextElement()); + } + } + + return new Enumeration() { + Iterator it = allRes.iterator(); + + @Override + public boolean hasMoreElements() { + return it.hasNext(); + } + + @Override + public URL nextElement() { + return it.next(); + } + }; + } + + @Override + public URL getResource(String name) { + URL res = null; + + res = findResource(name); + if (res == null) { + res = super.getResource(name); + } + return res; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/PluginManager.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/PluginManager.java new file mode 100644 index 0000000000..f8078841e4 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/plugin/PluginManager.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.plugin; + +import org.apache.dolphinscheduler.plugin.api.AlertPlugin; + +import java.util.Map; + +/** + * PluginManager + */ +public interface PluginManager { + + AlertPlugin findOne(String name); + + Map findAll(); + + void addPlugin(AlertPlugin plugin); +} diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index db3b241ca9..d066f7b6b7 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -62,5 +62,5 @@ yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s # system env path #dolphinscheduler.env.path=env/dolphinscheduler_env.sh - +development.state=false kerberos.expire.time=7 \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/plugin/FilePluginManagerTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/plugin/FilePluginManagerTest.java new file mode 100644 index 0000000000..1a57cb10fa --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/plugin/FilePluginManagerTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.plugin; + +import org.apache.dolphinscheduler.plugin.api.AlertPlugin; +import org.apache.dolphinscheduler.plugin.model.AlertInfo; +import org.apache.dolphinscheduler.plugin.model.PluginName; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.*; + +public class FilePluginManagerTest { + + private FilePluginManager filePluginManager; + private AlertPlugin alertPlugin; + + @Before + public void before() { + filePluginManager = new FilePluginManager(null, null, null); + alertPlugin = new AlertPlugin() { + @Override + public String getId() { + return "test"; + } + + @Override + public PluginName getName() { + return new PluginName().setChinese("ch").setEnglish("en"); + } + + @Override + public Map process(AlertInfo info) { + return new HashMap<>(); + } + }; + } + + @Test + public void findOne() { + filePluginManager.addPlugin(alertPlugin); + assertEquals(alertPlugin, filePluginManager.findOne(alertPlugin.getId())); + } + + @Test + public void findAll() { + assertNotNull(filePluginManager.findAll()); + } + + @Test + public void addPlugin() { + filePluginManager.addPlugin(alertPlugin); + assertNotNull(filePluginManager.findAll()); + } +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/plugin/PluginClassLoaderTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/plugin/PluginClassLoaderTest.java new file mode 100644 index 0000000000..8a6bfaee13 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/plugin/PluginClassLoaderTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.plugin; + +import org.junit.Before; +import org.junit.Test; + +import java.net.URL; + +import static org.junit.Assert.*; + +public class PluginClassLoaderTest { + + private PluginClassLoader pluginClassLoader; + private ClassLoader parent; + + @Before + public void setUp() { + parent = Thread.currentThread().getContextClassLoader(); + pluginClassLoader = new PluginClassLoader( + new URL[]{}, parent, + null, null); + } + + @Test + public void loadClassNull() { + Class clazz = null; + try { + clazz = pluginClassLoader.loadClass("java.lang.Object"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + assertEquals(null, clazz.getClassLoader()); + } + + @Test + public void loadClassApp() { + Class clazz = null; + try { + clazz = pluginClassLoader.loadClass("org.apache.dolphinscheduler.common.Constants"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + assertEquals(parent, clazz.getClassLoader()); + } + +} \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ZookeeperRecord.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ZookeeperRecord.java index 6b86ac5c11..7d45c66940 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ZookeeperRecord.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ZookeeperRecord.java @@ -56,17 +56,17 @@ public class ZookeeperRecord { /** * min Latency */ - private int minLatency; + private float minLatency; /** * avg Latency */ - private int avgLatency; + private float avgLatency; /** * max Latency */ - private int maxLatency; + private float maxLatency; /** * node count @@ -85,7 +85,7 @@ public class ZookeeperRecord { private int state; - public ZookeeperRecord(String hostname,int connections, int watches, long sent, long received, String mode, int minLatency, int avgLatency, int maxLatency, int nodeCount, int state,Date date) { + public ZookeeperRecord(String hostname,int connections, int watches, long sent, long received, String mode, float minLatency, float avgLatency, float maxLatency, int nodeCount, int state,Date date) { this.hostname = hostname; this.connections = connections; this.watches = watches; @@ -149,23 +149,23 @@ public class ZookeeperRecord { this.mode = mode; } - public int getMinLatency() { + public float getMinLatency() { return minLatency; } - public void setMinLatency(int minLatency) { + public void setMinLatency(float minLatency) { this.minLatency = minLatency; } - public int getAvgLatency() { + public float getAvgLatency() { return avgLatency; } - public void setAvgLatency(int avgLatency) { + public void setAvgLatency(float avgLatency) { this.avgLatency = avgLatency; } - public int getMaxLatency() { + public float getMaxLatency() { return maxLatency; } diff --git a/dolphinscheduler-dao/src/main/resources/datasource.properties b/dolphinscheduler-dao/src/main/resources/datasource.properties index 2f28ca2b0b..984aa17d57 100644 --- a/dolphinscheduler-dao/src/main/resources/datasource.properties +++ b/dolphinscheduler-dao/src/main/resources/datasource.properties @@ -14,17 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # - - -# postgre -#spring.datasource.driver-class-name=org.postgresql.Driver -#spring.datasource.url=jdbc:postgresql://localhost:5432/dolphinscheduler -# mysql + +# postgresql spring.datasource.driver-class-name=org.postgresql.Driver spring.datasource.url=jdbc:postgresql://localhost:5432/dolphinscheduler spring.datasource.username=test spring.datasource.password=test +# mysql # connection configuration #spring.datasource.initialSize=5 # min connection number diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index 82e641ec72..52563f5573 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -474,7 +474,6 @@ The following components are provided under a MIT 2.0 license. See project link The text of each license is also included at licenses/LICENSE-[project].txt. jul-to-slf4j 1.7.25: https://mvnrepository.com/artifact/org.slf4j/jul-to-slf4j/1.7.25, MIT - lombok 1.18.6: https://mvnrepository.com/artifact/org.projectlombok/lombok/1.18.6, MIT mssql-jdbc 6.1.0.jre8: https://mvnrepository.com/artifact/com.microsoft.sqlserver/mssql-jdbc/6.1.0.jre8, MIT slf4j-api 1.7.5: https://mvnrepository.com/artifact/org.slf4j/slf4j-api/1.7.5, MIT diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-lombok.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-lombok.txt deleted file mode 100644 index 5da7dbeff3..0000000000 --- a/dolphinscheduler-dist/release-docs/licenses/LICENSE-lombok.txt +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (C) 2009-2015 The Project Lombok Authors. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/dolphinscheduler-plugin-api/pom.xml b/dolphinscheduler-plugin-api/pom.xml new file mode 100644 index 0000000000..b958b21616 --- /dev/null +++ b/dolphinscheduler-plugin-api/pom.xml @@ -0,0 +1,52 @@ + + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler + 1.2.1-SNAPSHOT + + dolphinscheduler-plugin-api + ${project.artifactId} + jar + + + UTF-8 + + + + + org.slf4j + slf4j-api + + + junit + junit + test + + + commons-io + commons-io + + + + diff --git a/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/api/AlertPlugin.java b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/api/AlertPlugin.java new file mode 100644 index 0000000000..deb7ff6aa4 --- /dev/null +++ b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/api/AlertPlugin.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.api; + +import org.apache.dolphinscheduler.plugin.model.AlertInfo; +import org.apache.dolphinscheduler.plugin.model.PluginName; + +import java.util.Map; + +/** + * Plugin + */ +public interface AlertPlugin { + + /** + * Get alert plugin id + * + * @return alert plugin id, which should be unique + */ + String getId(); + + /** + * Get alert plugin name, which will show in front end portal + * + * @return plugin name + */ + PluginName getName(); + + Map process(AlertInfo info); + +} diff --git a/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/AlertData.java b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/AlertData.java new file mode 100644 index 0000000000..89ab5c4279 --- /dev/null +++ b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/AlertData.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.model; + +/** + * AlertData + */ +public class AlertData { + + /** + * alert primary key + */ + private int id; + /** + * title + */ + private String title; + /** + * content + */ + private String content; + /** + * log + */ + private String log; + /** + * alertgroup_id + */ + private int alertGroupId; + /** + * receivers + */ + private String receivers; + /** + * show_type + */ + private String showType; + /** + * receivers_cc + */ + private String receiversCc; + + public int getId() { + return id; + } + + public AlertData setId(int id) { + this.id = id; + return this; + } + + public String getTitle() { + return title; + } + + public AlertData setTitle(String title) { + this.title = title; + return this; + } + + public String getContent() { + return content; + } + + public AlertData setContent(String content) { + this.content = content; + return this; + } + + public String getLog() { + return log; + } + + public AlertData setLog(String log) { + this.log = log; + return this; + } + + public int getAlertGroupId() { + return alertGroupId; + } + + public AlertData setAlertGroupId(int alertGroupId) { + this.alertGroupId = alertGroupId; + return this; + } + + public String getReceivers() { + return receivers; + } + + public AlertData setReceivers(String receivers) { + this.receivers = receivers; + return this; + } + + public String getReceiversCc() { + return receiversCc; + } + + public AlertData setReceiversCc(String receiversCc) { + this.receiversCc = receiversCc; + return this; + } + + public String getShowType() { + return showType; + } + + public AlertData setShowType(String showType) { + this.showType = showType; + return this; + } + +} diff --git a/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/AlertInfo.java b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/AlertInfo.java new file mode 100644 index 0000000000..1d71ed7d8a --- /dev/null +++ b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/AlertInfo.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.model; + +import java.util.HashMap; +import java.util.Map; + +/** + * AlertInfo + */ +public class AlertInfo { + + private Map alertProps; + + private AlertData alertData; + + public AlertInfo() { + this.alertProps = new HashMap<>(); + } + + public Map getAlertProps() { + return alertProps; + } + + public AlertInfo setAlertProps(Map alertProps) { + this.alertProps = alertProps; + return this; + } + + public AlertInfo addProp(String key, Object value) { + this.alertProps.put(key, value); + return this; + } + + public Object getProp(String key) { + return this.alertProps.get(key); + } + + public AlertData getAlertData() { + return alertData; + } + + public AlertInfo setAlertData(AlertData alertData) { + this.alertData = alertData; + return this; + } +} diff --git a/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/PluginName.java b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/PluginName.java new file mode 100644 index 0000000000..8066e45f1d --- /dev/null +++ b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/model/PluginName.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.model; + +/** + * PluginName + */ +public class PluginName { + + private String chinese; + + private String english; + + public String getChinese() { + return chinese; + } + + public PluginName setChinese(String chinese) { + this.chinese = chinese; + return this; + } + + public String getEnglish() { + return english; + } + + public PluginName setEnglish(String english) { + this.english = english; + return this; + } +} diff --git a/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/spi/AlertPluginProvider.java b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/spi/AlertPluginProvider.java new file mode 100644 index 0000000000..594636f4eb --- /dev/null +++ b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/spi/AlertPluginProvider.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.spi; + +import org.apache.dolphinscheduler.plugin.api.AlertPlugin; + +/** + * PluginProvider + */ +public interface AlertPluginProvider { + + /** + * create an alert plugin + * + * @return an alert plugin + */ + AlertPlugin createPlugin(); + +} diff --git a/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/utils/PropertyUtils.java b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/utils/PropertyUtils.java new file mode 100644 index 0000000000..a244dd491e --- /dev/null +++ b/dolphinscheduler-plugin-api/src/main/java/org/apache/dolphinscheduler/plugin/utils/PropertyUtils.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.utils; + +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +/** + * property utils + * single instance + */ +public class PropertyUtils { + + private static final Logger logger = LoggerFactory.getLogger(PropertyUtils.class); + + private static final Properties properties = new Properties(); + + private PropertyUtils() { + throw new IllegalStateException("PropertyUtils class"); + } + + static { + String propertyFiles = "/plugin.properties"; + InputStream fis = null; + try { + fis = PropertyUtils.class.getResourceAsStream(propertyFiles); + properties.load(fis); + } catch (IOException e) { + logger.error(e.getMessage(), e); + if (fis != null) { + IOUtils.closeQuietly(fis); + } + } finally { + IOUtils.closeQuietly(fis); + } + } + + /** + * get property value + * + * @param key property name + * @param defaultVal default value + * @return property value + */ + public static String getString(String key, String defaultVal) { + String val = properties.getProperty(key.trim()); + return val == null ? defaultVal : val; + } + + /** + * get property value + * + * @param key property name + * @return property value + */ + public static String getString(String key) { + if (key == null) { + return null; + } + return properties.getProperty(key.trim()); + } + + /** + * get property value + * + * @param key property name + * @return get property int value , if key == null, then return -1 + */ + public static int getInt(String key) { + return getInt(key, -1); + } + + /** + * get int + * + * @param key key + * @param defaultValue default value + * @return property value + */ + public static int getInt(String key, int defaultValue) { + String value = properties.getProperty(key.trim()); + if (value == null) { + return defaultValue; + } + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + logger.info(e.getMessage(), e); + } + return defaultValue; + } + + /** + * get property value + * + * @param key property name + * @return property value + */ + public static boolean getBoolean(String key) { + String value = properties.getProperty(key.trim()); + if (value == null) { + return false; + } + + return Boolean.parseBoolean(value); + } + + /** + * get property value + * + * @param key property name + * @param defaultValue default value + * @return property value + */ + public static Boolean getBoolean(String key, boolean defaultValue) { + String value = properties.getProperty(key.trim()); + if (value == null) { + return defaultValue; + } + + return Boolean.parseBoolean(value); + } + + /** + * get property long value + * + * @param key key + * @param defaultVal default value + * @return property value + */ + public static long getLong(String key, long defaultVal) { + String val = getString(key); + return val == null ? defaultVal : Long.parseLong(val); + } + + /** + * get long + * + * @param key key + * @return property value + */ + public static long getLong(String key) { + return getLong(key, -1); + } + + /** + * get double + * + * @param key key + * @param defaultVal default value + * @return property value + */ + public static double getDouble(String key, double defaultVal) { + String val = properties.getProperty(key.trim()); + return val == null ? defaultVal : Double.parseDouble(val); + } + + /** + * @param key key + * @param type type + * @param defaultValue default value + * @param T + * @return get enum value + */ + public > T getEnum(String key, Class type, + T defaultValue) { + String val = properties.getProperty(key.trim()); + return val == null ? defaultValue : Enum.valueOf(type, val); + } + +} diff --git a/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/model/AlertDataTest.java b/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/model/AlertDataTest.java new file mode 100644 index 0000000000..c19b5bc29a --- /dev/null +++ b/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/model/AlertDataTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.model; + +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class AlertDataTest { + + private AlertData alertData; + + @Before + public void before() { + alertData = new AlertData(); + alertData.setId(1) + .setContent("content") + .setShowType("email") + .setTitle("title") + .setReceivers("receivers") + .setReceiversCc("cc") + .setLog("log") + .setAlertGroupId(1); + } + + @Test + public void getId() { + assertEquals(1, alertData.getId()); + } + + @Test + public void getTitle() { + assertEquals("title", alertData.getTitle()); + } + + @Test + public void getContent() { + assertEquals("content", alertData.getContent()); + } + + @Test + public void getLog() { + assertEquals("log", alertData.getLog()); + } + + @Test + public void getAlertGroupId() { + assertEquals(1, alertData.getAlertGroupId()); + } + + @Test + public void getReceivers() { + assertEquals("receivers", alertData.getReceivers()); + } + + @Test + public void getReceiversCc() { + assertEquals("cc", alertData.getReceiversCc()); + } + + @Test + public void getShowType() { + assertEquals("email", alertData.getShowType()); + } +} \ No newline at end of file diff --git a/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/model/AlertInfoTest.java b/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/model/AlertInfoTest.java new file mode 100644 index 0000000000..13eb595ac3 --- /dev/null +++ b/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/model/AlertInfoTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.model; + +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.*; + +public class AlertInfoTest { + + private AlertInfo alertInfo; + + @Before + public void before() { + alertInfo = new AlertInfo(); + } + + @Test + public void getAlertProps() { + Map map = new HashMap<>(); + alertInfo.setAlertProps(map); + assertNotNull(alertInfo.getAlertProps()); + } + + @Test + public void getProp() { + alertInfo.addProp("k", "v"); + assertEquals("v", alertInfo.getProp("k")); + } + + @Test + public void getAlertData() { + alertInfo.setAlertData(new AlertData()); + assertNotNull(alertInfo.getAlertData()); + } +} \ No newline at end of file diff --git a/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/utils/PropertyUtilsTest.java b/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/utils/PropertyUtilsTest.java new file mode 100644 index 0000000000..614a7c009c --- /dev/null +++ b/dolphinscheduler-plugin-api/src/test/java/org/apache/dolphinscheduler/plugin/utils/PropertyUtilsTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.utils; + +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.junit.Assert.*; + +public class PropertyUtilsTest { + + private static final Logger logger = LoggerFactory.getLogger(PropertyUtilsTest.class); + + /** + * Test getString + */ + @Test + public void testGetString() { + + String result = PropertyUtils.getString("test.string"); + logger.info(result); + assertEquals("teststring", result); + + //If key is null, then return null + result = PropertyUtils.getString(null); + assertNull(result); + } + + + /** + * Test getBoolean + */ + @Test + public void testGetBoolean() { + + //Expected true + Boolean result = PropertyUtils.getBoolean("test.true"); + assertTrue(result); + + //Expected false + result = PropertyUtils.getBoolean("test.false"); + assertFalse(result); + } + + /** + * Test getLong + */ + @Test + public void testGetLong() { + long result = PropertyUtils.getLong("test.long"); + assertSame(100L, result); + } + + /** + * Test getDouble + */ + @Test + public void testGetDouble() { + + //If key is undefine in alert.properties, and there is a defaultval, then return defaultval + double result = PropertyUtils.getDouble("abc", 5.0); + assertEquals(5.0, result, 0); + + result = PropertyUtils.getDouble("cba", 5.0); + assertEquals(3.1, result, 0.01); + } + +} \ No newline at end of file diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/alert.properties.j2 b/dolphinscheduler-plugin-api/src/test/resources/plugin.properties similarity index 90% rename from ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/alert.properties.j2 rename to dolphinscheduler-plugin-api/src/test/resources/plugin.properties index 73840b8c18..d2ea3831be 100644 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/alert.properties.j2 +++ b/dolphinscheduler-plugin-api/src/test/resources/plugin.properties @@ -15,6 +15,8 @@ # limitations under the License. # -{% for key, value in dolphin_alert_map.iteritems() -%} - {{key}}={{value}} -{% endfor %} \ No newline at end of file +test.string=teststring +test.false=false +test.true=true +cba=3.1 +test.long=100 \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index 84e4e54a50..ccd4bb2214 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -457,7 +457,7 @@ public class SqlTask extends AbstractTask { String showTypeName = sqlParameters.getShowType().replace(COMMA,"").trim(); if(EnumUtils.isValidEnum(ShowType.class,showTypeName)){ Map mailResult = MailUtils.sendMails(receviersList, - receviersCcList, title, content, ShowType.valueOf(showTypeName)); + receviersCcList, title, content, ShowType.valueOf(showTypeName).getDescp()); if(!(boolean) mailResult.get(STATUS)){ throw new RuntimeException("send mail failed!"); } diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/dolphinscheduler-server/src/main/resources/config/install_config.conf index cba117e048..fab6b32d51 100644 --- a/dolphinscheduler-server/src/main/resources/config/install_config.conf +++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf @@ -27,6 +27,9 @@ dbhost="192.168.xx.xx:3306" # db username username="xx" +# database name +dbname="dolphinscheduler" + # db passwprd # NOTICE: if there are special characters, please use the \ to escape, for example, `[` escape to `\[` password="xx" diff --git a/dolphinscheduler-ui/.eslintrc b/dolphinscheduler-ui/.eslintrc deleted file mode 100644 index e1d1a03e1c..0000000000 --- a/dolphinscheduler-ui/.eslintrc +++ /dev/null @@ -1,6 +0,0 @@ -globals: - $: true - expect: true -rules: - "no-new": "off" - "no-labels": [2, {"allowLoop": true}] diff --git a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/common.properties.j2 b/dolphinscheduler-ui/.eslintrc.yml similarity index 74% rename from ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/common.properties.j2 rename to dolphinscheduler-ui/.eslintrc.yml index 2220c4effa..8d4020f7c4 100644 --- a/ambari_plugin/common-services/DOLPHIN/2.0.0/package/templates/common.properties.j2 +++ b/dolphinscheduler-ui/.eslintrc.yml @@ -15,6 +15,20 @@ # limitations under the License. # -{% for key, value in dolphin_common_map.iteritems() -%} - {{key}}={{value}} -{% endfor %} \ No newline at end of file +env: + browser: true + es6: true + jquery: true +extends: + - 'plugin:vue/essential' + - standard +globals: + Atomics: readonly + SharedArrayBuffer: readonly + PUBLIC_PATH: readonly +parserOptions: + ecmaVersion: 2018 + sourceType: module +plugins: + - vue +rules: {} diff --git a/dolphinscheduler-ui/package.json b/dolphinscheduler-ui/package.json index b23969803b..1f5170637e 100644 --- a/dolphinscheduler-ui/package.json +++ b/dolphinscheduler-ui/package.json @@ -8,6 +8,7 @@ "dev": "cross-env NODE_ENV=development webpack-dev-server --config ./build/webpack.config.dev.js", "clean": "rimraf dist", "start": "npm run dev", + "lint": "eslint ./src --fix", "build:release": "npm run clean && cross-env NODE_ENV=production PUBLIC_PATH=/dolphinscheduler/ui webpack --config ./build/webpack.config.release.js" }, "dependencies": { @@ -50,12 +51,19 @@ "css-loader": "^0.28.8", "cssnano": "4.1.10", "env-parse": "^1.0.5", + "eslint": "^6.8.0", + "eslint-config-standard": "^14.1.1", + "eslint-plugin-import": "^2.20.2", + "eslint-plugin-node": "^11.1.0", + "eslint-plugin-promise": "^4.2.1", + "eslint-plugin-standard": "^4.0.1", + "eslint-plugin-vue": "^6.2.2", "file-loader": "^5.0.2", "globby": "^8.0.1", "html-loader": "^0.5.5", "html-webpack-plugin": "^3.2.0", "mini-css-extract-plugin": "^0.8.2", - "node-sass": "^4.13.1", + "node-sass": "^4.14.0", "postcss-loader": "^3.0.0", "progress-bar-webpack-plugin": "^1.12.1", "rimraf": "^2.6.2", diff --git a/dolphinscheduler-ui/src/js/conf/home/index.js b/dolphinscheduler-ui/src/js/conf/home/index.js index 1913088eca..3c518d8952 100644 --- a/dolphinscheduler-ui/src/js/conf/home/index.js +++ b/dolphinscheduler-ui/src/js/conf/home/index.js @@ -31,15 +31,15 @@ import Permissions from '@/module/permissions' import 'ans-ui/lib/ans-ui.min.css' import ans from 'ans-ui/lib/ans-ui.min' import en_US from 'ans-ui/lib/locale/en' // eslint-disable-line -import'normalize.css/normalize.css' +import 'normalize.css/normalize.css' import 'sass/conf/home/index.scss' -import'bootstrap/dist/css/bootstrap.min.css' +import 'bootstrap/dist/css/bootstrap.min.css' -import'bootstrap/dist/js/bootstrap.min.js' +import 'bootstrap/dist/js/bootstrap.min.js' import 'canvg/dist/browser/canvg.min.js' // Component internationalization -let useOpt = i18n.globalScope.LOCALE === 'en_US' ? { locale: en_US } : {} +const useOpt = i18n.globalScope.LOCALE === 'en_US' ? { locale: en_US } : {} // Vue.use(ans) Vue.use(ans, useOpt) @@ -74,7 +74,7 @@ Permissions.request().then(res => { methods: { initApp () { $('.global-loading').hide() - let bootstrapTooltip = $.fn.tooltip.noConflict() + const bootstrapTooltip = $.fn.tooltip.noConflict() $.fn.tooltip = bootstrapTooltip $('body').tooltip({ selector: '[data-toggle="tooltip"]', diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js index db8acf3073..56d0168893 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js @@ -16,7 +16,6 @@ */ import i18n from '@/module/i18n' -import Permissions from '@/module/permissions' /** * Operation bar config @@ -26,7 +25,7 @@ import Permissions from '@/module/permissions' * @desc tooltip */ const toolOper = (dagThis) => { - let disabled =!!dagThis.$store.state.dag.isDetails// Permissions.getAuth() === false ? false : !dagThis.$store.state.dag.isDetails + const disabled = !!dagThis.$store.state.dag.isDetails// Permissions.getAuth() === false ? false : !dagThis.$store.state.dag.isDetails return [ { code: 'pointer', @@ -67,7 +66,7 @@ const toolOper = (dagThis) => { * @desc tooltip * @code Backend definition identifier */ -let publishStatus = [ +const publishStatus = [ { id: 0, desc: `${i18n.$t('Unpublished')}`, @@ -90,7 +89,7 @@ let publishStatus = [ * @desc tooltip * @code identifier */ -let runningType = [ +const runningType = [ { desc: `${i18n.$t('Start Process')}`, code: 'START_PROCESS' @@ -146,85 +145,85 @@ let runningType = [ * @icoUnicode iconfont * @isSpin is loading (Need to execute the code block to write if judgment) */ -let tasksState = { - 'SUBMITTED_SUCCESS': { +const tasksState = { + SUBMITTED_SUCCESS: { id: 0, desc: `${i18n.$t('Submitted successfully')}`, color: '#A9A9A9', icoUnicode: 'ans-icon-dot-circle', isSpin: false }, - 'RUNNING_EXEUTION': { + RUNNING_EXEUTION: { id: 1, desc: `${i18n.$t('Executing')}`, color: '#0097e0', icoUnicode: 'ans-icon-gear', isSpin: true }, - 'READY_PAUSE': { + READY_PAUSE: { id: 2, desc: `${i18n.$t('Ready to pause')}`, color: '#07b1a3', icoUnicode: 'ans-icon-pause-solid', isSpin: false }, - 'PAUSE': { + PAUSE: { id: 3, desc: `${i18n.$t('Pause')}`, color: '#057c72', icoUnicode: 'ans-icon-pause', isSpin: false }, - 'READY_STOP': { + READY_STOP: { id: 4, desc: `${i18n.$t('Ready to stop')}`, color: '#FE0402', icoUnicode: 'ans-icon-coin', isSpin: false }, - 'STOP': { + STOP: { id: 5, desc: `${i18n.$t('Stop')}`, color: '#e90101', icoUnicode: 'ans-icon-stop', isSpin: false }, - 'FAILURE': { + FAILURE: { id: 6, desc: `${i18n.$t('failed')}`, color: '#000000', icoUnicode: 'ans-icon-fail-empty', isSpin: false }, - 'SUCCESS': { + SUCCESS: { id: 7, desc: `${i18n.$t('success')}`, color: '#33cc00', icoUnicode: 'ans-icon-success-empty', isSpin: false }, - 'NEED_FAULT_TOLERANCE': { + NEED_FAULT_TOLERANCE: { id: 8, desc: `${i18n.$t('Need fault tolerance')}`, color: '#FF8C00', icoUnicode: 'ans-icon-pen', isSpin: false }, - 'KILL': { + KILL: { id: 9, desc: `${i18n.$t('kill')}`, color: '#a70202', icoUnicode: 'ans-icon-minus-circle-empty', isSpin: false }, - 'WAITTING_THREAD': { + WAITTING_THREAD: { id: 10, desc: `${i18n.$t('Waiting for thread')}`, color: '#912eed', icoUnicode: 'ans-icon-sand-clock', isSpin: false }, - 'WAITTING_DEPEND': { + WAITTING_DEPEND: { id: 11, desc: `${i18n.$t('Waiting for dependence')}`, color: '#5101be', @@ -239,62 +238,61 @@ let tasksState = { * @desc tooltip * @color color (tree and gantt) */ -let tasksType = { - 'SHELL': { +const tasksType = { + SHELL: { desc: 'SHELL', color: '#646464' }, - 'SUB_PROCESS': { + SUB_PROCESS: { desc: 'SUB_PROCESS', color: '#0097e0' }, - 'PROCEDURE': { + PROCEDURE: { desc: 'PROCEDURE', color: '#525CCD' }, - 'SQL': { + SQL: { desc: 'SQL', color: '#7A98A1' }, - 'SPARK': { + SPARK: { desc: 'SPARK', color: '#E46F13' }, - 'FLINK': { + FLINK: { desc: 'FLINK', color: '#E46F13' }, - 'MR': { + MR: { desc: 'MapReduce', color: '#A0A5CC' }, - 'PYTHON': { + PYTHON: { desc: 'PYTHON', color: '#FED52D' }, - 'DEPENDENT': { + DEPENDENT: { desc: 'DEPENDENT', color: '#2FBFD8' }, - 'HTTP': { + HTTP: { desc: 'HTTP', color: '#E46F13' }, - 'DATAX': { + DATAX: { desc: 'DataX', color: '#1fc747' }, - 'SQOOP': { + SQOOP: { desc: 'SQOOP', color: '#E46F13' }, - 'CONDITIONS': { + CONDITIONS: { desc: 'CONDITIONS', color: '#E46F13' } } - export { toolOper, publishStatus, diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js index 240f3246aa..3f47076ce3 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js @@ -24,7 +24,7 @@ import store from '@/conf/home/store' /** * Prototype method */ -let Dag = function () { +const Dag = function () { this.dag = {} this.instance = {} } @@ -49,7 +49,7 @@ Dag.prototype.setConfig = function (o) { * create dag */ Dag.prototype.create = function () { - let self = this + const self = this jsPlumb.ready(() => { JSP.init({ dag: this.dag, @@ -98,36 +98,35 @@ Dag.prototype.toolbarEvent = function ({ item, code, is }) { * Echo data display */ Dag.prototype.backfill = function (arg) { - if(arg) { + if (arg) { let locationsValue = store.state.dag.locations - let locationsValue1 = store.state.dag.locations - let locationsValue2 = store.state.dag.locations - let arr = [] - for (let i in locationsValue1) { - let objs = new Object(); + const locationsValue1 = store.state.dag.locations + const locationsValue2 = store.state.dag.locations + const arr = [] + for (const i in locationsValue1) { + const objs = {} objs.id = i - arr.push(Object.assign(objs,locationsValue1[i])); //Attributes + arr.push(Object.assign(objs, locationsValue1[i])) // Attributes } - let tmp = [] - for(let i in locationsValue2) { - if(locationsValue2[i].targetarr !='' && locationsValue2[i].targetarr.split(',').length>1) { + const tmp = [] + for (const i in locationsValue2) { + if (locationsValue2[i].targetarr !== '' && locationsValue2[i].targetarr.split(',').length > 1) { tmp.push(locationsValue2[i]) } } - function copy (array) { - let newArray = [] - for(let item of array) { - newArray.push(item); + const copy = function (array) { + const newArray = [] + for (const item of array) { + newArray.push(item) } - return newArray; + return newArray } - - let newArr = copy(arr) - function getNewArr() { - for(let i= 0; i1) { + const newArr = copy(arr) + const getNewArr = function () { + for (let i = 0; i < newArr.length; i++) { + if (newArr[i].targetarr !== '' && newArr[i].targetarr.split(',').length > 1) { newArr[i].targetarr = newArr[i].targetarr.split(',').shift() } } @@ -141,154 +140,154 @@ Dag.prototype.backfill = function (arg) { * @param {String} idStr id key name * @param {String} childrenStr children key name */ - function fommat({arrayList, pidStr = 'targetarr', idStr = 'id', childrenStr = 'children'}) { - let listOjb = {}; // Used to store objects of the form {key: obj} - let treeList = []; // An array to store the final tree structure data - // Transform the data into {key: obj} format, which is convenient for the following data processing - for (let i = 0; i < arrayList.length; i++) { - listOjb[arrayList[i][idStr]] = arrayList[i] - } - // Format data based on pid - for (let j = 0; j < arrayList.length; j++) { - // Determine if the parent exists - // let haveParent = arrayList[j].targetarr.split(',').length>1?listOjb[arrayList[j].targetarr.split(',')[0]]:listOjb[arrayList[j][pidStr]] - let haveParent = listOjb[arrayList[j][pidStr]] - if (haveParent) { - // If there is no parent children field, create a children field - !haveParent[childrenStr] && (haveParent[childrenStr] = []) - // Insert child in parent - haveParent[childrenStr].push(arrayList[j]) - } else { - // If there is no parent, insert directly into the outermost layer - treeList.push(arrayList[j]) - } - } - return treeList + const fommat = function ({ arrayList, pidStr = 'targetarr', idStr = 'id', childrenStr = 'children' }) { + const listOjb = {} // Used to store objects of the form {key: obj} + const treeList = [] // An array to store the final tree structure data + // Transform the data into {key: obj} format, which is convenient for the following data processing + for (let i = 0; i < arrayList.length; i++) { + listOjb[arrayList[i][idStr]] = arrayList[i] } - let datas = fommat({arrayList: newArr,pidStr: 'targetarr'}) - // Count the number of leaf nodes - function getLeafCountTree(json) { - if(!json.children) { - json.colspan = 1; - return 1; + // Format data based on pid + for (let j = 0; j < arrayList.length; j++) { + // Determine if the parent exists + // let haveParent = arrayList[j].targetarr.split(',').length>1?listOjb[arrayList[j].targetarr.split(',')[0]]:listOjb[arrayList[j][pidStr]] + const haveParent = listOjb[arrayList[j][pidStr]] + if (haveParent) { + // If there is no parent children field, create a children field + !haveParent[childrenStr] && (haveParent[childrenStr] = []) + // Insert child in parent + haveParent[childrenStr].push(arrayList[j]) } else { - let leafCount = 0; - for(let i = 0 ; i < json.children.length ; i++){ - leafCount = leafCount + getLeafCountTree(json.children[i]); - } - json.colspan = leafCount; - return leafCount; + // If there is no parent, insert directly into the outermost layer + treeList.push(arrayList[j]) } } - // Number of tree node levels - let countTree = getLeafCountTree(datas[0]) - function getMaxFloor(treeData) { - let max = 0 - function each (data, floor) { - data.forEach(e => { - e.floor = floor - e.x=floor*170 - if (floor > max) { - max = floor - } - if (e.children) { - each(e.children, floor + 1) - } - }) - } - each(treeData,1) - return max + return treeList + } + const datas = fommat({ arrayList: newArr, pidStr: 'targetarr' }) + // Count the number of leaf nodes + const getLeafCountTree = function (json) { + if (!json.children) { + json.colspan = 1 + return 1 + } else { + let leafCount = 0 + for (let i = 0; i < json.children.length; i++) { + leafCount = leafCount + getLeafCountTree(json.children[i]) } - getMaxFloor(datas) - // The last child of each node - let lastchildren = []; - forxh(datas); - function forxh(list) { - for (let i = 0; i < list.length; i++) { - let chlist = list[i]; - if (chlist.children) { - forxh(chlist.children); - } else { - lastchildren.push(chlist); - } + json.colspan = leafCount + return leafCount + } + } + // Number of tree node levels + const countTree = getLeafCountTree(datas[0]) + const getMaxFloor = function (treeData) { + let max = 0 + function each (data, floor) { + data.forEach(e => { + e.floor = floor + e.x = floor * 170 + if (floor > max) { + max = floor } - } - // Get all parent nodes above the leaf node - function treeFindPath (tree, func, path,n) { - if (!tree) return [] - for (const data of tree) { - path.push(data.name) - if (func(data)) return path - if (data.children) { - const findChildren = treeFindPath(data.children, func, path,n) - if (findChildren.length) return findChildren - } - path.pop() + if (e.children) { + each(e.children, floor + 1) } - return [] - } - function toLine(data){ - return data.reduce((arrData, {id, name, targetarr, x, y, children = []}) => - arrData.concat([{id, name, targetarr, x, y}], toLine(children)), []) + }) + } + each(treeData, 1) + return max + } + getMaxFloor(datas) + // The last child of each node + let lastchildren = [] + const forxh = function (list) { + for (let i = 0; i < list.length; i++) { + const chlist = list[i] + if (chlist.children) { + forxh(chlist.children) + } else { + lastchildren.push(chlist) } - let listarr = toLine(datas); - let listarrs = toLine(datas) - let dataObject = {} - for(let i = 0; i + arrData.concat([{ id, name, targetarr, x, y }], toLine(children)), []) + } + const listarr = toLine(datas) + const listarrs = toLine(datas) + const dataObject = {} + for (let i = 0; i < listarrs.length; i++) { + delete (listarrs[i].id) + } - for(let a = 0; a value2) { - return 1; - } else { - return 0; - } - }; + if (value1 < value2) { + return -1 + } else if (value1 > value2) { + return 1 + } else { + return 0 } + } + } - lastchildren = lastchildren.sort(createComparisonFunction('x')) + lastchildren = lastchildren.sort(createComparisonFunction('x')) - // Coordinate value of each leaf node - for(let a = 0; a data.targetarr===lastchildren[i].targetarr,[],i+1) - for(let j = 0; j data.targetarr === lastchildren[i].targetarr, [], i + 1) + for (let j = 0; j < node.length; j++) { + for (let k = 0; k < listarrs.length; k++) { + if (node[j] === listarrs[k].name) { + listarrs[k].y = (i + 1) * 120 } } - for(let a = 0; a1) { - dataObject[Object.keys(locationsValue1)[0]].y = (countTree/2)*120+50 + } + } + for (let i = 0; i < tmp.length; i++) { + for (const objs in dataObject) { + if (tmp[i].name === dataObject[objs].name) { + dataObject[objs].targetarr = tmp[i].targetarr } + } + } + for (let a = 0; a < lastchildren.length; a++) { + dataObject[lastchildren[a].id].y = (a + 1) * 120 + } + if (countTree > 1) { + dataObject[Object.keys(locationsValue1)[0]].y = (countTree / 2) * 120 + 50 + } locationsValue = dataObject - let self = this + const self = this jsPlumb.ready(() => { JSP.init({ dag: this.dag, @@ -310,7 +309,7 @@ Dag.prototype.backfill = function (arg) { }) }) } else { - let self = this + const self = this jsPlumb.ready(() => { JSP.init({ dag: this.dag, diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue index ae15e20f74..6d24495882 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue @@ -77,7 +77,7 @@ icon="ans-icon-triangle-solid-right" size="xsmall" data-container="body" - v-if="type === 'instance'" + v-if="(type === 'instance' || 'definition') && urlParam.id !=undefined" style="vertical-align: middle;" @click="dagAutomaticLayout"> @@ -155,6 +155,7 @@ isLoading: false, taskId: null, arg: false, + } }, mixins: [disabledState], diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/commcon.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/commcon.js index cdf632f13d..37d471cccb 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/commcon.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/commcon.js @@ -43,7 +43,7 @@ const cycleList = [ * cycle value */ const dateValueList = { - 'hour': [ + hour: [ { value: 'last1Hour', label: `${i18n.$t('Last1Hour')}` @@ -57,7 +57,7 @@ const dateValueList = { label: `${i18n.$t('Last3Hours')}` } ], - 'day': [ + day: [ { value: 'today', label: `${i18n.$t('today')}` @@ -79,7 +79,7 @@ const dateValueList = { label: `${i18n.$t('Last7Days')}` } ], - 'week': [ + week: [ { value: 'thisWeek', label: `${i18n.$t('ThisWeek')}` @@ -117,7 +117,7 @@ const dateValueList = { label: `${i18n.$t('LastSunday')}` } ], - 'month': [ + month: [ { value: 'thisMonth', label: `${i18n.$t('ThisMonth')}` @@ -221,15 +221,15 @@ const sqlTypeList = [ const positionList = [ { id: 'PARAMETER', - code: "Parameter" + code: 'Parameter' }, { id: 'BODY', - code: "Body" + code: 'Body' }, { id: 'HEADERS', - code: "Headers" + code: 'Headers' } ] const nodeStatusList = [ diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/jumpAffirm/index.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/jumpAffirm/index.js index 88a258c6fe..360a0fb356 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/jumpAffirm/index.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/jumpAffirm/index.js @@ -21,7 +21,7 @@ import store from '@/conf/home/store' import router from '@/conf/home/router' import { uuid, findComponentDownward } from '@/module/util/' -let Affirm = {} +const Affirm = {} let $root = {} let $routerType = '' let $isPop = true @@ -59,7 +59,7 @@ Affirm.paramVerification = (name) => { if (!$isPop) { return true } - let dagStore = store.state.dag + const dagStore = store.state.dag let flag = false if ($routerType === 'definition-create') { // No nodes jump out directly @@ -100,7 +100,7 @@ Affirm.isPop = (fn) => { Vue.$modal.destroy() }) }, - close () { + close () { fn() Vue.$modal.destroy() } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/downChart.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/downChart.js index 3dd13895b0..1c851a175f 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/downChart.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/downChart.js @@ -21,7 +21,7 @@ import { tasksAll } from './util' import html2canvas from 'html2canvas' import { findComponentDownward } from '@/module/util/' -let DownChart = function () { +const DownChart = function () { this.dag = {} } @@ -31,10 +31,10 @@ let DownChart = function () { DownChart.prototype.maxVal = function () { return new Promise((resolve, reject) => { // All nodes - let tasksAllList = tasksAll() - let dom = $('.dag-container') - let y = parseInt(_.maxBy(tasksAllList, 'y').y + 60) - let x = parseInt(_.maxBy(tasksAllList, 'x').x + 100) + const tasksAllList = tasksAll() + const dom = $('.dag-container') + const y = parseInt(_.maxBy(tasksAllList, 'y').y + 60) + const x = parseInt(_.maxBy(tasksAllList, 'x').x + 100) resolve({ width: (x > 600 ? x : dom.width()) + 100, @@ -60,9 +60,9 @@ DownChart.prototype.download = function ({ dagThis }) { // svg handle const nodesToRecover = [] const nodesToRemove = [] - let parentNode = node.parentNode - let svg = node.outerHTML.trim() - let canvas = document.createElement('canvas') + const parentNode = node.parentNode + const svg = node.outerHTML.trim() + const canvas = document.createElement('canvas') canvg(canvas, svg) if (node.style.position) { canvas.style.position += node.style.position @@ -102,10 +102,10 @@ DownChart.prototype.download = function ({ dagThis }) { heigth: height, useCORS: true // Enable cross-domain configuration }).then((canvas) => { - let name = `${this.dag.name}.png` - let url = canvas.toDataURL('image/png', 1) + const name = `${this.dag.name}.png` + const url = canvas.toDataURL('image/png', 1) setTimeout(() => { - let triggerDownload = $('').attr('href', url).attr('download', name).appendTo('body') + const triggerDownload = $('').attr('href', url).attr('download', name).appendTo('body') triggerDownload[0].click() triggerDownload.remove() }, 100) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/dragZoom.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/dragZoom.js index b084e2e9c8..2027f8e1ea 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/dragZoom.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/dragZoom.js @@ -23,7 +23,7 @@ const DragZoom = function () { } DragZoom.prototype.init = function () { - let $canvas = $('#canvas') + const $canvas = $('#canvas') this.element = d3.select('#canvas') this.zoom = d3.behavior.zoom() .scaleExtent([0.5, 2]) @@ -35,5 +35,4 @@ DragZoom.prototype.init = function () { this.element.call(this.zoom).on('dblclick.zoom', null) } - export default new DragZoom() diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js index c77127d49a..b0bc0fe7c1 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js @@ -24,7 +24,6 @@ import { jsPlumb } from 'jsplumb' import DragZoom from './dragZoom' import store from '@/conf/home/store' import router from '@/conf/home/router' -import Permissions from '@/module/permissions' import { uuid, findComponentDownward } from '@/module/util/' import { @@ -32,10 +31,11 @@ import { rtTasksTpl, setSvgColor, saveTargetarr, - rtTargetarrArr } from './util' + rtTargetarrArr +} from './util' import mStart from '@/conf/home/pages/projects/pages/definition/pages/list/_source/start' -let JSP = function () { +const JSP = function () { this.dag = {} this.selectedElement = {} @@ -75,7 +75,7 @@ JSP.prototype.init = function ({ dag, instance, options }) { this.setConfig({ isDrag: !store.state.dag.isDetails, isAttachment: false, - isNewNodes: !store.state.dag.isDetails,//Permissions.getAuth() === false ? false : !store.state.dag.isDetails, + isNewNodes: !store.state.dag.isDetails, // Permissions.getAuth() === false ? false : !store.state.dag.isDetails, isDblclick: true, isContextmenu: true, isClick: false @@ -105,7 +105,7 @@ JSP.prototype.setConfig = function (o) { * Node binding event */ JSP.prototype.tasksEvent = function (selfId) { - let tasks = $(`#${selfId}`) + const tasks = $(`#${selfId}`) // Bind right event tasks.on('contextmenu', e => { this.tasksContextmenu(e) @@ -129,7 +129,7 @@ JSP.prototype.tasksEvent = function (selfId) { JSP.prototype.draggable = function () { if (this.config.isNewNodes) { let selfId - let self = this + const self = this $('.toolbar-btn .roundedRect').draggable({ scope: 'plant', helper: 'clone', @@ -146,7 +146,7 @@ JSP.prototype.draggable = function () { drop: function (ev, ui) { let id = 'tasks-' + Math.ceil(Math.random() * 100000) // eslint-disable-line // Get mouse coordinates - let left = parseInt(ui.offset.left - $(this).offset().left) + const left = parseInt(ui.offset.left - $(this).offset().left) let top = parseInt(ui.offset.top - $(this).offset().top) - 10 if (top < 25) { top = 25 @@ -162,7 +162,7 @@ JSP.prototype.draggable = function () { })) // Get the generated node - let thisDom = jsPlumb.getSelector('.statemachine-demo .w') + const thisDom = jsPlumb.getSelector('.statemachine-demo .w') // Generating a connection node self.JspInstance.batch(() => { @@ -193,13 +193,13 @@ JSP.prototype.jsonHandle = function ({ largeJson, locations }) { $('#canvas').append(rtTasksTpl({ id: v.id, name: v.name, - x: locations[v.id]['x'], - y: locations[v.id]['y'], - targetarr: locations[v.id]['targetarr'], + x: locations[v.id].x, + y: locations[v.id].y, + targetarr: locations[v.id].targetarr, isAttachment: this.config.isAttachment, taskType: v.type, runFlag: v.runFlag, - nodenumber: locations[v.id]['nodenumber'], + nodenumber: locations[v.id].nodenumber })) // contextmenu event @@ -263,29 +263,29 @@ JSP.prototype.initNode = function (el) { */ JSP.prototype.tasksContextmenu = function (event) { if (this.config.isContextmenu) { - let routerName = router.history.current.name + const routerName = router.history.current.name // state - let isOne = routerName === 'projects-definition-details' && this.dag.releaseState !== 'NOT_RELEASE' + const isOne = routerName === 'projects-definition-details' && this.dag.releaseState !== 'NOT_RELEASE' // hide - let isTwo = store.state.dag.isDetails + const isTwo = store.state.dag.isDetails - let html = [ + const html = [ `${i18n.$t('Start')}`, `${i18n.$t('Edit')}`, `${i18n.$t('Copy')}`, `${i18n.$t('Delete')}` ] - let operationHtml = () => { + const operationHtml = () => { return html.splice(',') } - let e = event - let $id = e.currentTarget.id - let $contextmenu = $('#contextmenu') - let $name = $(`#${$id}`).find('.name-p').text() - let $left = e.pageX + document.body.scrollLeft - 5 - let $top = e.pageY + document.body.scrollTop - 5 + const e = event + const $id = e.currentTarget.id + const $contextmenu = $('#contextmenu') + const $name = $(`#${$id}`).find('.name-p').text() + const $left = e.pageX + document.body.scrollLeft - 5 + const $top = e.pageY + document.body.scrollTop - 5 $contextmenu.css({ left: $left, top: $top, @@ -297,10 +297,10 @@ JSP.prototype.tasksContextmenu = function (event) { if (isOne) { // start run $('#startRunning').on('click', () => { - let name = store.state.dag.name - let id = router.history.current.params.id + const name = store.state.dag.name + const id = router.history.current.params.id store.dispatch('dag/getStartCheck', { processDefinitionId: id }).then(res => { - let modal = Vue.$modal.dialog({ + const modal = Vue.$modal.dialog({ closable: false, showMask: true, escClose: true, @@ -332,9 +332,9 @@ JSP.prototype.tasksContextmenu = function (event) { }) }) } - if (!isTwo) { + if (!isTwo) { // edit node - $(`#editNodes`).click(ev => { + $('#editNodes').click(ev => { findComponentDownward(this.dag.$root, 'dag-chart')._createNodes({ id: $id, type: $(`#${$id}`).attr('data-tasks-type') @@ -359,7 +359,7 @@ JSP.prototype.tasksContextmenu = function (event) { JSP.prototype.tasksDblclick = function (e) { // Untie event if (this.config.isDblclick) { - let id = $(e.currentTarget.offsetParent).attr('id') + const id = $(e.currentTarget.offsetParent).attr('id') findComponentDownward(this.dag.$root, 'dag-chart')._createNodes({ id: id, @@ -373,10 +373,10 @@ JSP.prototype.tasksDblclick = function (e) { */ JSP.prototype.tasksClick = function (e) { let $id - let self = this - let $body = $(`body`) + const self = this + const $body = $('body') if (this.config.isClick) { - let $connect = this.selectedElement.connect + const $connect = this.selectedElement.connect $('.w').removeClass('jtk-tasks-active') $(e.currentTarget).addClass('jtk-tasks-active') if ($connect) { @@ -406,7 +406,7 @@ JSP.prototype.tasksClick = function (e) { * paste */ JSP.prototype.removePaste = function () { - let $body = $(`body`) + const $body = $('body') // Unbind copy and paste events $body.unbind('copy').unbind('paste') // Remove selected node parameters @@ -421,7 +421,7 @@ JSP.prototype.removePaste = function () { JSP.prototype.connectClick = function (e) { // Set svg color setSvgColor(e, '#0097e0') - let $id = this.selectedElement.id + const $id = this.selectedElement.id if ($id) { $(`#${$id}`).removeClass('jtk-tasks-active') this.selectedElement.id = null @@ -434,24 +434,10 @@ JSP.prototype.connectClick = function (e) { * @param {Pointer} */ JSP.prototype.handleEventPointer = function (is) { - let wDom = $('.w') this.setConfig({ isClick: is, isAttachment: false }) - // wDom.removeClass('jtk-ep') - // if (!is) { - // wDom.removeClass('jtk-tasks-active') - // this.selectedElement = {} - // _.map($('#canvas svg'), v => { - // if ($(v).attr('class')) { - // _.map($(v).find('path'), v1 => { - // $(v1).attr('fill', '#555') - // $(v1).attr('stroke', '#555') - // }) - // } - // }) - // } } /** @@ -459,7 +445,7 @@ JSP.prototype.handleEventPointer = function (is) { * @param {Line} */ JSP.prototype.handleEventLine = function (is) { - let wDom = $('.w') + const wDom = $('.w') this.setConfig({ isAttachment: is }) @@ -471,8 +457,8 @@ JSP.prototype.handleEventLine = function (is) { * @param {Remove} */ JSP.prototype.handleEventRemove = function () { - let $id = this.selectedElement.id || null - let $connect = this.selectedElement.connect || null + const $id = this.selectedElement.id || null + const $connect = this.selectedElement.connect || null if ($id) { this.removeNodes(this.selectedElement.id) } else { @@ -489,9 +475,9 @@ JSP.prototype.handleEventRemove = function () { JSP.prototype.removeNodes = function ($id) { // Delete node processing(data-targetarr) _.map(tasksAll(), v => { - let targetarr = v.targetarr.split(',') + const targetarr = v.targetarr.split(',') if (targetarr.length) { - let newArr = _.filter(targetarr, v1 => v1 !== $id) + const newArr = _.filter(targetarr, v1 => v1 !== $id) $(`#${v.id}`).attr('data-targetarr', newArr.toString()) } }) @@ -502,7 +488,7 @@ JSP.prototype.removeNodes = function ($id) { $(`#${$id}`).remove() // callback onRemoveNodes event - this.options&&this.options.onRemoveNodes&&this.options.onRemoveNodes($id) + this.options && this.options.onRemoveNodes && this.options.onRemoveNodes($id) } /** @@ -513,15 +499,15 @@ JSP.prototype.removeConnect = function ($connect) { return } // Remove connections and remove node and node dependencies - let targetId = $connect.targetId - let sourceId = $connect.sourceId + const targetId = $connect.targetId + const sourceId = $connect.sourceId let targetarr = rtTargetarrArr(targetId) if (targetarr.length) { targetarr = _.filter(targetarr, v => v !== sourceId) $(`#${targetId}`).attr('data-targetarr', targetarr.toString()) } - if ($(`#${sourceId}`).attr('data-tasks-type')=='CONDITIONS') { - $(`#${sourceId}`).attr('data-nodenumber',Number($(`#${sourceId}`).attr('data-nodenumber'))-1) + if ($(`#${sourceId}`).attr('data-tasks-type') === 'CONDITIONS') { + $(`#${sourceId}`).attr('data-nodenumber', Number($(`#${sourceId}`).attr('data-nodenumber')) - 1) } this.JspInstance.deleteConnection($connect) @@ -533,24 +519,24 @@ JSP.prototype.removeConnect = function ($connect) { */ JSP.prototype.copyNodes = function ($id) { let newNodeInfo = _.cloneDeep(_.find(store.state.dag.tasks, v => v.id === $id)) - let newNodePors = store.state.dag.locations[$id] + const newNodePors = store.state.dag.locations[$id] // Unstored nodes do not allow replication if (!newNodePors) { return } // Generate random id - let newUuId = `${uuid() + uuid()}` - let id = newNodeInfo.id.length > 8 ? newNodeInfo.id.substr(0, 7) : newNodeInfo.id - let name = newNodeInfo.name.length > 8 ? newNodeInfo.name.substr(0, 7) : newNodeInfo.name + const newUuId = `${uuid() + uuid()}` + const id = newNodeInfo.id.length > 8 ? newNodeInfo.id.substr(0, 7) : newNodeInfo.id + const name = newNodeInfo.name.length > 8 ? newNodeInfo.name.substr(0, 7) : newNodeInfo.name // new id - let newId = `${id || ''}-${newUuId}` + const newId = `${id || ''}-${newUuId}` // new name - let newName = `${name || ''}-${newUuId}` + const newName = `${name || ''}-${newUuId}` // coordinate x - let newX = newNodePors.x + 100 + const newX = newNodePors.x + 100 // coordinate y - let newY = newNodePors.y + 40 + const newY = newNodePors.y + 40 // Generate template node $('#canvas').append(rtTasksTpl({ @@ -563,7 +549,7 @@ JSP.prototype.copyNodes = function ($id) { })) // Get the generated node - let thisDom = jsPlumb.getSelector('.statemachine-demo .w') + const thisDom = jsPlumb.getSelector('.statemachine-demo .w') // Copy node information newNodeInfo = Object.assign(newNodeInfo, { @@ -604,7 +590,7 @@ JSP.prototype.handleEventScreen = function ({ item, is }) { item.icon = 'ans-icon-max' screenOpen = false } - let $mainLayoutModel = $('.main-layout-model') + const $mainLayoutModel = $('.main-layout-model') if (screenOpen) { $mainLayoutModel.addClass('dag-screen') } else { @@ -619,21 +605,21 @@ JSP.prototype.handleEventScreen = function ({ item, is }) { */ JSP.prototype.saveStore = function () { return new Promise((resolve, reject) => { - let connects = [] - let locations = {} - let tasks = [] + const connects = [] + const locations = {} + const tasks = [] - let is = (id) => { + const is = (id) => { return !!_.filter(tasksAll(), v => v.id === id).length } // task _.map(_.cloneDeep(store.state.dag.tasks), v => { if (is(v.id)) { - let preTasks = [] - let id = $(`#${v.id}`) - let tar = id.attr('data-targetarr') - let idDep = tar ? id.attr('data-targetarr').split(',') : [] + const preTasks = [] + const id = $(`#${v.id}`) + const tar = id.attr('data-targetarr') + const idDep = tar ? id.attr('data-targetarr').split(',') : [] if (idDep.length) { _.map(idDep, v1 => { preTasks.push($(`#${v1}`).find('.name-p').text()) @@ -655,12 +641,11 @@ JSP.prototype.saveStore = function () { _.map(this.JspInstance.getConnections(), v => { connects.push({ - 'endPointSourceId': v.sourceId, - 'endPointTargetId': v.targetId + endPointSourceId: v.sourceId, + endPointTargetId: v.targetId }) }) - _.map(tasksAll(), v => { locations[v.id] = { name: v.name, @@ -671,7 +656,6 @@ JSP.prototype.saveStore = function () { } }) - // Storage node store.commit('dag/setTasks', tasks) // Store coordinate information @@ -692,14 +676,14 @@ JSP.prototype.saveStore = function () { JSP.prototype.handleEvent = function () { this.JspInstance.bind('beforeDrop', function (info) { - let sourceId = info['sourceId']// 出 - let targetId = info['targetId']// 入 + const sourceId = info.sourceId// 出 + const targetId = info.targetId// 入 /** * Recursive search for nodes */ let recursiveVal const recursiveTargetarr = (arr, targetId) => { - for (let i in arr) { + for (const i in arr) { if (arr[i] === targetId) { recursiveVal = targetId } else { @@ -719,10 +703,10 @@ JSP.prototype.handleEvent = function () { return false } - if ($(`#${sourceId}`).attr('data-tasks-type')=='CONDITIONS' && $(`#${sourceId}`).attr('data-nodenumber')==2) { + if ($(`#${sourceId}`).attr('data-tasks-type') === 'CONDITIONS' && $(`#${sourceId}`).attr('data-nodenumber') === 2) { return false } else { - $(`#${sourceId}`).attr('data-nodenumber',Number($(`#${sourceId}`).attr('data-nodenumber'))+1) + $(`#${sourceId}`).attr('data-nodenumber', Number($(`#${sourceId}`).attr('data-nodenumber')) + 1) } // Storage node dependency information @@ -744,7 +728,7 @@ JSP.prototype.jspBackfill = function ({ connects, locations, largeJson }) { locations: locations }) - let wNodes = jsPlumb.getSelector('.statemachine-demo .w') + const wNodes = jsPlumb.getSelector('.statemachine-demo .w') // Backfill line this.JspInstance.batch(() => { diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js index 17e7faf477..e44363911b 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js @@ -23,7 +23,7 @@ import store from '@/conf/home/store' * Node, to array */ const rtTargetarrArr = (id) => { - let ids = $(`#${id}`).attr('data-targetarr') + const ids = $(`#${id}`).attr('data-targetarr') return ids ? ids.split(',') : [] } @@ -31,8 +31,8 @@ const rtTargetarrArr = (id) => { * Store node id to targetarr */ const saveTargetarr = (valId, domId) => { - let $target = $(`#${domId}`) - let targetStr = $target.attr('data-targetarr') ? $target.attr('data-targetarr') + `,${valId}` : `${valId}` + const $target = $(`#${domId}`) + const targetStr = $target.attr('data-targetarr') ? $target.attr('data-targetarr') + `,${valId}` : `${valId}` $target.attr('data-targetarr', targetStr) } @@ -44,20 +44,20 @@ const rtBantpl = () => { * return node html */ const rtTasksTpl = ({ id, name, x, y, targetarr, isAttachment, taskType, runFlag, nodenumber }) => { - let tpl = `` + let tpl = '' tpl += `
` - tpl += `
` - tpl += `
` + tpl += '
' + tpl += '
' tpl += `
` tpl += `${name}` - tpl += `
` - tpl += `
` - tpl += `
` + tpl += '
' + tpl += '
' + tpl += '
' if (runFlag === 'FORBIDDEN') { tpl += rtBantpl() } - tpl += `
` - tpl += `
` + tpl += '
' + tpl += '' return tpl } @@ -66,9 +66,9 @@ const rtTasksTpl = ({ id, name, x, y, targetarr, isAttachment, taskType, runFlag * Get all tasks nodes */ const tasksAll = () => { - let a = [] + const a = [] $('#canvas .w').each(function (idx, elem) { - let e = $(elem) + const e = $(elem) a.push({ id: e.attr('id'), name: e.find('.name-p').text(), @@ -117,10 +117,10 @@ const setSvgColor = (e, color) => { * Get all node ids */ const allNodesId = () => { - let idArr = [] + const idArr = [] $('.w').each((i, o) => { - let $obj = $(o) - let $span = $obj.find('.name-p').text() + const $obj = $(o) + const $span = $obj.find('.name-p').text() if ($span) { idArr.push({ id: $obj.attr('id'), diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/monitor/pages/servers/_source/gaugeOption.js b/dolphinscheduler-ui/src/js/conf/home/pages/monitor/pages/servers/_source/gaugeOption.js index 6d64db5e02..a61f80f530 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/monitor/pages/servers/_source/gaugeOption.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/monitor/pages/servers/_source/gaugeOption.js @@ -30,21 +30,21 @@ export default function (value) { color: [ [ 0.5, new echarts.graphic.LinearGradient(0, 0, 1, 0, [{ // eslint-disable-line offset: 1, - color: '#1CAD52'// 50% Color in place + color: '#1CAD52'// 50% Color in place }, { offset: 0.8, - color: '#1CAD52'// 40% Color in place - }], false) ], // 100% Color in place + color: '#1CAD52'// 40% Color in place + }], false)], // 100% Color in place [ 0.7, new echarts.graphic.LinearGradient(0, 0, 1, 0, [{ // eslint-disable-line offset: 1, - color: '#FFC539'// 70% Color in place + color: '#FFC539'// 70% Color in place }, { offset: 0.8, - color: '#FEEC49'// 66% Color in place + color: '#FEEC49'// 66% Color in place }, { offset: 0, - color: '#C7DD6B'// 50% Color in place - }], false) ], + color: '#C7DD6B'// 50% Color in place + }], false)], [ 0.9, new echarts.graphic.LinearGradient(0, 0, 0, 1, [{ // eslint-disable-line offset: 1, color: '#E75F25' // 90% Color in place @@ -54,14 +54,14 @@ export default function (value) { }, { offset: 0, color: '#FFC539' // 70% Color in place - }], false) ], + }], false)], [1, new echarts.graphic.LinearGradient(0, 0, 0, 1, [ { // eslint-disable-line offset: 0.2, color: '#E75F25' // 92% Color in place }, { offset: 0, color: '#D9452C' // 90% Color in place - }], false) ] + }], false)] ], width: 10 } @@ -92,10 +92,10 @@ export default function (value) { show: true, lineStyle: { // Property linestyle controls line style color: [ // Dial Color - [ 0.5, '#20AE51' ], // 0-50%Color in place - [ 0.7, '#FFED44' ], // 51%-70%Color in place - [ 0.9, '#FF9618' ], // 70%-90%Color in place - [ 1, '#DA462C']// 90%-100%Color in place + [0.5, '#20AE51'], // 0-50%Color in place + [0.7, '#FFED44'], // 51%-70%Color in place + [0.9, '#FF9618'], // 70%-90%Color in place + [1, '#DA462C']// 90%-100%Color in place ], width: 30// Dial width } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/_source/instanceConditions/common.js b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/_source/instanceConditions/common.js index 091b8178f9..254098a65b 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/_source/instanceConditions/common.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/_source/instanceConditions/common.js @@ -20,7 +20,7 @@ import i18n from '@/module/i18n' /** * State code table */ -let stateType = [ +const stateType = [ { code: '', label: `${i18n.$t('none')}` diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue index 310725dd51..1a07fae037 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue @@ -19,34 +19,36 @@
- - - - - - - - - - @@ -55,9 +57,9 @@ -
+ + {{$t('#')}} + {{$t('Process Name')}} + {{$t('State')}} + {{$t('Create Time')}} + {{$t('Update Time')}} + {{$t('Description')}} + {{$t('Modify User')}} - {{$t('Timing state')}} + +
+ {{$t('Timing state')}} +
+ {{$t('Operation')}}
{{parseInt(pageNo === 1 ? ($index + 1) : (($index + 1) + (pageSize * (pageNo - 1))))}} + - + {{item.name}} @@ -90,7 +92,7 @@ - + @@ -135,6 +137,10 @@ + + + \ No newline at end of file diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/tree/_source/tree.js b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/tree/_source/tree.js index 5ff02e869f..af2d26831b 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/tree/_source/tree.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/tree/_source/tree.js @@ -21,7 +21,7 @@ import { tasksType, tasksState } from '@/conf/home/pages/dag/_source/config' let self = this -let Tree = function () { +const Tree = function () { self = this this.selfTree = {} this.tree = function () {} @@ -57,7 +57,7 @@ Tree.prototype.init = function ({ data, limit, selfTree }) { this.duration = 400 this.i = 0 this.tree = d3.layout.tree().nodeSize([0, 46]) - let tasks = this.tree.nodes(data) + const tasks = this.tree.nodes(data) this.diagonal = d3.svg .diagonal() @@ -142,9 +142,9 @@ Tree.prototype.treeToggles = function (clicked_d) { // eslint-disable-line */ Tree.prototype.treeUpdate = function (source) { return new Promise((resolve, reject) => { - let tasks = this.tree.nodes(this.root) - let height = Math.max(500, tasks.length * this.config.barHeight + this.config.margin.top + this.config.margin.bottom) - let width = (this.config.nodesMax * 70) + (this.squareNum * (this.config.squareSize + this.config.squarePading)) + this.config.margin.left + this.config.margin.right + 50 + const tasks = this.tree.nodes(this.root) + const height = Math.max(500, tasks.length * this.config.barHeight + this.config.margin.top + this.config.margin.bottom) + const width = (this.config.nodesMax * 70) + (this.squareNum * (this.config.squareSize + this.config.squarePading)) + this.config.margin.left + this.config.margin.right + 50 d3.select('svg') .transition() @@ -156,12 +156,12 @@ Tree.prototype.treeUpdate = function (source) { n.x = i * this.config.barHeight }) - let task = this.svg.selectAll('g.node') + const task = this.svg.selectAll('g.node') .data(tasks, (d) => { return d.id || (d.id = ++this.i) }) - let nodeEnter = task.enter() + const nodeEnter = task.enter() .append('g') .attr('class', this.nodesClass) .attr('transform', () => 'translate(' + source.y0 + ',' + source.x0 + ')') @@ -201,7 +201,7 @@ Tree.prototype.treeUpdate = function (source) { } }) .attr('class', 'state') - .style('fill', d => d.state && tasksState[d.state].color || '#ffffff') + .style('fill', d => (d.state && tasksState[d.state].color) || '#ffffff') .attr('data-toggle', 'tooltip') .attr('rx', d => d.type ? 0 : 12) .attr('ry', d => d.type ? 0 : 12) @@ -231,7 +231,6 @@ Tree.prototype.treeUpdate = function (source) { .attr('transform', d => 'translate(' + d.y + ',' + d.x + ')') .style('opacity', 1) - // Convert the exit node to the new location of the parent node。 task.exit().transition() .duration(this.duration) @@ -240,14 +239,14 @@ Tree.prototype.treeUpdate = function (source) { .remove() // Update link - let link = this.svg.selectAll('path.link') + const link = this.svg.selectAll('path.link') .data(this.tree.links(tasks), d => d.target.id) // Enter any new links in the previous location of the parent node。 link.enter().insert('path', 'g') .attr('class', 'link') .attr('d', (d) => { - let o = { x: source.x0, y: source.y0 } + const o = { x: source.x0, y: source.y0 } return this.diagonal({ source: o, target: o }) }) .transition() @@ -263,7 +262,7 @@ Tree.prototype.treeUpdate = function (source) { link.exit().transition() .duration(this.duration) .attr('d', (d) => { - let o = { x: source.x, y: source.y } + const o = { x: source.x, y: source.y } return this.diagonal({ source: o, target: o }) }) .remove() diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/tree/_source/util.js b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/tree/_source/util.js index eb37e82a6c..2df1783593 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/tree/_source/util.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/tree/_source/util.js @@ -22,7 +22,7 @@ import { tasksState } from '@/conf/home/pages/dag/_source/config' * Node prompt dom */ const rtInstancesTooltip = (data) => { - let str = `
` + let str = '
' str += `id : ${data.id}
` str += `host : ${data.host}
` str += `name : ${data.name}
` @@ -33,7 +33,7 @@ const rtInstancesTooltip = (data) => { str += `startTime : ${data.startTime ? formatDate(data.startTime) : '-'}
` str += `endTime : ${data.endTime ? formatDate(data.endTime) : '-'}
` str += `duration : ${data.duration}
` - str += `
` + str += '
' return str } @@ -42,7 +42,7 @@ const rtInstancesTooltip = (data) => { * Easy to calculate the width dynamically */ const rtCountMethod = list => { - let arr = [] + const arr = [] function count (list, t) { let toggle = false list.forEach(v => { diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/chartConfig.js b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/chartConfig.js index b8601a69c9..9fb18eaa91 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/chartConfig.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/chartConfig.js @@ -18,7 +18,7 @@ import _ from 'lodash' import { tasksState } from '@/conf/home/pages/dag/_source/config' -let pie = { +const pie = { series: [ { type: 'pie', @@ -37,7 +37,7 @@ let pie = { ] } -let bar = { +const bar = { title: { text: '' }, @@ -56,7 +56,7 @@ let bar = { }, tooltip: { formatter (v) { - let val = v[0].name.split(',') + const val = v[0].name.split(',') return `${val[0]} (${v[0].value})` } }, @@ -66,7 +66,7 @@ let bar = { }] } -let simple = { +const simple = { xAxis: { splitLine: { show: false @@ -93,7 +93,6 @@ let simple = { }, color: ['#D5050B', '#0398E1'] - } export { pie, bar, simple } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/gantt/_source/gantt.js b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/gantt/_source/gantt.js index e808c94627..5a4366b06f 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/gantt/_source/gantt.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/gantt/_source/gantt.js @@ -19,13 +19,13 @@ import * as d3 from 'd3' import { formatDate } from '@/module/filter/filter' import { tasksState } from '@/conf/home/pages/dag/_source/config' -let Gantt = function () { +const Gantt = function () { this.el = '' this.tasks = [] this.width = null this.height = null this.taskNames = [] - this.tickFormat = `%H:%M:%S` + this.tickFormat = '%H:%M:%S' this.margin = { top: 10, right: 40, @@ -41,20 +41,20 @@ Gantt.prototype.init = function ({ el, tasks }) { this.tasks = tasks this.taskNames = _.map(_.cloneDeep(tasks), v => v.taskName) this.taskNames = this.taskNames.reduce(function (prev, cur) { - prev.indexOf(cur) === -1 && prev.push(cur); - return prev; - },[]) + prev.indexOf(cur) === -1 && prev.push(cur) + return prev + }, []) this.height = parseInt(this.taskNames.length * 30) this.width = $(this.el).width() - this.margin.right - this.margin.left - 5 this.x = d3.time.scale() - .domain([ this.startTimeXAxis, this.endTimeXAxis ]) - .range([ 0, this.width ]) + .domain([this.startTimeXAxis, this.endTimeXAxis]) + .range([0, this.width]) .clamp(true) this.y = d3.scale.ordinal() .domain(this.taskNames) - .rangeRoundBands([ 0, this.height - this.margin.top - this.margin.bottom ], 0.1) + .rangeRoundBands([0, this.height - this.margin.top - this.margin.bottom], 0.1) this.xAxis = d3.svg.axis() .scale(this.x) @@ -97,13 +97,13 @@ Gantt.prototype.compXAxisTimes = function () { */ Gantt.prototype.initializeXAxis = function () { this.x = d3.time.scale() - .domain([ this.startTimeXAxis, this.endTimeXAxis ]) - .range([ 0, this.width ]) + .domain([this.startTimeXAxis, this.endTimeXAxis]) + .range([0, this.width]) .clamp(true) this.y = d3.scale.ordinal() .domain(this.taskNames) - .rangeRoundBands([ 0, this.height - this.margin.top - this.margin.bottom ], 0.1) + .rangeRoundBands([0, this.height - this.margin.top - this.margin.bottom], 0.1) this.xAxis = d3.svg.axis() .scale(this.x) @@ -151,9 +151,9 @@ Gantt.prototype.drawChart = function () { .attr('transform', 'translate(0, ' + (this.height - this.margin.top - this.margin.bottom) + ')') .transition() .call(this.xAxis) - .selectAll("text") - .attr("transform", `rotate(-${this.width / ($('.tick').length - 1) > 50 ? 0 : Math.acos(this.width / ($('.tick').length - 1) / 50) * 57 })`) - .style("text-anchor", `${this.width / ($('.tick').length - 1) > 50 ? 'middle' : 'end'}`) + .selectAll('text') + .attr('transform', `rotate(-${this.width / ($('.tick').length - 1) > 50 ? 0 : Math.acos(this.width / ($('.tick').length - 1) / 50) * 57})`) + .style('text-anchor', `${this.width / ($('.tick').length - 1) > 50 ? 'middle' : 'end'}`) svg.append('g') .attr('class', 'y axis') @@ -169,15 +169,14 @@ Gantt.prototype.drawChart = function () { * Tip prompt */ Gantt.prototype.tip = function (d) { - let str = `
` + let str = '
' str += `taskName : ${d.taskName}
` str += `status : ${tasksState[d.status].desc} (${d.status})
` str += `startTime : ${formatDate(d.isoStart)}
` str += `endTime : ${formatDate(d.isoEnd)}
` str += `duration : ${d.duration}
` - str += `
` + str += '
' return str } - export default new Gantt() diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue index af0036ab12..e0a13a65ba 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue @@ -19,48 +19,48 @@
- - - - - - - - - - - - - + - @@ -69,12 +69,11 @@ - + - -
+ + {{$t('#')}} + {{$t('Process Name')}} - {{$t('Executor')}} + + {{$t('State')}} + {{$t('Run Type')}} + {{$t('Scheduling Time')}} + {{$t('Start Time')}} + {{$t('End Time')}} + {{$t('Duration')}}s + {{$t('Run Times')}} - {{$t('host')}} - + {{$t('fault-tolerant sign')}} -
- {{$t('State')}} +
+ {{$t('host')}} + +
+ {{$t('Executor')}}
+ {{$t('Operation')}}
{{parseInt(pageNo === 1 ? ($index + 1) : (($index + 1) + (pageSize * (pageNo - 1))))}} + {{item.name}} - {{item.executorName}} - - + {{_rtRunningType(item.commandType)}} @@ -91,14 +90,14 @@ {{item.duration || '-'}} {{item.runTimes}}{{item.recovery}} {{item.host}} - {{item.recovery}} - + {{item.executorName}} + -
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/index.vue index 4a4206218a..7ed4126dc0 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/index.vue @@ -16,24 +16,24 @@ */