wuxiaofei 4 years ago
parent
commit
91e563daeb
  1. 1
      .github/ISSUE_TEMPLATE/bug_report.md
  2. 1
      .github/ISSUE_TEMPLATE/feature_request.md
  3. 1
      .github/ISSUE_TEMPLATE/improvement_suggestion.md
  4. 1
      .github/ISSUE_TEMPLATE/question.md
  5. 2
      .github/workflows/ci_backend.yml
  6. 2
      .github/workflows/ci_e2e.yml
  7. 2
      .github/workflows/ci_ut.yml
  8. 4
      .gitignore
  9. 54
      README.md
  10. 109
      ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-worker.xml
  11. 57
      docker/build/Dockerfile
  12. 36
      docker/build/README.md
  13. 35
      docker/build/README_zh_CN.md
  14. 0
      docker/build/checkpoint.sh
  15. 12
      docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh
  16. 9
      docker/build/conf/dolphinscheduler/logback/logback-alert.xml
  17. 9
      docker/build/conf/dolphinscheduler/logback/logback-api.xml
  18. 9
      docker/build/conf/dolphinscheduler/logback/logback-master.xml
  19. 9
      docker/build/conf/dolphinscheduler/logback/logback-worker.xml
  20. 51
      docker/build/conf/nginx/dolphinscheduler.conf
  21. 47
      docker/build/conf/zookeeper/zoo.cfg
  22. 9
      docker/build/hooks/build
  23. 0
      docker/build/hooks/push
  24. 18
      docker/build/startup-init-conf.sh
  25. 55
      docker/build/startup.sh
  26. 2
      docker/docker-swarm/check
  27. 80
      docker/docker-swarm/docker-compose.yml
  28. 84
      docker/docker-swarm/docker-stack.yml
  29. 12
      docker/docker-swarm/dolphinscheduler_env.sh
  30. 10
      docker/kubernetes/dolphinscheduler/Chart.yaml
  31. 37
      docker/kubernetes/dolphinscheduler/README.md
  32. 25
      docker/kubernetes/dolphinscheduler/requirements.yaml
  33. 25
      docker/kubernetes/dolphinscheduler/requirements.yaml~HEAD
  34. 25
      docker/kubernetes/dolphinscheduler/requirements.yaml~HEAD_0
  35. 25
      docker/kubernetes/dolphinscheduler/requirements.yaml~dev
  36. 25
      docker/kubernetes/dolphinscheduler/requirements.yaml~dev_0
  37. 6
      docker/kubernetes/dolphinscheduler/templates/NOTES.txt
  38. 4
      docker/kubernetes/dolphinscheduler/templates/_helpers.tpl
  39. 1
      docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml
  40. 21
      docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml
  41. 8
      docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml
  42. 2
      docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml
  43. 7
      docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml
  44. 68
      docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml
  45. 48
      docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
  46. 119
      docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml
  47. 2
      docker/kubernetes/dolphinscheduler/templates/ingress.yaml
  48. 2
      docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml
  49. 2
      docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml
  50. 20
      docker/kubernetes/dolphinscheduler/templates/secret-external-fs-s3a.yaml
  51. 96
      docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
  52. 184
      docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
  53. 4
      docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml
  54. 310
      docker/kubernetes/dolphinscheduler/values.yaml
  55. 7
      dolphinscheduler-alert-plugin/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannel.java
  56. 44
      dolphinscheduler-alert-plugin/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkParamsConstants.java
  57. 132
      dolphinscheduler-alert-plugin/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java
  58. 2
      dolphinscheduler-alert-plugin/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java
  59. 7
      dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannel.java
  60. 20
      dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailParamsConstants.java
  61. 23
      dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannelFactoryTest.java
  62. 6
      dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannelTest.java
  63. 1
      dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java
  64. 82
      dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/pom.xml
  65. 38
      dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannel.java
  66. 82
      dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannelFactory.java
  67. 26
      dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertPlugin.java
  68. 49
      dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuParamsConstants.java
  69. 214
      dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java
  70. 50
      dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/HttpRequestUtil.java
  71. 45
      dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/test/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannelFactoryTest.java
  72. 99
      dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/test/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSenderTest.java
  73. 7
      dolphinscheduler-alert-plugin/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannel.java
  74. 7
      dolphinscheduler-alert-plugin/dolphinscheduler-alert-http/src/test/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannelTest.java
  75. 7
      dolphinscheduler-alert-plugin/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannel.java
  76. 2
      dolphinscheduler-alert-plugin/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannelFactoryTest.java
  77. 7
      dolphinscheduler-alert-plugin/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannel.java
  78. 1
      dolphinscheduler-alert-plugin/pom.xml
  79. 7
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java
  80. 55
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EmailManager.java
  81. 13
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/AlertPluginManager.java
  82. 9
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/processor/AlertRequestProcessor.java
  83. 39
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java
  84. 2
      dolphinscheduler-alert/src/main/resources/alert.properties
  85. 9
      dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/AlertServerTest.java
  86. 2
      dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java
  87. 11
      dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/processor/AlertRequestProcessorTest.java
  88. 36
      dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java
  89. 42
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java
  90. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
  91. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LocaleChangeInterceptor.java
  92. 21
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java
  93. 9
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java
  94. 12
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java
  95. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java
  96. 12
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java
  97. 88
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java
  98. 10
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
  99. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java
  100. 4
      dolphinscheduler-api/src/main/resources/i18n/messages.properties
  101. Some files were not shown because too many files have changed in this diff Show More

1
.github/ISSUE_TEMPLATE/bug_report.md

@ -2,7 +2,6 @@
name: Bug report name: Bug report
about: Create a report to help us improve about: Create a report to help us improve
title: "[Bug][Module Name] Bug title " title: "[Bug][Module Name] Bug title "
labels: bug
assignees: '' assignees: ''
--- ---

1
.github/ISSUE_TEMPLATE/feature_request.md

@ -2,7 +2,6 @@
name: Feature request name: Feature request
about: Suggest an idea for this project about: Suggest an idea for this project
title: "[Feature][Module Name] Feature title" title: "[Feature][Module Name] Feature title"
labels: new feature
assignees: '' assignees: ''
--- ---

1
.github/ISSUE_TEMPLATE/improvement_suggestion.md

@ -2,7 +2,6 @@
name: Improvement suggestion name: Improvement suggestion
about: Improvement suggestion for this project about: Improvement suggestion for this project
title: "[Improvement][Module Name] Improvement title" title: "[Improvement][Module Name] Improvement title"
labels: improvement
assignees: '' assignees: ''
--- ---

1
.github/ISSUE_TEMPLATE/question.md

@ -2,7 +2,6 @@
name: Question name: Question
about: Have a question wanted to be help about: Have a question wanted to be help
title: "[Question] Question title" title: "[Question] Question title"
labels: question
assignees: '' assignees: ''
--- ---

2
.github/workflows/ci_backend.yml

@ -49,7 +49,7 @@ jobs:
with: with:
submodule: true submodule: true
- name: Check License Header - name: Check License Header
uses: apache/skywalking-eyes@9bd5feb uses: apache/skywalking-eyes@ec88b7d850018c8983f87729ea88549e100c5c82
- name: Set up JDK 1.8 - name: Set up JDK 1.8
uses: actions/setup-java@v1 uses: actions/setup-java@v1
with: with:

2
.github/workflows/ci_e2e.yml

@ -33,7 +33,7 @@ jobs:
with: with:
submodule: true submodule: true
- name: Check License Header - name: Check License Header
uses: apache/skywalking-eyes@9bd5feb uses: apache/skywalking-eyes@ec88b7d850018c8983f87729ea88549e100c5c82
- uses: actions/cache@v1 - uses: actions/cache@v1
with: with:
path: ~/.m2/repository path: ~/.m2/repository

2
.github/workflows/ci_ut.yml

@ -36,7 +36,7 @@ jobs:
with: with:
submodule: true submodule: true
- name: Check License Header - name: Check License Header
uses: apache/skywalking-eyes@9bd5feb uses: apache/skywalking-eyes@ec88b7d850018c8983f87729ea88549e100c5c82
env: env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Only enable review / suggestion here GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Only enable review / suggestion here
- uses: actions/cache@v1 - uses: actions/cache@v1

4
.gitignore vendored

@ -7,6 +7,10 @@
.target .target
.idea/ .idea/
target/ target/
dist/
all-dependencies.txt
self-modules.txt
third-party-dependencies.txt
.settings .settings
.nbproject .nbproject
.classpath .classpath

54
README.md

@ -7,46 +7,44 @@ Dolphin Scheduler Official Website
[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache-dolphinscheduler&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache-dolphinscheduler&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler)
> Dolphin Scheduler for Big Data
[![Stargazers over time](https://starchart.cc/apache/incubator-dolphinscheduler.svg)](https://starchart.cc/apache/incubator-dolphinscheduler) [![Stargazers over time](https://starchart.cc/apache/incubator-dolphinscheduler.svg)](https://starchart.cc/apache/incubator-dolphinscheduler)
[![EN doc](https://img.shields.io/badge/document-English-blue.svg)](README.md) [![EN doc](https://img.shields.io/badge/document-English-blue.svg)](README.md)
[![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](README_zh_CN.md) [![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](README_zh_CN.md)
### Design features: ### Design Features:
Dolphin Scheduler is a distributed and easy-to-extend visual DAG workflow scheduling system. It dedicates to solving the complex dependencies in data processing to make the scheduling system `out of the box` for the data processing process. DolphinScheduler is a distributed and extensible workflow scheduler platform with powerful DAG visual interfaces, dedicated to solving complex job dependencies in the data pipeline and providing various types of jobs available `out of the box`.
Its main objectives are as follows: Its main objectives are as follows:
- Associate the tasks according to the dependencies of the tasks in a DAG graph, which can visualize the running state of the task in real-time. - Associate the tasks according to the dependencies of the tasks in a DAG graph, which can visualize the running state of the task in real-time.
- Support many task types: Shell, MR, Spark, SQL (MySQL, PostgreSQL, hive, spark SQL), Python, Sub_Process, Procedure, etc. - Support various task types: Shell, MR, Spark, SQL (MySQL, PostgreSQL, hive, spark SQL), Python, Sub_Process, Procedure, etc.
- Support process scheduling, dependency scheduling, manual scheduling, manual pause/stop/recovery, support for failed retry/alarm, recovery from specified nodes, Kill task, etc. - Support scheduling of workflows and dependencies, manual scheduling to pause/stop/recover task, support failure task retry/alarm, recover specified nodes from failure, kill task, etc.
- Support the priority of process & task, task failover, and task timeout alarm or failure. - Support the priority of workflows & tasks, task failover, and task timeout alarm or failure.
- Support process global parameters and node custom parameter settings. - Support workflow global parameters and node customized parameter settings.
- Support online upload/download of resource files, management, etc. Support online file creation and editing. - Support online upload/download/management of resource files, etc. Support online file creation and editing.
- Support task log online viewing and scrolling, online download log, etc. - Support task log online viewing and scrolling and downloading, etc.
- Implement cluster HA, decentralize Master cluster and Worker cluster through Zookeeper. - Have implemented cluster HA, decentralize Master cluster and Worker cluster through Zookeeper.
- Support the viewing of Master/Worker CPU load, memory, and CPU usage metrics. - Support the viewing of Master/Worker CPU load, memory, and CPU usage metrics.
- Support presenting tree or Gantt chart of workflow history as well as the statistics results of task & process status in each workflow. - Support displaying workflow history in tree/Gantt chart, as well as statistical analysis on the task status & process status in each workflow.
- Support backfilling data. - Support back-filling data.
- Support multi-tenant. - Support multi-tenant.
- Support internationalization. - Support internationalization.
- There are more waiting for partners to explore... - More features waiting for partners to explore...
### What's in Dolphin Scheduler ### What's in DolphinScheduler
Stability | Easy to use | Features | Scalability | Stability | Easy to use | Features | Scalability |
-- | -- | -- | -- -- | -- | -- | --
Decentralized multi-master and multi-worker | Visualization process defines key information such as task status, task type, retry times, task running machine, visual variables, and so on at a glance.  |  Support pause, recover operation | Support custom task types Decentralized multi-master and multi-worker | Visualization of workflow key information, such as task status, task type, retry times, task operation machine information, visual variables, and so on at a glance.  |  Support pause, recover operation | Support customized task types
HA is supported by itself | All process definition operations are visualized, dragging tasks to draw DAGs, configuring data sources and resources. At the same time, for third-party systems, the API mode operation is provided. | Users on Dolphin Scheduler can achieve many-to-one or one-to-one mapping relationship through tenants and Hadoop users, which is very important for scheduling large data jobs. | The scheduler uses distributed scheduling, and the overall scheduling capability will increase linearly with the scale of the cluster. Master and Worker support dynamic online and offline. support HA | Visualization of all workflow operations, dragging tasks to draw DAGs, configuring data sources and resources. At the same time, for third-party systems, provide API mode operations. | Users on DolphinScheduler can achieve many-to-one or one-to-one mapping relationship through tenants and Hadoop users, which is very important for scheduling large data jobs. | The scheduler supports distributed scheduling, and the overall scheduling capability will increase linearly with the scale of the cluster. Master and Worker support dynamic adjustment.
Overload processing: Overload processing: By using the task queue mechanism, the number of schedulable tasks on a single machine can be flexibly configured. Machine jam can be avoided with high tolerance to numbers of tasks cached in task queue. | One-click deployment | Support traditional shell tasks, and big data platform task scheduling: MR, Spark, SQL (MySQL, PostgreSQL, hive, spark SQL), Python, Procedure, Sub_Process | | Overload processing: By using the task queue mechanism, the number of schedulable tasks on a single machine can be flexibly configured. Machine jam can be avoided with high tolerance to numbers of tasks cached in task queue. | One-click deployment | Support traditional shell tasks, and big data platform task scheduling: MR, Spark, SQL (MySQL, PostgreSQL, hive, spark SQL), Python, Procedure, Sub_Process | |
### System partial screenshot ### User Interface Screenshots
![home page](https://user-images.githubusercontent.com/15833811/75218288-bf286400-57d4-11ea-8263-d639c6511d5f.jpg) ![home page](https://user-images.githubusercontent.com/15833811/75218288-bf286400-57d4-11ea-8263-d639c6511d5f.jpg)
![dag](https://user-images.githubusercontent.com/15833811/75236750-3374fe80-57f9-11ea-857d-62a66a5a559d.png) ![dag](https://user-images.githubusercontent.com/15833811/75236750-3374fe80-57f9-11ea-857d-62a66a5a559d.png)
@ -57,13 +55,9 @@ Overload processing: Overload processing: By using the task queue mechanism, the
![security](https://user-images.githubusercontent.com/15833811/75236441-bfd2f180-57f8-11ea-88bd-f24311e01b7e.png) ![security](https://user-images.githubusercontent.com/15833811/75236441-bfd2f180-57f8-11ea-88bd-f24311e01b7e.png)
![treeview](https://user-images.githubusercontent.com/15833811/75217191-3fe56100-57d1-11ea-8856-f19180d9a879.png) ![treeview](https://user-images.githubusercontent.com/15833811/75217191-3fe56100-57d1-11ea-8856-f19180d9a879.png)
### QuickStart in Docker
Please referer the official website document:[[QuickStart in Docker](https://dolphinscheduler.apache.org/en-us/docs/1.3.4/user_doc/docker-deployment.html)]
### Recent R&D plan
The work plan of Dolphin Scheduler: [R&D plan](https://github.com/apache/incubator-dolphinscheduler/projects/1), which `In Develop` card shows the features that are currently being developed and TODO card lists what needs to be done(including feature ideas).
### How to contribute
Welcome to participate in contributing, please refer to this website to find out more: [[How to contribute](https://dolphinscheduler.apache.org/en-us/docs/development/contribute.html)]
### How to Build ### How to Build
@ -80,14 +74,16 @@ dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${latest.release
### Thanks ### Thanks
Dolphin Scheduler is based on a lot of excellent open-source projects, such as google guava, guice, grpc, netty, ali bonecp, quartz, and many open-source projects of Apache and so on. DolphinScheduler is based on a lot of excellent open-source projects, such as google guava, guice, grpc, netty, ali bonecp, quartz, and many open-source projects of Apache and so on.
We would like to express our deep gratitude to all the open-source projects which contribute to making the dream of Dolphin Scheduler comes true. We hope that we are not only the beneficiaries of open-source, but also give back to the community. Besides, we expect the partners who have the same passion and conviction to open-source will join in and contribute to the open-source community! We would like to express our deep gratitude to all the open-source projects used in Dolphin Scheduler. We hope that we are not only the beneficiaries of open-source, but also give back to the community. Besides, we hope everyone who have the same enthusiasm and passion for open source could join in and contribute to the open-source community!
### Get Help ### Get Help
1. Submit an issue 1. Submit an [[issue](https://github.com/apache/incubator-dolphinscheduler/issues/new/choose)]
1. Subscribe to the mail list: https://dolphinscheduler.apache.org/en-us/docs/development/subscribe.html, then email dev@dolphinscheduler.apache.org 1. Subscribe to the mail list: https://dolphinscheduler.apache.org/en-us/docs/development/subscribe.html, then email dev@dolphinscheduler.apache.org
### How to Contribute
The community welcomes everyone to participate in contributing, please refer to this website to find out more: [[How to contribute](https://dolphinscheduler.apache.org/en-us/community/development/contribute.html)]
### License ### License
Please refer to the [LICENSE](https://github.com/apache/incubator-dolphinscheduler/blob/dev/LICENSE) file. Please refer to the [LICENSE](https://github.com/apache/incubator-dolphinscheduler/blob/dev/LICENSE) file.

109
ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-worker.xml

@ -15,53 +15,64 @@
~ limitations under the License. ~ limitations under the License.
--> -->
<configuration> <configuration>
<property> <property>
<name>worker.exec.threads</name> <name>worker.exec.threads</name>
<value>100</value> <value>100</value>
<value-attributes> <value-attributes>
<type>int</type> <type>int</type>
</value-attributes> </value-attributes>
<description>worker execute thread num</description> <description>worker execute thread num</description>
<on-ambari-upgrade add="true"/> <on-ambari-upgrade add="true"/>
</property> </property>
<property> <property>
<name>worker.heartbeat.interval</name> <name>worker.heartbeat.interval</name>
<value>10</value> <value>10</value>
<value-attributes> <value-attributes>
<type>int</type> <type>int</type>
</value-attributes> </value-attributes>
<description>worker heartbeat interval</description> <description>worker heartbeat interval</description>
<on-ambari-upgrade add="true"/> <on-ambari-upgrade add="true"/>
</property> </property>
<property> <property>
<name>worker.max.cpuload.avg</name> <name>worker.max.cpuload.avg</name>
<value>100</value> <value>100</value>
<value-attributes> <value-attributes>
<type>int</type> <type>int</type>
</value-attributes> </value-attributes>
<description>only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2</description> <description>only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2
<on-ambari-upgrade add="true"/> </description>
</property> <on-ambari-upgrade add="true"/>
<property> </property>
<name>worker.reserved.memory</name> <property>
<value>0.3</value> <name>worker.reserved.memory</name>
<description>only larger than reserved memory, worker server can work. default value : physical memory * 1/10, unit is G.</description> <value>0.3</value>
<on-ambari-upgrade add="true"/> <description>only larger than reserved memory, worker server can work. default value : physical memory * 1/10,
</property> unit is G.
</description>
<property> <on-ambari-upgrade add="true"/>
<name>worker.listen.port</name> </property>
<value>1234</value> <property>
<value-attributes> <name>worker.listen.port</name>
<type>int</type> <value>1234</value>
</value-attributes> <value-attributes>
<description>worker listen port</description> <type>int</type>
<on-ambari-upgrade add="true"/> </value-attributes>
</property> <description>worker listen port</description>
<property> <on-ambari-upgrade add="true"/>
<name>worker.groups</name> </property>
<value>default</value> <property>
<description>default worker group</description> <name>worker.groups</name>
<on-ambari-upgrade add="true"/> <value>default</value>
</property> <description>default worker group</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.weigth</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>worker weight</description>
<on-ambari-upgrade add="true"/>
</property>
</configuration> </configuration>

57
docker/build/Dockerfile

@ -15,55 +15,37 @@
# limitations under the License. # limitations under the License.
# #
FROM nginx:alpine FROM openjdk:8-jdk-alpine
ARG VERSION ARG VERSION
ENV TZ Asia/Shanghai ENV TZ Asia/Shanghai
ENV LANG C.UTF-8 ENV LANG C.UTF-8
ENV DEBIAN_FRONTEND noninteractive ENV DOCKER true
#1. install dos2unix shadow bash openrc python sudo vim wget iputils net-tools ssh pip tini kazoo. # 1. install command/library/software
#If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example: # If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example:
#RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories # RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories
# RUN sed -i 's/dl-cdn.alpinelinux.org/mirror.tuna.tsinghua.edu.cn/g' /etc/apk/repositories
RUN apk update && \ RUN apk update && \
apk add --update --no-cache dos2unix shadow bash openrc python2 python3 sudo vim wget iputils net-tools openssh-server py-pip tini && \ apk add --no-cache tzdata dos2unix bash python2 python3 procps sudo shadow tini postgresql-client && \
apk add --update --no-cache procps && \ cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
openrc boot && \ apk del tzdata && \
pip install kazoo rm -rf /var/cache/apk/*
#2. install jdk # 2. add dolphinscheduler
RUN apk add --update --no-cache openjdk8
ENV JAVA_HOME /usr/lib/jvm/java-1.8-openjdk
ENV PATH $JAVA_HOME/bin:$PATH
#3. add dolphinscheduler
ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz /opt/ ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz /opt/
RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ /opt/dolphinscheduler/ RUN ln -s /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin /opt/dolphinscheduler
ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
#4. install database, if use mysql as your backend database, the `mysql-client` package should be installed # 3. add configuration and modify permissions and set soft links
RUN apk add --update --no-cache postgresql postgresql-contrib
#5. modify nginx
RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \
rm -rf /etc/nginx/conf.d/*
COPY ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d
#6. add configuration and modify permissions and set soft links
COPY ./checkpoint.sh /root/checkpoint.sh COPY ./checkpoint.sh /root/checkpoint.sh
COPY ./startup-init-conf.sh /root/startup-init-conf.sh COPY ./startup-init-conf.sh /root/startup-init-conf.sh
COPY ./startup.sh /root/startup.sh COPY ./startup.sh /root/startup.sh
COPY ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/ COPY ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/
COPY ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/ COPY ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/
COPY conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/ COPY ./conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/
RUN chmod +x /root/checkpoint.sh && \ RUN dos2unix /root/checkpoint.sh && \
chmod +x /root/startup-init-conf.sh && \
chmod +x /root/startup.sh && \
chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \
chmod +x /opt/dolphinscheduler/script/*.sh && \
chmod +x /opt/dolphinscheduler/bin/*.sh && \
dos2unix /root/checkpoint.sh && \
dos2unix /root/startup-init-conf.sh && \ dos2unix /root/startup-init-conf.sh && \
dos2unix /root/startup.sh && \ dos2unix /root/startup.sh && \
dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \ dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \
@ -71,13 +53,10 @@ RUN chmod +x /root/checkpoint.sh && \
dos2unix /opt/dolphinscheduler/bin/*.sh && \ dos2unix /opt/dolphinscheduler/bin/*.sh && \
rm -rf /bin/sh && \ rm -rf /bin/sh && \
ln -s /bin/bash /bin/sh && \ ln -s /bin/bash /bin/sh && \
mkdir -p /tmp/xls && \ mkdir -p /var/mail /tmp/xls && \
#7. remove apk index cache and disable coredup for sudo
rm -rf /var/cache/apk/* && \
echo "Set disable_coredump false" >> /etc/sudo.conf echo "Set disable_coredump false" >> /etc/sudo.conf
# 4. expose port
#8. expose port EXPOSE 5678 1234 12345 50051
EXPOSE 2181 2888 3888 5432 5678 1234 12345 50051 8888
ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"] ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"]

36
docker/build/README.md

@ -15,9 +15,9 @@ Official Website: https://dolphinscheduler.apache.org
#### You can start a dolphinscheduler instance #### You can start a dolphinscheduler instance
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \ -e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \
-p 8888:8888 \ -p 12345:12345 \
dolphinscheduler all dolphinscheduler all
``` ```
@ -33,7 +33,7 @@ You can specify **existing postgres service**. Example:
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-p 8888:8888 \ -p 12345:12345 \
dolphinscheduler all dolphinscheduler all
``` ```
@ -43,7 +43,7 @@ You can specify **existing zookeeper service**. Example:
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181" -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \
-p 8888:8888 \ -p 12345:12345 \
dolphinscheduler all dolphinscheduler all
``` ```
@ -90,14 +90,6 @@ $ docker run -dit --name dolphinscheduler \
dolphinscheduler alert-server dolphinscheduler alert-server
``` ```
* Start a **frontend**, For example:
```
$ docker run -dit --name dolphinscheduler \
-e FRONTEND_API_SERVER_HOST="192.168.x.x" -e FRONTEND_API_SERVER_PORT="12345" \
-p 8888:8888 \
dolphinscheduler frontend
```
**Note**: You must be specify `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server. **Note**: You must be specify `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server.
@ -146,7 +138,7 @@ This environment variable sets the host for database. The default value is `127.
This environment variable sets the port for database. The default value is `5432`. This environment variable sets the port for database. The default value is `5432`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. **Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`DATABASE_USERNAME`** **`DATABASE_USERNAME`**
@ -306,18 +298,6 @@ This environment variable sets enterprise wechat agent id for `alert-server`. Th
This environment variable sets enterprise wechat users for `alert-server`. The default value is empty. This environment variable sets enterprise wechat users for `alert-server`. The default value is empty.
**`FRONTEND_API_SERVER_HOST`**
This environment variable sets api server host for `frontend`. The default value is `127.0.0.1`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `api-server`.
**`FRONTEND_API_SERVER_PORT`**
This environment variable sets api server port for `frontend`. The default value is `123451`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `api-server`.
## Initialization scripts ## Initialization scripts
If you would like to do additional initialization in an image derived from this one, add one or more environment variable under `/root/start-init-conf.sh`, and modify template files in `/opt/dolphinscheduler/conf/*.tpl`. If you would like to do additional initialization in an image derived from this one, add one or more environment variable under `/root/start-init-conf.sh`, and modify template files in `/opt/dolphinscheduler/conf/*.tpl`.
@ -326,7 +306,7 @@ For example, to add an environment variable `API_SERVER_PORT` in `/root/start-in
``` ```
export API_SERVER_PORT=5555 export API_SERVER_PORT=5555
``` ```
and to modify `/opt/dolphinscheduler/conf/application-api.properties.tpl` template file, add server port: and to modify `/opt/dolphinscheduler/conf/application-api.properties.tpl` template file, add server port:
``` ```
@ -343,8 +323,4 @@ $(cat ${DOLPHINSCHEDULER_HOME}/conf/${line})
EOF EOF
" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*} " > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*}
done done
echo "generate nginx config"
sed -i "s/FRONTEND_API_SERVER_HOST/${FRONTEND_API_SERVER_HOST}/g" /etc/nginx/conf.d/dolphinscheduler.conf
sed -i "s/FRONTEND_API_SERVER_PORT/${FRONTEND_API_SERVER_PORT}/g" /etc/nginx/conf.d/dolphinscheduler.conf
``` ```

35
docker/build/README_zh_CN.md

@ -15,9 +15,9 @@ Official Website: https://dolphinscheduler.apache.org
#### 你可以运行一个dolphinscheduler实例 #### 你可以运行一个dolphinscheduler实例
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \ -e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \
-p 8888:8888 \ -p 12345:12345 \
dolphinscheduler all dolphinscheduler all
``` ```
@ -33,7 +33,7 @@ dolphinscheduler all
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-p 8888:8888 \ -p 12345:12345 \
dolphinscheduler all dolphinscheduler all
``` ```
@ -43,7 +43,7 @@ dolphinscheduler all
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181" -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \
-p 8888:8888 \ -p 12345:12345 \
dolphinscheduler all dolphinscheduler all
``` ```
@ -90,15 +90,6 @@ $ docker run -dit --name dolphinscheduler \
dolphinscheduler alert-server dolphinscheduler alert-server
``` ```
* 启动一个 **frontend**, 如下:
```
$ docker run -dit --name dolphinscheduler \
-e FRONTEND_API_SERVER_HOST="192.168.x.x" -e FRONTEND_API_SERVER_PORT="12345" \
-p 8888:8888 \
dolphinscheduler frontend
```
**注意**: 当你运行dolphinscheduler中的部分服务时,你必须指定这些环境变量 `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM` **注意**: 当你运行dolphinscheduler中的部分服务时,你必须指定这些环境变量 `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM`
## 如何构建一个docker镜像 ## 如何构建一个docker镜像
@ -306,18 +297,6 @@ Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些
配置`alert-server`的邮件服务企业微信`USERS`,默认值 `空` 配置`alert-server`的邮件服务企业微信`USERS`,默认值 `空`
**`FRONTEND_API_SERVER_HOST`**
配置`frontend`的连接`api-server`的地址,默认值 `127.0.0.1`
**Note**: 当单独运行`api-server`时,你应该指定`api-server`这个值。
**`FRONTEND_API_SERVER_PORT`**
配置`frontend`的连接`api-server`的端口,默认值 `12345`
**Note**: 当单独运行`api-server`时,你应该指定`api-server`这个值。
## 初始化脚本 ## 初始化脚本
如果你想在编译的时候或者运行的时候附加一些其它的操作及新增一些环境变量,你可以在`/root/start-init-conf.sh`文件中进行修改,同时如果涉及到配置文件的修改,请在`/opt/dolphinscheduler/conf/*.tpl`中修改相应的配置文件 如果你想在编译的时候或者运行的时候附加一些其它的操作及新增一些环境变量,你可以在`/root/start-init-conf.sh`文件中进行修改,同时如果涉及到配置文件的修改,请在`/opt/dolphinscheduler/conf/*.tpl`中修改相应的配置文件
@ -326,7 +305,7 @@ Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些
``` ```
export API_SERVER_PORT=5555 export API_SERVER_PORT=5555
``` ```
当添加以上环境变量后,你应该在相应的模板文件`/opt/dolphinscheduler/conf/application-api.properties.tpl`中添加这个环境变量配置: 当添加以上环境变量后,你应该在相应的模板文件`/opt/dolphinscheduler/conf/application-api.properties.tpl`中添加这个环境变量配置:
``` ```
@ -343,8 +322,4 @@ $(cat ${DOLPHINSCHEDULER_HOME}/conf/${line})
EOF EOF
" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*} " > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*}
done done
echo "generate nginx config"
sed -i "s/FRONTEND_API_SERVER_HOST/${FRONTEND_API_SERVER_HOST}/g" /etc/nginx/conf.d/dolphinscheduler.conf
sed -i "s/FRONTEND_API_SERVER_PORT/${FRONTEND_API_SERVER_PORT}/g" /etc/nginx/conf.d/dolphinscheduler.conf
``` ```

0
docker/build/checkpoint.sh

12
docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh vendored

@ -15,6 +15,14 @@
# limitations under the License. # limitations under the License.
# #
export PYTHON_HOME=/usr/bin/python2 export HADOOP_HOME=/opt/soft/hadoop
export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
export SPARK_HOME1=/opt/soft/spark1
export SPARK_HOME2=/opt/soft/spark2
export PYTHON_HOME=/usr/bin/python
export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk
export PATH=$PYTHON_HOME/bin:$JAVA_HOME/bin:$PATH export HIVE_HOME=/opt/soft/hive
export FLINK_HOME=/opt/soft/flink
export DATAX_HOME=/opt/soft/datax/bin/datax.py
export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH

9
docker/build/conf/dolphinscheduler/logback/logback-alert.xml

@ -20,14 +20,6 @@
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" --> <configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
<property name="log.base" value="logs"/> <property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<appender name="ALERTLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> <appender name="ALERTLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${log.base}/dolphinscheduler-alert.log</file> <file>${log.base}/dolphinscheduler-alert.log</file>
@ -45,7 +37,6 @@
</appender> </appender>
<root level="INFO"> <root level="INFO">
<appender-ref ref="STDOUT"/>
<appender-ref ref="ALERTLOGFILE"/> <appender-ref ref="ALERTLOGFILE"/>
</root> </root>

9
docker/build/conf/dolphinscheduler/logback/logback-api.xml

@ -20,14 +20,6 @@
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" --> <configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
<property name="log.base" value="logs"/> <property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<!-- api server logback config start --> <!-- api server logback config start -->
<appender name="APILOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> <appender name="APILOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
@ -55,7 +47,6 @@
<root level="INFO"> <root level="INFO">
<appender-ref ref="STDOUT"/>
<appender-ref ref="APILOGFILE"/> <appender-ref ref="APILOGFILE"/>
</root> </root>

9
docker/build/conf/dolphinscheduler/logback/logback-master.xml

@ -20,14 +20,6 @@
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" --> <configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
<property name="log.base" value="logs"/> <property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<conversionRule conversionWord="messsage" <conversionRule conversionWord="messsage"
converterClass="org.apache.dolphinscheduler.server.log.SensitiveDataConverter"/> converterClass="org.apache.dolphinscheduler.server.log.SensitiveDataConverter"/>
@ -74,7 +66,6 @@
<!-- master server logback config end --> <!-- master server logback config end -->
<root level="INFO"> <root level="INFO">
<appender-ref ref="STDOUT"/>
<appender-ref ref="TASKLOGFILE"/> <appender-ref ref="TASKLOGFILE"/>
<appender-ref ref="MASTERLOGFILE"/> <appender-ref ref="MASTERLOGFILE"/>
</root> </root>

9
docker/build/conf/dolphinscheduler/logback/logback-worker.xml

@ -20,14 +20,6 @@
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" --> <configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
<property name="log.base" value="logs"/> <property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<!-- worker server logback config start --> <!-- worker server logback config start -->
<conversionRule conversionWord="messsage" <conversionRule conversionWord="messsage"
@ -75,7 +67,6 @@
<!-- worker server logback config end --> <!-- worker server logback config end -->
<root level="INFO"> <root level="INFO">
<appender-ref ref="STDOUT"/>
<appender-ref ref="TASKLOGFILE"/> <appender-ref ref="TASKLOGFILE"/>
<appender-ref ref="WORKERLOGFILE"/> <appender-ref ref="WORKERLOGFILE"/>
</root> </root>

51
docker/build/conf/nginx/dolphinscheduler.conf

@ -1,51 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
server {
listen 8888;
server_name localhost;
#charset koi8-r;
#access_log /var/log/nginx/host.access.log main;
location / {
root /opt/dolphinscheduler/ui;
index index.html index.html;
}
location /dolphinscheduler/ui{
alias /opt/dolphinscheduler/ui;
}
location /dolphinscheduler {
proxy_pass http://FRONTEND_API_SERVER_HOST:FRONTEND_API_SERVER_PORT;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header x_real_ipP $remote_addr;
proxy_set_header remote_addr $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_http_version 1.1;
proxy_connect_timeout 300s;
proxy_read_timeout 300s;
proxy_send_timeout 300s;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
#error_page 404 /404.html;
# redirect server error pages to the static page /50x.html
#
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root /usr/share/nginx/html;
}
}

47
docker/build/conf/zookeeper/zoo.cfg

@ -1,47 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial
# synchronization phase can take
initLimit=10
# The number of ticks that can pass between
# sending a request and getting an acknowledgement
syncLimit=5
# the directory where the snapshot is stored.
# do not use /tmp for storage, /tmp here is just
# example sakes.
dataDir=/tmp/zookeeper
# the port at which the clients will connect
clientPort=2181
# the maximum number of client connections.
# increase this if you need to handle more clients
#maxClientCnxns=60
#
# Be sure to read the maintenance section of the
# administrator guide before turning on autopurge.
#
# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
#
# The number of snapshots to retain in dataDir
#autopurge.snapRetainCount=3
# Purge task interval in hours
# Set to "0" to disable auto purge feature
#autopurge.purgeInterval=1
#Four Letter Words commands:stat,ruok,conf,isro
4lw.commands.whitelist=*

9
docker/build/hooks/build

@ -48,7 +48,12 @@ echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubati
mv "$(pwd)"/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-"${VERSION}"-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/ mv "$(pwd)"/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-"${VERSION}"-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/
# docker build # docker build
echo -e "docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/\n" BUILD_COMMAND="docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/"
sudo docker build --build-arg VERSION="${VERSION}" -t $DOCKER_REPO:"${VERSION}" "$(pwd)/docker/build/" echo -e "$BUILD_COMMAND\n"
if (docker info 2> /dev/null | grep -i "ERROR"); then
sudo $BUILD_COMMAND
else
$BUILD_COMMAND
fi
echo "------ dolphinscheduler end - build -------" echo "------ dolphinscheduler end - build -------"

0
docker/build/hooks/push

18
docker/build/startup-init-conf.sh

@ -39,9 +39,9 @@ export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"}
export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"} export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"}
export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"} export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"}
export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""} export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""}
export RESOURCE_STORAGE_TYPE=${RESOURCE_STORAGE_TYPE:-"NONE"} export RESOURCE_STORAGE_TYPE=${RESOURCE_STORAGE_TYPE:-"HDFS"}
export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/ds"} export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/dolphinscheduler"}
export FS_DEFAULT_FS=${FS_DEFAULT_FS:-"s3a://xxxx"} export FS_DEFAULT_FS=${FS_DEFAULT_FS:-"file:///"}
export FS_S3A_ENDPOINT=${FS_S3A_ENDPOINT:-"s3.xxx.amazonaws.com"} export FS_S3A_ENDPOINT=${FS_S3A_ENDPOINT:-"s3.xxx.amazonaws.com"}
export FS_S3A_ACCESS_KEY=${FS_S3A_ACCESS_KEY:-"xxxxxxx"} export FS_S3A_ACCESS_KEY=${FS_S3A_ACCESS_KEY:-"xxxxxxx"}
export FS_S3A_SECRET_KEY=${FS_S3A_SECRET_KEY:-"xxxxxxx"} export FS_S3A_SECRET_KEY=${FS_S3A_SECRET_KEY:-"xxxxxxx"}
@ -81,7 +81,7 @@ export WORKER_WEIGHT=${WORKER_WEIGHT:-"100"}
#============================================================================ #============================================================================
# alert plugin dir # alert plugin dir
export ALERT_PLUGIN_DIR=${ALERT_PLUGIN_DIR:-"/opt/dolphinscheduler"} export ALERT_PLUGIN_DIR=${ALERT_PLUGIN_DIR:-"/opt/dolphinscheduler"}
# XLS FILE # xls file
export XLS_FILE_PATH=${XLS_FILE_PATH:-"/tmp/xls"} export XLS_FILE_PATH=${XLS_FILE_PATH:-"/tmp/xls"}
# mail # mail
export MAIL_SERVER_HOST=${MAIL_SERVER_HOST:-""} export MAIL_SERVER_HOST=${MAIL_SERVER_HOST:-""}
@ -99,12 +99,6 @@ export ENTERPRISE_WECHAT_SECRET=${ENTERPRISE_WECHAT_SECRET:-""}
export ENTERPRISE_WECHAT_AGENT_ID=${ENTERPRISE_WECHAT_AGENT_ID:-""} export ENTERPRISE_WECHAT_AGENT_ID=${ENTERPRISE_WECHAT_AGENT_ID:-""}
export ENTERPRISE_WECHAT_USERS=${ENTERPRISE_WECHAT_USERS:-""} export ENTERPRISE_WECHAT_USERS=${ENTERPRISE_WECHAT_USERS:-""}
#============================================================================
# Frontend
#============================================================================
export FRONTEND_API_SERVER_HOST=${FRONTEND_API_SERVER_HOST:-"127.0.0.1"}
export FRONTEND_API_SERVER_PORT=${FRONTEND_API_SERVER_PORT:-"12345"}
echo "generate app config" echo "generate app config"
ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do
eval "cat << EOF eval "cat << EOF
@ -112,7 +106,3 @@ $(cat ${DOLPHINSCHEDULER_HOME}/conf/${line})
EOF EOF
" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*} " > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*}
done done
echo "generate nginx config"
sed -i "s/FRONTEND_API_SERVER_HOST/${FRONTEND_API_SERVER_HOST}/g" /etc/nginx/conf.d/dolphinscheduler.conf
sed -i "s/FRONTEND_API_SERVER_PORT/${FRONTEND_API_SERVER_PORT}/g" /etc/nginx/conf.d/dolphinscheduler.conf

55
docker/build/startup.sh

@ -22,8 +22,8 @@ DOLPHINSCHEDULER_BIN=${DOLPHINSCHEDULER_HOME}/bin
DOLPHINSCHEDULER_SCRIPT=${DOLPHINSCHEDULER_HOME}/script DOLPHINSCHEDULER_SCRIPT=${DOLPHINSCHEDULER_HOME}/script
DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs
# start database # wait database
initDatabase() { waitDatabase() {
echo "test ${DATABASE_TYPE} service" echo "test ${DATABASE_TYPE} service"
while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
counter=$((counter+1)) counter=$((counter+1))
@ -43,19 +43,22 @@ initDatabase() {
exit 1 exit 1
fi fi
else else
v=$(sudo -u postgres PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "select 1") v=$(PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "select 1")
if [ "$(echo ${v} | grep 'FATAL' | wc -l)" -eq 1 ]; then if [ "$(echo ${v} | grep 'FATAL' | wc -l)" -eq 1 ]; then
echo "Error: Can't connect to database...${v}" echo "Error: Can't connect to database...${v}"
exit 1 exit 1
fi fi
fi fi
}
# init database
initDatabase() {
echo "import sql data" echo "import sql data"
${DOLPHINSCHEDULER_SCRIPT}/create-dolphinscheduler.sh ${DOLPHINSCHEDULER_SCRIPT}/create-dolphinscheduler.sh
} }
# start zk # wait zk
initZK() { waitZK() {
echo "connect remote zookeeper" echo "connect remote zookeeper"
echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do
while ! nc -z ${line%:*} ${line#*:}; do while ! nc -z ${line%:*} ${line#*:}; do
@ -70,12 +73,6 @@ initZK() {
done done
} }
# start nginx
initNginx() {
echo "start nginx"
nginx &
}
# start master-server # start master-server
initMasterServer() { initMasterServer() {
echo "start master-server" echo "start master-server"
@ -115,59 +112,54 @@ initAlertServer() {
printUsage() { printUsage() {
echo -e "Dolphin Scheduler is a distributed and easy-to-expand visual DAG workflow scheduling system," echo -e "Dolphin Scheduler is a distributed and easy-to-expand visual DAG workflow scheduling system,"
echo -e "dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.\n" echo -e "dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.\n"
echo -e "Usage: [ all | master-server | worker-server | api-server | alert-server | frontend ]\n" echo -e "Usage: [ all | master-server | worker-server | api-server | alert-server ]\n"
printf "%-13s: %s\n" "all" "Run master-server, worker-server, api-server, alert-server and frontend." printf "%-13s: %s\n" "all" "Run master-server, worker-server, api-server and alert-server"
printf "%-13s: %s\n" "master-server" "MasterServer is mainly responsible for DAG task split, task submission monitoring." printf "%-13s: %s\n" "master-server" "MasterServer is mainly responsible for DAG task split, task submission monitoring."
printf "%-13s: %s\n" "worker-server" "WorkerServer is mainly responsible for task execution and providing log services.." printf "%-13s: %s\n" "worker-server" "WorkerServer is mainly responsible for task execution and providing log services."
printf "%-13s: %s\n" "api-server" "ApiServer is mainly responsible for processing requests from the front-end UI layer." printf "%-13s: %s\n" "api-server" "ApiServer is mainly responsible for processing requests and providing the front-end UI layer."
printf "%-13s: %s\n" "alert-server" "AlertServer mainly include Alarms." printf "%-13s: %s\n" "alert-server" "AlertServer mainly include Alarms."
printf "%-13s: %s\n" "frontend" "Frontend mainly provides various visual operation interfaces of the system."
} }
# init config file # init config file
source /root/startup-init-conf.sh source /root/startup-init-conf.sh
LOGFILE=/var/log/nginx/access.log
case "$1" in case "$1" in
(all) (all)
initZK waitZK
waitDatabase
initDatabase initDatabase
initMasterServer initMasterServer
initWorkerServer initWorkerServer
initApiServer initApiServer
initAlertServer initAlertServer
initLoggerServer initLoggerServer
initNginx LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log
LOGFILE=/var/log/nginx/access.log
;; ;;
(master-server) (master-server)
initZK waitZK
initDatabase waitDatabase
initMasterServer initMasterServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-master.log LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-master.log
;; ;;
(worker-server) (worker-server)
initZK waitZK
initDatabase waitDatabase
initWorkerServer initWorkerServer
initLoggerServer initLoggerServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log
;; ;;
(api-server) (api-server)
initZK waitZK
waitDatabase
initDatabase initDatabase
initApiServer initApiServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log
;; ;;
(alert-server) (alert-server)
initDatabase waitDatabase
initAlertServer initAlertServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-alert.log LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-alert.log
;; ;;
(frontend)
initNginx
LOGFILE=/var/log/nginx/access.log
;;
(help) (help)
printUsage printUsage
exit 1 exit 1
@ -179,8 +171,7 @@ case "$1" in
esac esac
# init directories and log files # init directories and log files
mkdir -p ${DOLPHINSCHEDULER_LOGS} && mkdir -p /var/log/nginx/ && cat /dev/null >> ${LOGFILE} mkdir -p ${DOLPHINSCHEDULER_LOGS} && cat /dev/null >> ${LOGFILE}
echo "tail begin" echo "tail begin"
exec bash -c "tail -n 1 -f ${LOGFILE}" exec bash -c "tail -n 1 -f ${LOGFILE}"

2
docker/docker-swarm/check

@ -25,7 +25,7 @@ else
echo "Server start failed "$server_num echo "Server start failed "$server_num
exit 1 exit 1
fi fi
ready=`curl http://127.0.0.1:8888/dolphinscheduler/login -d 'userName=admin&userPassword=dolphinscheduler123' -v | grep "login success" | wc -l` ready=`curl http://127.0.0.1:12345/dolphinscheduler/login -d 'userName=admin&userPassword=dolphinscheduler123' -v | grep "login success" | wc -l`
if [ $ready -eq 1 ] if [ $ready -eq 1 ]
then then
echo "Servers is ready" echo "Servers is ready"

80
docker/docker-swarm/docker-compose.yml

@ -31,6 +31,7 @@ services:
volumes: volumes:
- dolphinscheduler-postgresql:/bitnami/postgresql - dolphinscheduler-postgresql:/bitnami/postgresql
- dolphinscheduler-postgresql-initdb:/docker-entrypoint-initdb.d - dolphinscheduler-postgresql-initdb:/docker-entrypoint-initdb.d
restart: unless-stopped
networks: networks:
- dolphinscheduler - dolphinscheduler
@ -45,13 +46,14 @@ services:
ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
volumes: volumes:
- dolphinscheduler-zookeeper:/bitnami/zookeeper - dolphinscheduler-zookeeper:/bitnami/zookeeper
restart: unless-stopped
networks: networks:
- dolphinscheduler - dolphinscheduler
dolphinscheduler-api: dolphinscheduler-api:
image: apache/dolphinscheduler:latest image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-api container_name: dolphinscheduler-api
command: ["api-server"] command: api-server
ports: ports:
- 12345:12345 - 12345:12345
environment: environment:
@ -62,6 +64,9 @@ services:
DATABASE_PASSWORD: root DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
RESOURCE_STORAGE_TYPE: HDFS
RESOURCE_UPLOAD_PATH: /dolphinscheduler
FS_DEFAULT_FS: file:///
healthcheck: healthcheck:
test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
interval: 30s interval: 30s
@ -72,37 +77,16 @@ services:
- dolphinscheduler-postgresql - dolphinscheduler-postgresql
- dolphinscheduler-zookeeper - dolphinscheduler-zookeeper
volumes: volumes:
- ./dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks: - dolphinscheduler-resource-local:/dolphinscheduler
- dolphinscheduler restart: unless-stopped
dolphinscheduler-frontend:
image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-frontend
command: ["frontend"]
ports:
- 8888:8888
environment:
TZ: Asia/Shanghai
FRONTEND_API_SERVER_HOST: dolphinscheduler-api
FRONTEND_API_SERVER_PORT: 12345
healthcheck:
test: ["CMD", "nc", "-z", "localhost", "8888"]
interval: 30s
timeout: 5s
retries: 3
start_period: 30s
depends_on:
- dolphinscheduler-api
volumes:
- ./dolphinscheduler-logs:/var/log/nginx
networks: networks:
- dolphinscheduler - dolphinscheduler
dolphinscheduler-alert: dolphinscheduler-alert:
image: apache/dolphinscheduler:latest image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-alert container_name: dolphinscheduler-alert
command: ["alert-server"] command: alert-server
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
XLS_FILE_PATH: "/tmp/xls" XLS_FILE_PATH: "/tmp/xls"
@ -133,14 +117,15 @@ services:
depends_on: depends_on:
- dolphinscheduler-postgresql - dolphinscheduler-postgresql
volumes: volumes:
- ./dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-logs:/opt/dolphinscheduler/logs
restart: unless-stopped
networks: networks:
- dolphinscheduler - dolphinscheduler
dolphinscheduler-master: dolphinscheduler-master:
image: apache/dolphinscheduler:latest image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-master container_name: dolphinscheduler-master
command: ["master-server"] command: master-server
ports: ports:
- 5678:5678 - 5678:5678
environment: environment:
@ -168,14 +153,15 @@ services:
- dolphinscheduler-postgresql - dolphinscheduler-postgresql
- dolphinscheduler-zookeeper - dolphinscheduler-zookeeper
volumes: volumes:
- ./dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-logs:/opt/dolphinscheduler/logs
restart: unless-stopped
networks: networks:
- dolphinscheduler - dolphinscheduler
dolphinscheduler-worker: dolphinscheduler-worker:
image: apache/dolphinscheduler:latest image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-worker container_name: dolphinscheduler-worker
command: ["worker-server"] command: worker-server
ports: ports:
- 1234:1234 - 1234:1234
- 50051:50051 - 50051:50051
@ -188,30 +174,40 @@ services:
WORKER_RESERVED_MEMORY: "0.1" WORKER_RESERVED_MEMORY: "0.1"
WORKER_GROUP: "default" WORKER_GROUP: "default"
WORKER_WEIGHT: "100" WORKER_WEIGHT: "100"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler
XLS_FILE_PATH: "/tmp/xls"
MAIL_SERVER_HOST: ""
MAIL_SERVER_PORT: ""
MAIL_SENDER: ""
MAIL_USER: ""
MAIL_PASSWD: ""
MAIL_SMTP_STARTTLS_ENABLE: "false"
MAIL_SMTP_SSL_ENABLE: "false"
MAIL_SMTP_SSL_TRUST: ""
DATABASE_HOST: dolphinscheduler-postgresql DATABASE_HOST: dolphinscheduler-postgresql
DATABASE_PORT: 5432 DATABASE_PORT: 5432
DATABASE_USERNAME: root DATABASE_USERNAME: root
DATABASE_PASSWORD: root DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
RESOURCE_STORAGE_TYPE: HDFS
RESOURCE_UPLOAD_PATH: /dolphinscheduler
FS_DEFAULT_FS: file:///
healthcheck: healthcheck:
test: ["CMD", "/root/checkpoint.sh", "WorkerServer"] test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]
interval: 30s interval: 30s
timeout: 5s timeout: 5s
retries: 3 retries: 3
start_period: 30s start_period: 30s
depends_on: depends_on:
- dolphinscheduler-postgresql - dolphinscheduler-postgresql
- dolphinscheduler-zookeeper - dolphinscheduler-zookeeper
volumes: volumes:
- type: bind - ./dolphinscheduler_env.sh:/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh
source: ./dolphinscheduler_env.sh - dolphinscheduler-worker-data:/tmp/dolphinscheduler
target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh - dolphinscheduler-logs:/opt/dolphinscheduler/logs
- type: volume - dolphinscheduler-resource-local:/dolphinscheduler
source: dolphinscheduler-worker-data restart: unless-stopped
target: /tmp/dolphinscheduler
- ./dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks: networks:
- dolphinscheduler - dolphinscheduler
@ -224,7 +220,5 @@ volumes:
dolphinscheduler-postgresql-initdb: dolphinscheduler-postgresql-initdb:
dolphinscheduler-zookeeper: dolphinscheduler-zookeeper:
dolphinscheduler-worker-data: dolphinscheduler-worker-data:
dolphinscheduler-logs:
configs: dolphinscheduler-resource-local:
dolphinscheduler-worker-task-env:
file: ./dolphinscheduler_env.sh

84
docker/docker-swarm/docker-stack.yml

@ -20,13 +20,13 @@ services:
dolphinscheduler-postgresql: dolphinscheduler-postgresql:
image: bitnami/postgresql:latest image: bitnami/postgresql:latest
ports:
- 5432:5432
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
POSTGRESQL_USERNAME: root POSTGRESQL_USERNAME: root
POSTGRESQL_PASSWORD: root POSTGRESQL_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler POSTGRESQL_DATABASE: dolphinscheduler
ports:
- 5432:5432
volumes: volumes:
- dolphinscheduler-postgresql:/bitnami/postgresql - dolphinscheduler-postgresql:/bitnami/postgresql
networks: networks:
@ -37,12 +37,12 @@ services:
dolphinscheduler-zookeeper: dolphinscheduler-zookeeper:
image: bitnami/zookeeper:latest image: bitnami/zookeeper:latest
ports:
- 2181:2181
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
ALLOW_ANONYMOUS_LOGIN: "yes" ALLOW_ANONYMOUS_LOGIN: "yes"
ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
ports:
- 2181:2181
volumes: volumes:
- dolphinscheduler-zookeeper:/bitnami/zookeeper - dolphinscheduler-zookeeper:/bitnami/zookeeper
networks: networks:
@ -53,7 +53,9 @@ services:
dolphinscheduler-api: dolphinscheduler-api:
image: apache/dolphinscheduler:latest image: apache/dolphinscheduler:latest
command: ["api-server"] command: api-server
ports:
- 12345:12345
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
DATABASE_HOST: dolphinscheduler-postgresql DATABASE_HOST: dolphinscheduler-postgresql
@ -62,39 +64,17 @@ services:
DATABASE_PASSWORD: root DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
ports: RESOURCE_STORAGE_TYPE: HDFS
- 12345:12345 RESOURCE_UPLOAD_PATH: /dolphinscheduler
FS_DEFAULT_FS: file:///
healthcheck: healthcheck:
test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
interval: 30 interval: 30s
timeout: 5s timeout: 5s
retries: 3 retries: 3
start_period: 30s start_period: 30s
volumes: volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks:
- dolphinscheduler
deploy:
mode: replicated
replicas: 1
dolphinscheduler-frontend:
image: apache/dolphinscheduler:latest
command: ["frontend"]
ports:
- 8888:8888
environment:
TZ: Asia/Shanghai
FRONTEND_API_SERVER_HOST: dolphinscheduler-api
FRONTEND_API_SERVER_PORT: 12345
healthcheck:
test: ["CMD", "nc", "-z", "localhost", "8888"]
interval: 30
timeout: 5s
retries: 3
start_period: 30s
volumes:
- dolphinscheduler-logs:/var/log/nginx
networks: networks:
- dolphinscheduler - dolphinscheduler
deploy: deploy:
@ -103,7 +83,7 @@ services:
dolphinscheduler-alert: dolphinscheduler-alert:
image: apache/dolphinscheduler:latest image: apache/dolphinscheduler:latest
command: ["alert-server"] command: alert-server
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
XLS_FILE_PATH: "/tmp/xls" XLS_FILE_PATH: "/tmp/xls"
@ -127,13 +107,13 @@ services:
DATABASE_DATABASE: dolphinscheduler DATABASE_DATABASE: dolphinscheduler
healthcheck: healthcheck:
test: ["CMD", "/root/checkpoint.sh", "AlertServer"] test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
interval: 30 interval: 30s
timeout: 5s timeout: 5s
retries: 3 retries: 3
start_period: 30s start_period: 30s
volumes: volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks: networks:
- dolphinscheduler - dolphinscheduler
deploy: deploy:
mode: replicated mode: replicated
@ -141,10 +121,10 @@ services:
dolphinscheduler-master: dolphinscheduler-master:
image: apache/dolphinscheduler:latest image: apache/dolphinscheduler:latest
command: ["master-server"] command: master-server
ports: ports:
- 5678:5678 - 5678:5678
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
MASTER_EXEC_THREADS: "100" MASTER_EXEC_THREADS: "100"
MASTER_EXEC_TASK_NUM: "20" MASTER_EXEC_TASK_NUM: "20"
@ -161,7 +141,7 @@ services:
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck: healthcheck:
test: ["CMD", "/root/checkpoint.sh", "MasterServer"] test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
interval: 30 interval: 30s
timeout: 5s timeout: 5s
retries: 3 retries: 3
start_period: 30s start_period: 30s
@ -175,11 +155,11 @@ services:
dolphinscheduler-worker: dolphinscheduler-worker:
image: apache/dolphinscheduler:latest image: apache/dolphinscheduler:latest
command: ["worker-server"] command: worker-server
ports: ports:
- 1234:1234 - 1234:1234
- 50051:50051 - 50051:50051
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
WORKER_EXEC_THREADS: "100" WORKER_EXEC_THREADS: "100"
WORKER_HEARTBEAT_INTERVAL: "10" WORKER_HEARTBEAT_INTERVAL: "10"
@ -188,25 +168,37 @@ services:
WORKER_RESERVED_MEMORY: "0.1" WORKER_RESERVED_MEMORY: "0.1"
WORKER_GROUP: "default" WORKER_GROUP: "default"
WORKER_WEIGHT: "100" WORKER_WEIGHT: "100"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler
XLS_FILE_PATH: "/tmp/xls"
MAIL_SERVER_HOST: ""
MAIL_SERVER_PORT: ""
MAIL_SENDER: ""
MAIL_USER: ""
MAIL_PASSWD: ""
MAIL_SMTP_STARTTLS_ENABLE: "false"
MAIL_SMTP_SSL_ENABLE: "false"
MAIL_SMTP_SSL_TRUST: ""
DATABASE_HOST: dolphinscheduler-postgresql DATABASE_HOST: dolphinscheduler-postgresql
DATABASE_PORT: 5432 DATABASE_PORT: 5432
DATABASE_USERNAME: root DATABASE_USERNAME: root
DATABASE_PASSWORD: root DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
RESOURCE_STORAGE_TYPE: HDFS
RESOURCE_UPLOAD_PATH: /dolphinscheduler
FS_DEFAULT_FS: file:///
healthcheck: healthcheck:
test: ["CMD", "/root/checkpoint.sh", "WorkerServer"] test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]
interval: 30 interval: 30s
timeout: 5s timeout: 5s
retries: 3 retries: 3
start_period: 30s start_period: 30s
volumes:
- dolphinscheduler-worker-data:/tmp/dolphinscheduler
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
configs: configs:
- source: dolphinscheduler-worker-task-env - source: dolphinscheduler-worker-task-env
target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh
volumes:
- dolphinscheduler-worker-data:/tmp/dolphinscheduler
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks: networks:
- dolphinscheduler - dolphinscheduler
deploy: deploy:

12
docker/docker-swarm/dolphinscheduler_env.sh

@ -15,6 +15,14 @@
# limitations under the License. # limitations under the License.
# #
export PYTHON_HOME=/usr/bin/python2 export HADOOP_HOME=/opt/soft/hadoop
export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
export SPARK_HOME1=/opt/soft/spark1
export SPARK_HOME2=/opt/soft/spark2
export PYTHON_HOME=/usr/bin/python
export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk
export PATH=$PYTHON_HOME:$JAVA_HOME/bin:$PATH export HIVE_HOME=/opt/soft/hive
export FLINK_HOME=/opt/soft/flink
export DATAX_HOME=/opt/soft/datax/bin/datax.py
export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH

10
docker/kubernetes/dolphinscheduler/Chart.yaml

@ -22,7 +22,7 @@ home: https://dolphinscheduler.apache.org
icon: https://dolphinscheduler.apache.org/img/hlogo_colorful.svg icon: https://dolphinscheduler.apache.org/img/hlogo_colorful.svg
keywords: keywords:
- dolphinscheduler - dolphinscheduler
- Scheduler - scheduler
# A chart can be either an 'application' or a 'library' chart. # A chart can be either an 'application' or a 'library' chart.
# #
# Application charts are a collection of templates that can be packaged into versioned archives # Application charts are a collection of templates that can be packaged into versioned archives
@ -35,18 +35,18 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes # This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version. # to the chart and its templates, including the app version.
version: 1.0.0 version: 1.2.0
# This is the version number of the application being deployed. This version number should be # This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. # incremented each time you make changes to the application.
appVersion: 1.3.0 appVersion: 1.4.0
dependencies: dependencies:
- name: postgresql - name: postgresql
version: 8.x.x version: 10.x.x
repository: https://charts.bitnami.com/bitnami repository: https://charts.bitnami.com/bitnami
condition: postgresql.enabled condition: postgresql.enabled
- name: zookeeper - name: zookeeper
version: 5.x.x version: 6.x.x
repository: https://charts.bitnami.com/bitnami repository: https://charts.bitnami.com/bitnami
condition: zookeeper.enabled condition: zookeeper.enabled

37
docker/kubernetes/dolphinscheduler/README.md

@ -7,19 +7,20 @@ This chart bootstraps a [Dolphin Scheduler](https://dolphinscheduler.apache.org)
## Prerequisites ## Prerequisites
- Kubernetes 1.10+ - Helm 3.1.0+
- Kubernetes 1.12+
- PV provisioner support in the underlying infrastructure - PV provisioner support in the underlying infrastructure
## Installing the Chart ## Installing the Chart
To install the chart with the release name `my-release`: To install the chart with the release name `dolphinscheduler`:
```bash ```bash
$ git clone https://github.com/apache/incubator-dolphinscheduler.git $ git clone https://github.com/apache/incubator-dolphinscheduler.git
$ cd incubator-dolphinscheduler/kubernetes/dolphinscheduler $ cd incubator-dolphinscheduler/docker/kubernetes/dolphinscheduler
$ helm repo add bitnami https://charts.bitnami.com/bitnami $ helm repo add bitnami https://charts.bitnami.com/bitnami
$ helm dependency update . $ helm dependency update .
$ helm install --name dolphinscheduler . $ helm install dolphinscheduler .
``` ```
These commands deploy Dolphin Scheduler on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. These commands deploy Dolphin Scheduler on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation.
@ -30,7 +31,7 @@ These commands deploy Dolphin Scheduler on the Kubernetes cluster in the default
To uninstall/delete the `dolphinscheduler` deployment: To uninstall/delete the `dolphinscheduler` deployment:
```bash ```bash
$ helm delete --purge dolphinscheduler $ helm uninstall dolphinscheduler
``` ```
The command removes all the Kubernetes components associated with the chart and deletes the release. The command removes all the Kubernetes components associated with the chart and deletes the release.
@ -220,32 +221,6 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `api.persistentVolumeClaim.storageClassName` | `api` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | | `api.persistentVolumeClaim.storageClassName` | `api` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `api.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` | | `api.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` |
| | | | | | | |
| `frontend.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` |
| `frontend.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` |
| `frontend.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` |
| `frontend.replicas` | Replicas is the desired number of replicas of the given Template | `1` |
| `frontend.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
| `frontend.tolerations` | If specified, the pod's tolerations | `{}` |
| `frontend.affinity` | If specified, the pod's scheduling constraints | `{}` |
| `frontend.resources` | The `resource` limit and request config for frontend server. | `{}` |
| `frontend.annotations` | The `annotations` for frontend server. | `{}` |
| `frontend.livenessProbe.enabled` | Turn on and off liveness probe | `true` |
| `frontend.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` |
| `frontend.livenessProbe.periodSeconds` | How often to perform the probe | `30` |
| `frontend.livenessProbe.timeoutSeconds` | When the probe times out | `5` |
| `frontend.livenessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
| `frontend.livenessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
| `frontend.readinessProbe.enabled` | Turn on and off readiness probe | `true` |
| `frontend.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` |
| `frontend.readinessProbe.periodSeconds` | How often to perform the probe | `30` |
| `frontend.readinessProbe.timeoutSeconds` | When the probe times out | `5` |
| `frontend.readinessProbe.failureThreshold` | Minimum consecutive successes for the probe | `3` |
| `frontend.readinessProbe.successThreshold` | Minimum consecutive failures for the probe | `1` |
| `frontend.persistentVolumeClaim.enabled` | Set `frontend.persistentVolumeClaim.enabled` to `true` to mount a new volume for `frontend` | `false` |
| `frontend.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` |
| `frontend.persistentVolumeClaim.storageClassName` | `frontend` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `frontend.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` |
| | | |
| `ingress.enabled` | Enable ingress | `false` | | `ingress.enabled` | Enable ingress | `false` |
| `ingress.host` | Ingress host | `dolphinscheduler.org` | | `ingress.host` | Ingress host | `dolphinscheduler.org` |
| `ingress.path` | Ingress path | `/` | | `ingress.path` | Ingress path | `/` |

25
docker/kubernetes/dolphinscheduler/requirements.yaml

@ -1,25 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
dependencies:
- name: postgresql
version: 8.x.x
repository: https://charts.bitnami.com/bitnami
condition: postgresql.enabled
- name: zookeeper
version: 5.x.x
repository: https://charts.bitnami.com/bitnami
condition: redis.enabled

25
docker/kubernetes/dolphinscheduler/requirements.yaml~HEAD

@ -1,25 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
dependencies:
- name: postgresql
version: 8.x.x
repository: https://charts.bitnami.com/bitnami
condition: postgresql.enabled
- name: zookeeper
version: 5.x.x
repository: https://charts.bitnami.com/bitnami
condition: redis.enabled

25
docker/kubernetes/dolphinscheduler/requirements.yaml~HEAD_0

@ -1,25 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
dependencies:
- name: postgresql
version: 8.x.x
repository: https://charts.bitnami.com/bitnami
condition: postgresql.enabled
- name: zookeeper
version: 5.x.x
repository: https://charts.bitnami.com/bitnami
condition: redis.enabled

25
docker/kubernetes/dolphinscheduler/requirements.yaml~dev

@ -1,25 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
dependencies:
- name: postgresql
version: 8.x.x
repository: https://charts.bitnami.com/bitnami
condition: postgresql.enabled
- name: zookeeper
version: 5.x.x
repository: https://charts.bitnami.com/bitnami
condition: redis.enabled

25
docker/kubernetes/dolphinscheduler/requirements.yaml~dev_0

@ -1,25 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
dependencies:
- name: postgresql
version: 8.x.x
repository: https://charts.bitnami.com/bitnami
condition: postgresql.enabled
- name: zookeeper
version: 5.x.x
repository: https://charts.bitnami.com/bitnami
condition: redis.enabled

6
docker/kubernetes/dolphinscheduler/templates/NOTES.txt

@ -15,9 +15,9 @@
# limitations under the License. # limitations under the License.
# #
** Please be patient while the chart is being deployed ** ** Please be patient while the chart Dolphinscheduler {{ .Chart.AppVersion }} is being deployed **
1. Get the Dolphinscheduler URL by running: Get the Dolphinscheduler URL by running:
{{- if .Values.ingress.enabled }} {{- if .Values.ingress.enabled }}
@ -26,6 +26,6 @@
{{- else }} {{- else }}
kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-frontend 8888:8888 kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-api 12345:12345
{{- end }} {{- end }}

4
docker/kubernetes/dolphinscheduler/templates/_helpers.tpl

@ -135,7 +135,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
Create a default dolphinscheduler worker base dir. Create a default dolphinscheduler worker base dir.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
*/}} */}}
{{- define "dolphinscheduler.worker.base.dir" -}} {{- define "dolphinscheduler.data.basedir.path" -}}
{{- $name := default "/tmp/dolphinscheduler" .Values.worker.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH -}} {{- $name := default "/tmp/dolphinscheduler" .Values.common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH -}}
{{- printf "%s" $name | trunc 63 | trimSuffix "/" -}} {{- printf "%s" $name | trunc 63 | trimSuffix "/" -}}
{{- end -}} {{- end -}}

1
docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml

@ -24,6 +24,7 @@ metadata:
app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/managed-by: {{ .Release.Service }}
data: data:
DOLPHINSCHEDULER_OPTS: {{ .Values.alert.configmap.DOLPHINSCHEDULER_OPTS | quote }}
ALERT_PLUGIN_DIR: {{ .Values.alert.configmap.ALERT_PLUGIN_DIR | quote }} ALERT_PLUGIN_DIR: {{ .Values.alert.configmap.ALERT_PLUGIN_DIR | quote }}
XLS_FILE_PATH: {{ .Values.alert.configmap.XLS_FILE_PATH | quote }} XLS_FILE_PATH: {{ .Values.alert.configmap.XLS_FILE_PATH | quote }}
MAIL_SERVER_HOST: {{ .Values.alert.configmap.MAIL_SERVER_HOST | quote }} MAIL_SERVER_HOST: {{ .Values.alert.configmap.MAIL_SERVER_HOST | quote }}

21
docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-frontend.yaml → docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml

@ -14,22 +14,15 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# #
{{- if .Values.api.configmap }}
apiVersion: v1 apiVersion: v1
kind: Service kind: ConfigMap
metadata: metadata:
name: {{ include "dolphinscheduler.fullname" . }}-frontend name: {{ include "dolphinscheduler.fullname" . }}-api
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api
app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/managed-by: {{ .Release.Service }}
spec: data:
ports: DOLPHINSCHEDULER_OPTS: {{ .Values.api.configmap.DOLPHINSCHEDULER_OPTS | quote }}
- port: 8888 {{- end }}
targetPort: tcp-port
protocol: TCP
name: tcp-port
selector:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: frontend

8
docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml

@ -24,12 +24,14 @@ metadata:
app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/managed-by: {{ .Release.Service }}
data: data:
DOLPHINSCHEDULER_ENV_PATH: {{ .Values.common.configmap.DOLPHINSCHEDULER_ENV_PATH | quote }} DOLPHINSCHEDULER_ENV: |-
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ .Values.common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH | quote }} {{- range .Values.common.configmap.DOLPHINSCHEDULER_ENV }}
{{ . }}
{{- end }}
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ include "dolphinscheduler.data.basedir.path" . | quote }}
RESOURCE_STORAGE_TYPE: {{ .Values.common.configmap.RESOURCE_STORAGE_TYPE | quote }} RESOURCE_STORAGE_TYPE: {{ .Values.common.configmap.RESOURCE_STORAGE_TYPE | quote }}
RESOURCE_UPLOAD_PATH: {{ .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }} RESOURCE_UPLOAD_PATH: {{ .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }}
FS_DEFAULT_FS: {{ .Values.common.configmap.FS_DEFAULT_FS | quote }} FS_DEFAULT_FS: {{ .Values.common.configmap.FS_DEFAULT_FS | quote }}
FS_S3A_ENDPOINT: {{ .Values.common.configmap.FS_S3A_ENDPOINT | quote }} FS_S3A_ENDPOINT: {{ .Values.common.configmap.FS_S3A_ENDPOINT | quote }}
FS_S3A_ACCESS_KEY: {{ .Values.common.configmap.FS_S3A_ACCESS_KEY | quote }} FS_S3A_ACCESS_KEY: {{ .Values.common.configmap.FS_S3A_ACCESS_KEY | quote }}
FS_S3A_SECRET_KEY: {{ .Values.common.configmap.FS_S3A_SECRET_KEY | quote }}
{{- end }} {{- end }}

2
docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml

@ -24,6 +24,7 @@ metadata:
app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/managed-by: {{ .Release.Service }}
data: data:
DOLPHINSCHEDULER_OPTS: {{ .Values.master.configmap.DOLPHINSCHEDULER_OPTS | quote }}
MASTER_EXEC_THREADS: {{ .Values.master.configmap.MASTER_EXEC_THREADS | quote }} MASTER_EXEC_THREADS: {{ .Values.master.configmap.MASTER_EXEC_THREADS | quote }}
MASTER_EXEC_TASK_NUM: {{ .Values.master.configmap.MASTER_EXEC_TASK_NUM | quote }} MASTER_EXEC_TASK_NUM: {{ .Values.master.configmap.MASTER_EXEC_TASK_NUM | quote }}
MASTER_HEARTBEAT_INTERVAL: {{ .Values.master.configmap.MASTER_HEARTBEAT_INTERVAL | quote }} MASTER_HEARTBEAT_INTERVAL: {{ .Values.master.configmap.MASTER_HEARTBEAT_INTERVAL | quote }}
@ -32,5 +33,4 @@ data:
MASTER_MAX_CPULOAD_AVG: {{ .Values.master.configmap.MASTER_MAX_CPULOAD_AVG | quote }} MASTER_MAX_CPULOAD_AVG: {{ .Values.master.configmap.MASTER_MAX_CPULOAD_AVG | quote }}
MASTER_RESERVED_MEMORY: {{ .Values.master.configmap.MASTER_RESERVED_MEMORY | quote }} MASTER_RESERVED_MEMORY: {{ .Values.master.configmap.MASTER_RESERVED_MEMORY | quote }}
MASTER_LISTEN_PORT: {{ .Values.master.configmap.MASTER_LISTEN_PORT | quote }} MASTER_LISTEN_PORT: {{ .Values.master.configmap.MASTER_LISTEN_PORT | quote }}
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ include "dolphinscheduler.worker.base.dir" . | quote }}
{{- end }} {{- end }}

7
docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml

@ -24,17 +24,12 @@ metadata:
app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/managed-by: {{ .Release.Service }}
data: data:
DOLPHINSCHEDULER_OPTS: {{ .Values.worker.configmap.DOLPHINSCHEDULER_OPTS | quote }}
WORKER_EXEC_THREADS: {{ .Values.worker.configmap.WORKER_EXEC_THREADS | quote }} WORKER_EXEC_THREADS: {{ .Values.worker.configmap.WORKER_EXEC_THREADS | quote }}
WORKER_HEARTBEAT_INTERVAL: {{ .Values.worker.configmap.WORKER_HEARTBEAT_INTERVAL | quote }} WORKER_HEARTBEAT_INTERVAL: {{ .Values.worker.configmap.WORKER_HEARTBEAT_INTERVAL | quote }}
WORKER_FETCH_TASK_NUM: {{ .Values.worker.configmap.WORKER_FETCH_TASK_NUM | quote }}
WORKER_MAX_CPULOAD_AVG: {{ .Values.worker.configmap.WORKER_MAX_CPULOAD_AVG | quote }} WORKER_MAX_CPULOAD_AVG: {{ .Values.worker.configmap.WORKER_MAX_CPULOAD_AVG | quote }}
WORKER_RESERVED_MEMORY: {{ .Values.worker.configmap.WORKER_RESERVED_MEMORY | quote }} WORKER_RESERVED_MEMORY: {{ .Values.worker.configmap.WORKER_RESERVED_MEMORY | quote }}
WORKER_LISTEN_PORT: {{ .Values.worker.configmap.WORKER_LISTEN_PORT | quote }} WORKER_LISTEN_PORT: {{ .Values.worker.configmap.WORKER_LISTEN_PORT | quote }}
WORKER_GROUP: {{ .Values.worker.configmap.WORKER_GROUP | quote }} WORKER_GROUP: {{ .Values.worker.configmap.WORKER_GROUP | quote }}
WORKER_WEIGHT: {{ .Values.worker.configmap.WORKER_WEIGHT | quote }} WORKER_WEIGHT: {{ .Values.worker.configmap.WORKER_WEIGHT | quote }}
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ include "dolphinscheduler.worker.base.dir" . | quote }}
dolphinscheduler_env.sh: |-
{{- range .Values.worker.configmap.DOLPHINSCHEDULER_ENV }}
{{ . }}
{{- end }}
{{- end }} {{- end }}

68
docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml

@ -57,35 +57,6 @@ spec:
{{- if .Values.alert.tolerations }} {{- if .Values.alert.tolerations }}
tolerations: {{- toYaml . | nindent 8 }} tolerations: {{- toYaml . | nindent 8 }}
{{- end }} {{- end }}
initContainers:
- name: init-database
image: busybox:1.31.0
command:
- /bin/sh
- -ec
- |
while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
counter=$((counter+1))
if [ $counter == 5 ]; then
echo "Error: Couldn't connect to database."
exit 1
fi
echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
sleep 60
done
env:
- name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
{{- if .Values.image.pullSecrets }} {{- if .Values.image.pullSecrets }}
imagePullSecrets: imagePullSecrets:
- name: {{ include "dolphinscheduler.image.pullSecrets" . }} - name: {{ include "dolphinscheduler.image.pullSecrets" . }}
@ -93,14 +64,17 @@ spec:
containers: containers:
- name: {{ include "dolphinscheduler.fullname" . }}-alert - name: {{ include "dolphinscheduler.fullname" . }}-alert
image: {{ include "dolphinscheduler.image.repository" . | quote }} image: {{ include "dolphinscheduler.image.repository" . | quote }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
args: args:
- "alert-server" - "alert-server"
imagePullPolicy: {{ .Values.image.pullPolicy }}
env: env:
- name: DOLPHINSCHEDULER_OPTS
value: {{ default "" .Values.alert.jvmOptions }}
- name: TZ - name: TZ
value: {{ .Values.timezone }} value: {{ .Values.timezone }}
- name: DOLPHINSCHEDULER_OPTS
valueFrom:
configMapKeyRef:
key: DOLPHINSCHEDULER_OPTS
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: ALERT_PLUGIN_DIR - name: ALERT_PLUGIN_DIR
valueFrom: valueFrom:
configMapKeyRef: configMapKeyRef:
@ -228,36 +202,6 @@ spec:
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.params | quote }} value: {{ .Values.externalDatabase.params | quote }}
{{- end }} {{- end }}
- name: RESOURCE_STORAGE_TYPE
valueFrom:
configMapKeyRef:
key: RESOURCE_STORAGE_TYPE
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: RESOURCE_UPLOAD_PATH
valueFrom:
configMapKeyRef:
key: RESOURCE_UPLOAD_PATH
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_DEFAULT_FS
valueFrom:
configMapKeyRef:
key: FS_DEFAULT_FS
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_ENDPOINT
valueFrom:
configMapKeyRef:
key: FS_S3A_ENDPOINT
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_ACCESS_KEY
valueFrom:
configMapKeyRef:
key: FS_S3A_ACCESS_KEY
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_SECRET_KEY
valueFrom:
configMapKeyRef:
key: FS_S3A_SECRET_KEY
name: {{ include "dolphinscheduler.fullname" . }}-common
{{- if .Values.alert.resources }} {{- if .Values.alert.resources }}
resources: resources:
limits: limits:

48
docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml

@ -57,35 +57,6 @@ spec:
{{- if .Values.api.tolerations }} {{- if .Values.api.tolerations }}
tolerations: {{- toYaml . | nindent 8 }} tolerations: {{- toYaml . | nindent 8 }}
{{- end }} {{- end }}
initContainers:
- name: init-database
image: busybox:1.31.0
command:
- /bin/sh
- -ec
- |
while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
counter=$((counter+1))
if [ $counter == 5 ]; then
echo "Error: Couldn't connect to database."
exit 1
fi
echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
sleep 60
done
env:
- name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
{{- if .Values.image.pullSecrets }} {{- if .Values.image.pullSecrets }}
imagePullSecrets: imagePullSecrets:
- name: {{ include "dolphinscheduler.image.pullSecrets" . }} - name: {{ include "dolphinscheduler.image.pullSecrets" . }}
@ -93,17 +64,20 @@ spec:
containers: containers:
- name: {{ include "dolphinscheduler.fullname" . }}-api - name: {{ include "dolphinscheduler.fullname" . }}-api
image: {{ include "dolphinscheduler.image.repository" . | quote }} image: {{ include "dolphinscheduler.image.repository" . | quote }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
args: args:
- "api-server" - "api-server"
ports: ports:
- containerPort: 12345 - containerPort: 12345
name: tcp-port name: tcp-port
imagePullPolicy: {{ .Values.image.pullPolicy }}
env: env:
- name: DOLPHINSCHEDULER_OPTS
value: {{ default "" .Values.api.jvmOptions }}
- name: TZ - name: TZ
value: {{ .Values.timezone }} value: {{ .Values.timezone }}
- name: DOLPHINSCHEDULER_OPTS
valueFrom:
configMapKeyRef:
key: DOLPHINSCHEDULER_OPTS
name: {{ include "dolphinscheduler.fullname" . }}-api
- name: DATABASE_TYPE - name: DATABASE_TYPE
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: "postgresql" value: "postgresql"
@ -164,7 +138,7 @@ spec:
{{- end }} {{- end }}
- name: ZOOKEEPER_ROOT - name: ZOOKEEPER_ROOT
{{- if .Values.zookeeper.enabled }} {{- if .Values.zookeeper.enabled }}
value: "/dolphinscheduler" value: {{ .Values.zookeeper.zookeeperRoot }}
{{- else }} {{- else }}
value: {{ .Values.externalZookeeper.zookeeperRoot }} value: {{ .Values.externalZookeeper.zookeeperRoot }}
{{- end }} {{- end }}
@ -183,6 +157,7 @@ spec:
configMapKeyRef: configMapKeyRef:
key: FS_DEFAULT_FS key: FS_DEFAULT_FS
name: {{ include "dolphinscheduler.fullname" . }}-common name: {{ include "dolphinscheduler.fullname" . }}-common
{{- if eq .Values.common.configmap.RESOURCE_STORAGE_TYPE "S3" }}
- name: FS_S3A_ENDPOINT - name: FS_S3A_ENDPOINT
valueFrom: valueFrom:
configMapKeyRef: configMapKeyRef:
@ -195,9 +170,10 @@ spec:
name: {{ include "dolphinscheduler.fullname" . }}-common name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_SECRET_KEY - name: FS_S3A_SECRET_KEY
valueFrom: valueFrom:
configMapKeyRef: secretKeyRef:
key: FS_S3A_SECRET_KEY key: fs-s3a-secret-key
name: {{ include "dolphinscheduler.fullname" . }}-common name: {{ printf "%s-%s" .Release.Name "fs-s3a" }}
{{- end }}
{{- if .Values.api.resources }} {{- if .Values.api.resources }}
resources: resources:
limits: limits:

119
docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml

@ -1,119 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "dolphinscheduler.fullname" . }}-frontend
labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: frontend
spec:
replicas: {{ .Values.frontend.replicas }}
selector:
matchLabels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: frontend
strategy:
type: {{ .Values.frontend.strategy.type | quote }}
rollingUpdate:
maxSurge: {{ .Values.frontend.strategy.rollingUpdate.maxSurge | quote }}
maxUnavailable: {{ .Values.frontend.strategy.rollingUpdate.maxUnavailable | quote }}
template:
metadata:
labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: frontend
{{- if .Values.alert.annotations }}
annotations:
{{- toYaml .Values.alert.annotations | nindent 8 }}
{{- end }}
spec:
{{- if .Values.frontend.affinity }}
affinity: {{- toYaml .Values.frontend.affinity | nindent 8 }}
{{- end }}
{{- if .Values.frontend.nodeSelector }}
nodeSelector: {{- toYaml .Values.frontend.nodeSelector | nindent 8 }}
{{- end }}
{{- if .Values.frontend.tolerations }}
tolerations: {{- toYaml . | nindent 8 }}
{{- end }}
{{- if .Values.image.pullSecrets }}
imagePullSecrets:
- name: {{ include "dolphinscheduler.image.pullSecrets" . }}
{{- end }}
containers:
- name: {{ include "dolphinscheduler.fullname" . }}-frontend
image: {{ include "dolphinscheduler.image.repository" . | quote }}
args:
- "frontend"
ports:
- containerPort: 8888
name: tcp-port
imagePullPolicy: {{ .Values.image.pullPolicy }}
env:
- name: TZ
value: {{ .Values.timezone }}
- name: FRONTEND_API_SERVER_HOST
value: '{{ include "dolphinscheduler.fullname" . }}-api'
- name: FRONTEND_API_SERVER_PORT
value: "12345"
{{- if .Values.frontend.resources }}
resources:
limits:
memory: {{ .Values.frontend.resources.limits.memory | quote }}
cpu: {{ .Values.frontend.resources.limits.cpu | quote }}
requests:
memory: {{ .Values.frontend.resources.requests.memory | quote }}
cpu: {{ .Values.frontend.resources.requests.cpu | quote }}
{{- end }}
{{- if .Values.frontend.livenessProbe.enabled }}
livenessProbe:
tcpSocket:
port: 8888
initialDelaySeconds: {{ .Values.frontend.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.frontend.livenessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.frontend.livenessProbe.timeoutSeconds }}
successThreshold: {{ .Values.frontend.livenessProbe.successThreshold }}
failureThreshold: {{ .Values.frontend.livenessProbe.failureThreshold }}
{{- end }}
{{- if .Values.frontend.readinessProbe.enabled }}
readinessProbe:
tcpSocket:
port: 8888
initialDelaySeconds: {{ .Values.frontend.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.frontend.readinessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.frontend.readinessProbe.timeoutSeconds }}
successThreshold: {{ .Values.frontend.readinessProbe.successThreshold }}
failureThreshold: {{ .Values.frontend.readinessProbe.failureThreshold }}
{{- end }}
volumeMounts:
- mountPath: "/var/log/nginx"
name: {{ include "dolphinscheduler.fullname" . }}-frontend
volumes:
- name: {{ include "dolphinscheduler.fullname" . }}-frontend
{{- if .Values.frontend.persistentVolumeClaim.enabled }}
persistentVolumeClaim:
claimName: {{ include "dolphinscheduler.fullname" . }}-frontend
{{- else }}
emptyDir: {}
{{- end }}

2
docker/kubernetes/dolphinscheduler/templates/ingress.yaml

@ -30,7 +30,7 @@ spec:
paths: paths:
- path: {{ .Values.ingress.path }} - path: {{ .Values.ingress.path }}
backend: backend:
serviceName: {{ include "dolphinscheduler.fullname" . }}-frontend serviceName: {{ include "dolphinscheduler.fullname" . }}-api
servicePort: tcp-port servicePort: tcp-port
{{- if .Values.ingress.tls.enabled }} {{- if .Values.ingress.tls.enabled }}
tls: tls:

2
docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-alert.yaml

@ -25,7 +25,7 @@ metadata:
app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/managed-by: {{ .Release.Service }}
spec: spec:
accessModes: accessModes:
{{- range .Values.alert.persistentVolumeClaim.accessModes }} {{- range .Values.alert.persistentVolumeClaim.accessModes }}
- {{ . | quote }} - {{ . | quote }}
{{- end }} {{- end }}
storageClassName: {{ .Values.alert.persistentVolumeClaim.storageClassName | quote }} storageClassName: {{ .Values.alert.persistentVolumeClaim.storageClassName | quote }}

2
docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-api.yaml

@ -25,7 +25,7 @@ metadata:
app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/managed-by: {{ .Release.Service }}
spec: spec:
accessModes: accessModes:
{{- range .Values.api.persistentVolumeClaim.accessModes }} {{- range .Values.api.persistentVolumeClaim.accessModes }}
- {{ . | quote }} - {{ . | quote }}
{{- end }} {{- end }}
storageClassName: {{ .Values.api.persistentVolumeClaim.storageClassName | quote }} storageClassName: {{ .Values.api.persistentVolumeClaim.storageClassName | quote }}

20
docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-frontend.yaml → docker/kubernetes/dolphinscheduler/templates/secret-external-fs-s3a.yaml

@ -14,22 +14,16 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# #
{{- if .Values.frontend.persistentVolumeClaim.enabled }} {{- if eq .Values.common.configmap.RESOURCE_STORAGE_TYPE "S3" }}
apiVersion: v1 apiVersion: v1
kind: PersistentVolumeClaim kind: Secret
metadata: metadata:
name: {{ include "dolphinscheduler.fullname" . }}-frontend name: {{ printf "%s-%s" .Release.Name "fs-s3a" }}
labels: labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-frontend app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-fs-s3a
app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/managed-by: {{ .Release.Service }}
spec: type: Opaque
accessModes: data:
{{- range .Values.frontend.persistentVolumeClaim.accessModes }} fs-s3a-secret-key: {{ .Values.common.configmap.FS_S3A_SECRET_KEY | b64enc | quote }}
- {{ . | quote }}
{{- end }}
storageClassName: {{ .Values.frontend.persistentVolumeClaim.storageClassName | quote }}
resources:
requests:
storage: {{ .Values.frontend.persistentVolumeClaim.storage | quote }}
{{- end }} {{- end }}

96
docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml

@ -54,59 +54,6 @@ spec:
{{- if .Values.master.tolerations }} {{- if .Values.master.tolerations }}
tolerations: {{- toYaml . | nindent 8 }} tolerations: {{- toYaml . | nindent 8 }}
{{- end }} {{- end }}
initContainers:
- name: init-zookeeper
image: busybox:1.31.0
command:
- /bin/sh
- -ec
- |
echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do
while ! nc -z ${line%:*} ${line#*:}; do
counter=$((counter+1))
if [ $counter == 5 ]; then
echo "Error: Couldn't connect to zookeeper."
exit 1
fi
echo "Trying to connect to zookeeper at ${line}. Attempt $counter."
sleep 60
done
done
env:
- name: ZOOKEEPER_QUORUM
{{- if .Values.zookeeper.enabled }}
value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}"
{{- else }}
value: {{ .Values.externalZookeeper.zookeeperQuorum }}
{{- end }}
- name: init-database
image: busybox:1.31.0
command:
- /bin/sh
- -ec
- |
while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
counter=$((counter+1))
if [ $counter == 5 ]; then
echo "Error: Couldn't connect to database."
exit 1
fi
echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
sleep 60
done
env:
- name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
{{- if .Values.image.pullSecrets }} {{- if .Values.image.pullSecrets }}
imagePullSecrets: imagePullSecrets:
- name: {{ include "dolphinscheduler.image.pullSecrets" . }} - name: {{ include "dolphinscheduler.image.pullSecrets" . }}
@ -114,17 +61,20 @@ spec:
containers: containers:
- name: {{ include "dolphinscheduler.fullname" . }}-master - name: {{ include "dolphinscheduler.fullname" . }}-master
image: {{ include "dolphinscheduler.image.repository" . | quote }} image: {{ include "dolphinscheduler.image.repository" . | quote }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
args: args:
- "master-server" - "master-server"
ports: ports:
- containerPort: {{ .Values.master.configmap.MASTER_LISTEN_PORT }} - containerPort: {{ .Values.master.configmap.MASTER_LISTEN_PORT }}
name: "master-port" name: "master-port"
imagePullPolicy: {{ .Values.image.pullPolicy }}
env: env:
- name: DOLPHINSCHEDULER_OPTS
value: {{ default "" .Values.master.jvmOptions }}
- name: TZ - name: TZ
value: {{ .Values.timezone }} value: {{ .Values.timezone }}
- name: DOLPHINSCHEDULER_OPTS
valueFrom:
configMapKeyRef:
key: DOLPHINSCHEDULER_OPTS
name: {{ include "dolphinscheduler.fullname" . }}-master
- name: MASTER_EXEC_THREADS - name: MASTER_EXEC_THREADS
valueFrom: valueFrom:
configMapKeyRef: configMapKeyRef:
@ -168,7 +118,7 @@ spec:
- name: DOLPHINSCHEDULER_DATA_BASEDIR_PATH - name: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
valueFrom: valueFrom:
configMapKeyRef: configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-master name: {{ include "dolphinscheduler.fullname" . }}-common
key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
- name: DATABASE_TYPE - name: DATABASE_TYPE
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
@ -230,40 +180,10 @@ spec:
{{- end }} {{- end }}
- name: ZOOKEEPER_ROOT - name: ZOOKEEPER_ROOT
{{- if .Values.zookeeper.enabled }} {{- if .Values.zookeeper.enabled }}
value: "/dolphinscheduler" value: {{ .Values.zookeeper.zookeeperRoot }}
{{- else }} {{- else }}
value: {{ .Values.externalZookeeper.zookeeperRoot }} value: {{ .Values.externalZookeeper.zookeeperRoot }}
{{- end }} {{- end }}
- name: RESOURCE_STORAGE_TYPE
valueFrom:
configMapKeyRef:
key: RESOURCE_STORAGE_TYPE
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: RESOURCE_UPLOAD_PATH
valueFrom:
configMapKeyRef:
key: RESOURCE_UPLOAD_PATH
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_DEFAULT_FS
valueFrom:
configMapKeyRef:
key: FS_DEFAULT_FS
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_ENDPOINT
valueFrom:
configMapKeyRef:
key: FS_S3A_ENDPOINT
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_ACCESS_KEY
valueFrom:
configMapKeyRef:
key: FS_S3A_ACCESS_KEY
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_SECRET_KEY
valueFrom:
configMapKeyRef:
key: FS_S3A_SECRET_KEY
name: {{ include "dolphinscheduler.fullname" . }}-common
{{- if .Values.master.resources }} {{- if .Values.master.resources }}
resources: resources:
limits: limits:

184
docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml

@ -54,59 +54,6 @@ spec:
{{- if .Values.worker.tolerations }} {{- if .Values.worker.tolerations }}
tolerations: {{- toYaml . | nindent 8 }} tolerations: {{- toYaml . | nindent 8 }}
{{- end }} {{- end }}
initContainers:
- name: init-zookeeper
image: busybox:1.31.0
command:
- /bin/sh
- -ec
- |
echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do
while ! nc -z ${line%:*} ${line#*:}; do
counter=$((counter+1))
if [ $counter == 5 ]; then
echo "Error: Couldn't connect to zookeeper."
exit 1
fi
echo "Trying to connect to zookeeper at ${line}. Attempt $counter."
sleep 60
done
done
env:
- name: ZOOKEEPER_QUORUM
{{- if .Values.zookeeper.enabled }}
value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}"
{{- else }}
value: {{ .Values.externalZookeeper.zookeeperQuorum }}
{{- end }}
- name: init-database
image: busybox:1.31.0
command:
- /bin/sh
- -ec
- |
while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
counter=$((counter+1))
if [ $counter == 5 ]; then
echo "Error: Couldn't connect to database."
exit 1
fi
echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
sleep 60
done
env:
- name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
{{- if .Values.image.pullSecrets }} {{- if .Values.image.pullSecrets }}
imagePullSecrets: imagePullSecrets:
- name: {{ include "dolphinscheduler.image.pullSecrets" . }} - name: {{ include "dolphinscheduler.image.pullSecrets" . }}
@ -114,29 +61,27 @@ spec:
containers: containers:
- name: {{ include "dolphinscheduler.fullname" . }}-worker - name: {{ include "dolphinscheduler.fullname" . }}-worker
image: {{ include "dolphinscheduler.image.repository" . | quote }} image: {{ include "dolphinscheduler.image.repository" . | quote }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
args: args:
- "worker-server" - "worker-server"
ports: ports:
- containerPort: {{ .Values.worker.configmap.WORKER_LISTEN_PORT }} - containerPort: {{ .Values.worker.configmap.WORKER_LISTEN_PORT }}
name: "worker-port" name: "worker-port"
- containerPort: 50051 - containerPort: 50051
name: "logs-port" name: "logger-port"
imagePullPolicy: {{ .Values.image.pullPolicy }}
env: env:
- name: DOLPHINSCHEDULER_OPTS
value: {{ default "" .Values.worker.jvmOptions }}
- name: TZ - name: TZ
value: {{ .Values.timezone }} value: {{ .Values.timezone }}
- name: WORKER_EXEC_THREADS - name: DOLPHINSCHEDULER_OPTS
valueFrom: valueFrom:
configMapKeyRef: configMapKeyRef:
key: DOLPHINSCHEDULER_OPTS
name: {{ include "dolphinscheduler.fullname" . }}-worker name: {{ include "dolphinscheduler.fullname" . }}-worker
key: WORKER_EXEC_THREADS - name: WORKER_EXEC_THREADS
- name: WORKER_FETCH_TASK_NUM
valueFrom: valueFrom:
configMapKeyRef: configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-worker name: {{ include "dolphinscheduler.fullname" . }}-worker
key: WORKER_FETCH_TASK_NUM key: WORKER_EXEC_THREADS
- name: WORKER_HEARTBEAT_INTERVAL - name: WORKER_HEARTBEAT_INTERVAL
valueFrom: valueFrom:
configMapKeyRef: configMapKeyRef:
@ -170,8 +115,58 @@ spec:
- name: DOLPHINSCHEDULER_DATA_BASEDIR_PATH - name: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
valueFrom: valueFrom:
configMapKeyRef: configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-master name: {{ include "dolphinscheduler.fullname" . }}-common
key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
- name: ALERT_PLUGIN_DIR
valueFrom:
configMapKeyRef:
key: ALERT_PLUGIN_DIR
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: XLS_FILE_PATH
valueFrom:
configMapKeyRef:
key: XLS_FILE_PATH
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SERVER_HOST
valueFrom:
configMapKeyRef:
key: MAIL_SERVER_HOST
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SERVER_PORT
valueFrom:
configMapKeyRef:
key: MAIL_SERVER_PORT
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SENDER
valueFrom:
configMapKeyRef:
key: MAIL_SENDER
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_USER
valueFrom:
configMapKeyRef:
key: MAIL_USER
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_PASSWD
valueFrom:
configMapKeyRef:
key: MAIL_PASSWD
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SMTP_STARTTLS_ENABLE
valueFrom:
configMapKeyRef:
key: MAIL_SMTP_STARTTLS_ENABLE
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SMTP_SSL_ENABLE
valueFrom:
configMapKeyRef:
key: MAIL_SMTP_SSL_ENABLE
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SMTP_SSL_TRUST
valueFrom:
configMapKeyRef:
key: MAIL_SMTP_SSL_TRUST
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: DATABASE_TYPE - name: DATABASE_TYPE
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: "postgresql" value: "postgresql"
@ -232,7 +227,7 @@ spec:
{{- end }} {{- end }}
- name: ZOOKEEPER_ROOT - name: ZOOKEEPER_ROOT
{{- if .Values.zookeeper.enabled }} {{- if .Values.zookeeper.enabled }}
value: "/dolphinscheduler" value: {{ .Values.zookeeper.zookeeperRoot }}
{{- else }} {{- else }}
value: {{ .Values.externalZookeeper.zookeeperRoot }} value: {{ .Values.externalZookeeper.zookeeperRoot }}
{{- end }} {{- end }}
@ -251,6 +246,7 @@ spec:
configMapKeyRef: configMapKeyRef:
key: FS_DEFAULT_FS key: FS_DEFAULT_FS
name: {{ include "dolphinscheduler.fullname" . }}-common name: {{ include "dolphinscheduler.fullname" . }}-common
{{- if eq .Values.common.configmap.RESOURCE_STORAGE_TYPE "S3" }}
- name: FS_S3A_ENDPOINT - name: FS_S3A_ENDPOINT
valueFrom: valueFrom:
configMapKeyRef: configMapKeyRef:
@ -263,54 +259,10 @@ spec:
name: {{ include "dolphinscheduler.fullname" . }}-common name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_SECRET_KEY - name: FS_S3A_SECRET_KEY
valueFrom: valueFrom:
configMapKeyRef: secretKeyRef:
key: FS_S3A_SECRET_KEY key: fs-s3a-secret-key
name: {{ include "dolphinscheduler.fullname" . }}-common name: {{ printf "%s-%s" .Release.Name "fs-s3a" }}
- name: XLS_FILE_PATH {{- end }}
valueFrom:
configMapKeyRef:
key: XLS_FILE_PATH
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SERVER_HOST
valueFrom:
configMapKeyRef:
key: MAIL_SERVER_HOST
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SERVER_PORT
valueFrom:
configMapKeyRef:
key: MAIL_SERVER_PORT
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SENDER
valueFrom:
configMapKeyRef:
key: MAIL_SENDER
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_USER
valueFrom:
configMapKeyRef:
key: MAIL_USER
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_PASSWD
valueFrom:
configMapKeyRef:
key: MAIL_PASSWD
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SMTP_STARTTLS_ENABLE
valueFrom:
configMapKeyRef:
key: MAIL_SMTP_STARTTLS_ENABLE
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SMTP_SSL_ENABLE
valueFrom:
configMapKeyRef:
key: MAIL_SMTP_SSL_ENABLE
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SMTP_SSL_TRUST
valueFrom:
configMapKeyRef:
key: MAIL_SMTP_SSL_TRUST
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: ENTERPRISE_WECHAT_ENABLE - name: ENTERPRISE_WECHAT_ENABLE
valueFrom: valueFrom:
configMapKeyRef: configMapKeyRef:
@ -372,13 +324,13 @@ spec:
failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }} failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }}
{{- end }} {{- end }}
volumeMounts: volumeMounts:
- mountPath: {{ include "dolphinscheduler.worker.base.dir" . | quote }} - mountPath: {{ include "dolphinscheduler.data.basedir.path" . | quote }}
name: {{ include "dolphinscheduler.fullname" . }}-worker-data name: {{ include "dolphinscheduler.fullname" . }}-worker-data
- mountPath: "/opt/dolphinscheduler/logs" - mountPath: "/opt/dolphinscheduler/logs"
name: {{ include "dolphinscheduler.fullname" . }}-worker-logs name: {{ include "dolphinscheduler.fullname" . }}-worker-logs
- mountPath: "/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh" - mountPath: "/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"
subPath: "dolphinscheduler_env.sh" subPath: "dolphinscheduler_env.sh"
name: {{ include "dolphinscheduler.fullname" . }}-worker-configmap name: {{ include "dolphinscheduler.fullname" . }}-common-env
volumes: volumes:
- name: {{ include "dolphinscheduler.fullname" . }}-worker-data - name: {{ include "dolphinscheduler.fullname" . }}-worker-data
{{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }} {{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }}
@ -394,12 +346,12 @@ spec:
{{- else }} {{- else }}
emptyDir: {} emptyDir: {}
{{- end }} {{- end }}
- name: {{ include "dolphinscheduler.fullname" . }}-worker-configmap - name: {{ include "dolphinscheduler.fullname" . }}-common-env
configMap: configMap:
defaultMode: 0777 defaultMode: 0777
name: {{ include "dolphinscheduler.fullname" . }}-worker name: {{ include "dolphinscheduler.fullname" . }}-common
items: items:
- key: dolphinscheduler_env.sh - key: DOLPHINSCHEDULER_ENV
path: dolphinscheduler_env.sh path: dolphinscheduler_env.sh
{{- if .Values.worker.persistentVolumeClaim.enabled }} {{- if .Values.worker.persistentVolumeClaim.enabled }}
volumeClaimTemplates: volumeClaimTemplates:

4
docker/kubernetes/dolphinscheduler/templates/svc-dolphinscheduler-worker-headless.yaml

@ -30,9 +30,9 @@ spec:
protocol: TCP protocol: TCP
name: worker-port name: worker-port
- port: 50051 - port: 50051
targetPort: logs-port targetPort: logger-port
protocol: TCP protocol: TCP
name: logs-port name: logger-port
selector: selector:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker
app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/instance: {{ .Release.Name }}

310
docker/kubernetes/dolphinscheduler/values.yaml

@ -58,62 +58,74 @@ externalDatabase:
# If not exists external zookeeper, by default, Dolphinscheduler's zookeeper will use it. # If not exists external zookeeper, by default, Dolphinscheduler's zookeeper will use it.
zookeeper: zookeeper:
enabled: true enabled: true
taskQueue: "zookeeper" fourlwCommandsWhitelist: srvr,ruok,wchs,cons
config: null
service: service:
port: "2181" port: "2181"
persistence: persistence:
enabled: false enabled: false
size: "20Gi" size: "20Gi"
storageClass: "-" storageClass: "-"
zookeeperRoot: "/dolphinscheduler"
# If exists external zookeeper, and set zookeeper.enable value to false. # If exists external zookeeper, and set zookeeper.enable value to false.
# If zookeeper.enable is false, Dolphinscheduler's zookeeper will use it. # If zookeeper.enable is false, Dolphinscheduler's zookeeper will use it.
externalZookeeper: externalZookeeper:
taskQueue: "zookeeper"
zookeeperQuorum: "127.0.0.1:2181" zookeeperQuorum: "127.0.0.1:2181"
zookeeperRoot: "/dolphinscheduler" zookeeperRoot: "/dolphinscheduler"
common: common:
## ConfigMap
configmap: configmap:
DOLPHINSCHEDULER_ENV_PATH: "/tmp/dolphinscheduler/env" DOLPHINSCHEDULER_ENV:
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler/files" - "export HADOOP_HOME=/opt/soft/hadoop"
RESOURCE_STORAGE_TYPE: "NONE" - "export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop"
RESOURCE_UPLOAD_PATH: "/ds" - "export SPARK_HOME1=/opt/soft/spark1"
FS_DEFAULT_FS: "s3a://xxxx" - "export SPARK_HOME2=/opt/soft/spark2"
- "export PYTHON_HOME=/usr/bin/python"
- "export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk"
- "export HIVE_HOME=/opt/soft/hive"
- "export FLINK_HOME=/opt/soft/flink"
- "export DATAX_HOME=/opt/soft/datax/bin/datax.py"
- "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
RESOURCE_STORAGE_TYPE: "HDFS"
RESOURCE_UPLOAD_PATH: "/dolphinscheduler"
FS_DEFAULT_FS: "file:///"
FS_S3A_ENDPOINT: "s3.xxx.amazonaws.com" FS_S3A_ENDPOINT: "s3.xxx.amazonaws.com"
FS_S3A_ACCESS_KEY: "xxxxxxx" FS_S3A_ACCESS_KEY: "xxxxxxx"
FS_S3A_SECRET_KEY: "xxxxxxx" FS_S3A_SECRET_KEY: "xxxxxxx"
master: master:
## PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down.
podManagementPolicy: "Parallel" podManagementPolicy: "Parallel"
## Replicas is the desired number of replicas of the given Template.
replicas: "3" replicas: "3"
# NodeSelector is a selector which must be true for the pod to fit on a node. ## You can use annotations to attach arbitrary non-identifying metadata to objects.
# Selector which must match a node's labels for the pod to be scheduled on that node. ## Clients such as tools and libraries can retrieve this metadata.
# More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ annotations: {}
## Affinity is a group of affinity scheduling rules. If specified, the pod's scheduling constraints.
## More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
affinity: {}
## NodeSelector is a selector which must be true for the pod to fit on a node.
## Selector which must match a node's labels for the pod to be scheduled on that node.
## More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
nodeSelector: {} nodeSelector: {}
# Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, ## Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission,
# effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. ## effectively unioning the set of nodes tolerated by the pod and the RuntimeClass.
tolerations: [] tolerations: []
# Affinity is a group of affinity scheduling rules. ## Compute Resources required by this container. Cannot be updated.
# If specified, the pod's scheduling constraints. ## More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container
# More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
affinity: {}
# The jvm options for java instance startup
jvmOptions: ""
resources: {} resources: {}
# limits: # resources:
# memory: "18Gi" # limits:
# cpu: "4" # memory: "8Gi"
# requests: # cpu: "4"
# memory: "2Gi" # requests:
# cpu: "500m" # memory: "2Gi"
# You can use annotations to attach arbitrary non-identifying metadata to objects. # cpu: "500m"
# Clients such as tools and libraries can retrieve this metadata. ## ConfigMap
annotations: {}
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
configmap: configmap:
DOLPHINSCHEDULER_OPTS: ""
MASTER_EXEC_THREADS: "100" MASTER_EXEC_THREADS: "100"
MASTER_EXEC_TASK_NUM: "20" MASTER_EXEC_TASK_NUM: "20"
MASTER_HEARTBEAT_INTERVAL: "10" MASTER_HEARTBEAT_INTERVAL: "10"
@ -122,6 +134,8 @@ master:
MASTER_MAX_CPULOAD_AVG: "100" MASTER_MAX_CPULOAD_AVG: "100"
MASTER_RESERVED_MEMORY: "0.1" MASTER_RESERVED_MEMORY: "0.1"
MASTER_LISTEN_PORT: "5678" MASTER_LISTEN_PORT: "5678"
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
livenessProbe: livenessProbe:
enabled: true enabled: true
initialDelaySeconds: "30" initialDelaySeconds: "30"
@ -138,7 +152,7 @@ master:
timeoutSeconds: "5" timeoutSeconds: "5"
failureThreshold: "3" failureThreshold: "3"
successThreshold: "1" successThreshold: "1"
## volumeClaimTemplates is a list of claims that pods are allowed to reference. ## PersistentVolumeClaim represents a reference to a PersistentVolumeClaim in the same namespace.
## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. ## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod.
## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. ## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template.
## A claim in this list takes precedence over any volumes in the template, with the same name. ## A claim in this list takes precedence over any volumes in the template, with the same name.
@ -150,31 +164,43 @@ master:
storage: "20Gi" storage: "20Gi"
worker: worker:
## PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down.
podManagementPolicy: "Parallel" podManagementPolicy: "Parallel"
## Replicas is the desired number of replicas of the given Template.
replicas: "3" replicas: "3"
# NodeSelector is a selector which must be true for the pod to fit on a node. ## You can use annotations to attach arbitrary non-identifying metadata to objects.
# Selector which must match a node's labels for the pod to be scheduled on that node. ## Clients such as tools and libraries can retrieve this metadata.
# More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ annotations: {}
## Affinity is a group of affinity scheduling rules. If specified, the pod's scheduling constraints.
## More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
affinity: {}
## NodeSelector is a selector which must be true for the pod to fit on a node.
## Selector which must match a node's labels for the pod to be scheduled on that node.
## More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
nodeSelector: {} nodeSelector: {}
# Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, ## Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission,
# effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. ## effectively unioning the set of nodes tolerated by the pod and the RuntimeClass.
tolerations: [] tolerations: []
# Affinity is a group of affinity scheduling rules. ## Compute Resources required by this container. Cannot be updated.
# If specified, the pod's scheduling constraints. ## More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container
# More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
affinity: {}
# The jvm options for java instance startup
jvmOptions: ""
resources: {} resources: {}
# limits: # resources:
# memory: "18Gi" # limits:
# cpu: "4" # memory: "8Gi"
# requests: # cpu: "4"
# memory: "2Gi" # requests:
# cpu: "500m" # memory: "2Gi"
# You can use annotations to attach arbitrary non-identifying metadata to objects. # cpu: "500m"
# Clients such as tools and libraries can retrieve this metadata. ## ConfigMap
annotations: {} configmap:
DOLPHINSCHEDULER_OPTS: ""
WORKER_EXEC_THREADS: "100"
WORKER_HEARTBEAT_INTERVAL: "10"
WORKER_MAX_CPULOAD_AVG: "100"
WORKER_RESERVED_MEMORY: "0.1"
WORKER_LISTEN_PORT: "1234"
WORKER_GROUP: "default"
WORKER_WEIGHT: "100"
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
livenessProbe: livenessProbe:
@ -193,27 +219,7 @@ worker:
timeoutSeconds: "5" timeoutSeconds: "5"
failureThreshold: "3" failureThreshold: "3"
successThreshold: "1" successThreshold: "1"
configmap: ## PersistentVolumeClaim represents a reference to a PersistentVolumeClaim in the same namespace.
WORKER_EXEC_THREADS: "100"
WORKER_HEARTBEAT_INTERVAL: "10"
WORKER_FETCH_TASK_NUM: "3"
WORKER_MAX_CPULOAD_AVG: "100"
WORKER_RESERVED_MEMORY: "0.1"
WORKER_LISTEN_PORT: "1234"
WORKER_GROUP: "default"
WORKER_WEIGHT: "100"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
DOLPHINSCHEDULER_ENV:
- "export HADOOP_HOME=/opt/soft/hadoop"
- "export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop"
- "export SPARK_HOME1=/opt/soft/spark1"
- "export SPARK_HOME2=/opt/soft/spark2"
- "export PYTHON_HOME=/opt/soft/python"
- "export JAVA_HOME=/opt/soft/java"
- "export HIVE_HOME=/opt/soft/hive"
- "export FLINK_HOME=/opt/soft/flink"
- "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH"
## volumeClaimTemplates is a list of claims that pods are allowed to reference.
## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. ## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod.
## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template. ## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template.
## A claim in this list takes precedence over any volumes in the template, with the same name. ## A claim in this list takes precedence over any volumes in the template, with the same name.
@ -235,38 +241,40 @@ worker:
storage: "20Gi" storage: "20Gi"
alert: alert:
## Number of desired pods. This is a pointer to distinguish between explicit zero and not specified. Defaults to 1.
replicas: "1"
## The deployment strategy to use to replace existing pods with new ones.
strategy: strategy:
type: "RollingUpdate" type: "RollingUpdate"
rollingUpdate: rollingUpdate:
maxSurge: "25%" maxSurge: "25%"
maxUnavailable: "25%" maxUnavailable: "25%"
replicas: "1" ## You can use annotations to attach arbitrary non-identifying metadata to objects.
# NodeSelector is a selector which must be true for the pod to fit on a node. ## Clients such as tools and libraries can retrieve this metadata.
# Selector which must match a node's labels for the pod to be scheduled on that node. annotations: {}
# More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ ## NodeSelector is a selector which must be true for the pod to fit on a node.
## Selector which must match a node's labels for the pod to be scheduled on that node.
## More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
affinity: {}
## Compute Resources required by this container. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container
nodeSelector: {} nodeSelector: {}
# Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, ## Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission,
# effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. ## effectively unioning the set of nodes tolerated by the pod and the RuntimeClass.
tolerations: [] tolerations: []
# Affinity is a group of affinity scheduling rules. ## Affinity is a group of affinity scheduling rules. If specified, the pod's scheduling constraints.
# If specified, the pod's scheduling constraints. ## More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
# More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
affinity: {}
# The jvm options for java instance startup
jvmOptions: ""
resources: {} resources: {}
# limits: # resources:
# memory: "4Gi" # limits:
# cpu: "1" # memory: "2Gi"
# requests: # cpu: "1"
# memory: "2Gi" # requests:
# cpu: "500m" # memory: "1Gi"
# You can use annotations to attach arbitrary non-identifying metadata to objects. # cpu: "500m"
# Clients such as tools and libraries can retrieve this metadata. ## ConfigMap
annotations: {}
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
configmap: configmap:
DOLPHINSCHEDULER_OPTS: ""
ALERT_PLUGIN_DIR: "/opt/dolphinscheduler/alert/plugin" ALERT_PLUGIN_DIR: "/opt/dolphinscheduler/alert/plugin"
XLS_FILE_PATH: "/tmp/xls" XLS_FILE_PATH: "/tmp/xls"
MAIL_SERVER_HOST: "" MAIL_SERVER_HOST: ""
@ -282,63 +290,6 @@ alert:
ENTERPRISE_WECHAT_SECRET: "" ENTERPRISE_WECHAT_SECRET: ""
ENTERPRISE_WECHAT_AGENT_ID: "" ENTERPRISE_WECHAT_AGENT_ID: ""
ENTERPRISE_WECHAT_USERS: "" ENTERPRISE_WECHAT_USERS: ""
livenessProbe:
enabled: true
initialDelaySeconds: "30"
periodSeconds: "30"
timeoutSeconds: "5"
failureThreshold: "3"
successThreshold: "1"
## Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
readinessProbe:
enabled: true
initialDelaySeconds: "30"
periodSeconds: "30"
timeoutSeconds: "5"
failureThreshold: "3"
successThreshold: "1"
## volumeClaimTemplates is a list of claims that pods are allowed to reference.
## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod.
## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template.
## A claim in this list takes precedence over any volumes in the template, with the same name.
persistentVolumeClaim:
enabled: false
accessModes:
- "ReadWriteOnce"
storageClassName: "-"
storage: "20Gi"
api:
strategy:
type: "RollingUpdate"
rollingUpdate:
maxSurge: "25%"
maxUnavailable: "25%"
replicas: "1"
# NodeSelector is a selector which must be true for the pod to fit on a node.
# Selector which must match a node's labels for the pod to be scheduled on that node.
# More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
nodeSelector: {}
# Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission,
# effectively unioning the set of nodes tolerated by the pod and the RuntimeClass.
tolerations: []
# Affinity is a group of affinity scheduling rules.
# If specified, the pod's scheduling constraints.
# More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
affinity: {}
# The jvm options for java instance startup
jvmOptions: ""
resources: {}
# limits:
# memory: "4Gi"
# cpu: "2"
# requests:
# memory: "2Gi"
# cpu: "500m"
# You can use annotations to attach arbitrary non-identifying metadata to objects.
# Clients such as tools and libraries can retrieve this metadata.
annotations: {}
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
livenessProbe: livenessProbe:
@ -357,10 +308,8 @@ api:
timeoutSeconds: "5" timeoutSeconds: "5"
failureThreshold: "3" failureThreshold: "3"
successThreshold: "1" successThreshold: "1"
## volumeClaimTemplates is a list of claims that pods are allowed to reference. ## PersistentVolumeClaim represents a reference to a PersistentVolumeClaim in the same namespace.
## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. ## More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims
## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template.
## A claim in this list takes precedence over any volumes in the template, with the same name.
persistentVolumeClaim: persistentVolumeClaim:
enabled: false enabled: false
accessModes: accessModes:
@ -368,34 +317,41 @@ api:
storageClassName: "-" storageClassName: "-"
storage: "20Gi" storage: "20Gi"
frontend: api:
## Number of desired pods. This is a pointer to distinguish between explicit zero and not specified. Defaults to 1.
replicas: "1"
## The deployment strategy to use to replace existing pods with new ones.
strategy: strategy:
type: "RollingUpdate" type: "RollingUpdate"
rollingUpdate: rollingUpdate:
maxSurge: "25%" maxSurge: "25%"
maxUnavailable: "25%" maxUnavailable: "25%"
replicas: "1" ## You can use annotations to attach arbitrary non-identifying metadata to objects.
# NodeSelector is a selector which must be true for the pod to fit on a node. ## Clients such as tools and libraries can retrieve this metadata.
# Selector which must match a node's labels for the pod to be scheduled on that node. annotations: {}
# More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ ## NodeSelector is a selector which must be true for the pod to fit on a node.
## Selector which must match a node's labels for the pod to be scheduled on that node.
## More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
affinity: {}
## Compute Resources required by this container. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container
nodeSelector: {} nodeSelector: {}
# Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, ## Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission,
# effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. ## effectively unioning the set of nodes tolerated by the pod and the RuntimeClass.
tolerations: [] tolerations: []
# Affinity is a group of affinity scheduling rules. ## Affinity is a group of affinity scheduling rules. If specified, the pod's scheduling constraints.
# If specified, the pod's scheduling constraints. ## More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
# More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
affinity: {}
resources: {} resources: {}
# limits: # resources:
# memory: "256Mi" # limits:
# memory: "2Gi"
# cpu: "1" # cpu: "1"
# requests: # requests:
# memory: "256Mi" # memory: "1Gi"
# cpu: "500m" # cpu: "500m"
# You can use annotations to attach arbitrary non-identifying metadata to objects. ## ConfigMap
# Clients such as tools and libraries can retrieve this metadata. configmap:
annotations: {} DOLPHINSCHEDULER_OPTS: ""
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
livenessProbe: livenessProbe:
@ -414,10 +370,8 @@ frontend:
timeoutSeconds: "5" timeoutSeconds: "5"
failureThreshold: "3" failureThreshold: "3"
successThreshold: "1" successThreshold: "1"
## volumeClaimTemplates is a list of claims that pods are allowed to reference. ## PersistentVolumeClaim represents a reference to a PersistentVolumeClaim in the same namespace.
## The StatefulSet controller is responsible for mapping network identities to claims in a way that maintains the identity of a pod. ## More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims
## Every claim in this list must have at least one matching (by name) volumeMount in one container in the template.
## A claim in this list takes precedence over any volumes in the template, with the same name.
persistentVolumeClaim: persistentVolumeClaim:
enabled: false enabled: false
accessModes: accessModes:

7
dolphinscheduler-alert-plugin/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannel.java

@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.spi.alert.AlertChannel;
import org.apache.dolphinscheduler.spi.alert.AlertData; import org.apache.dolphinscheduler.spi.alert.AlertData;
import org.apache.dolphinscheduler.spi.alert.AlertInfo; import org.apache.dolphinscheduler.spi.alert.AlertInfo;
import org.apache.dolphinscheduler.spi.alert.AlertResult; import org.apache.dolphinscheduler.spi.alert.AlertResult;
import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
import java.util.Map; import java.util.Map;
@ -34,8 +33,10 @@ public class DingTalkAlertChannel implements AlertChannel {
public AlertResult process(AlertInfo alertInfo) { public AlertResult process(AlertInfo alertInfo) {
AlertData alertData = alertInfo.getAlertData(); AlertData alertData = alertInfo.getAlertData();
String alertParams = alertInfo.getAlertParams(); Map<String, String> paramsMap = alertInfo.getAlertParams();
Map<String, String> paramsMap = PluginParamsTransfer.getPluginParamsMap(alertParams); if (null == paramsMap) {
return new AlertResult("false", "ding talk params is null");
}
return new DingTalkSender(paramsMap).sendDingTalkMsg(alertData.getTitle(), alertData.getContent()); return new DingTalkSender(paramsMap).sendDingTalkMsg(alertData.getTitle(), alertData.getContent());
} }
} }

44
dolphinscheduler-alert-plugin/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkParamsConstants.java

@ -22,36 +22,24 @@ package org.apache.dolphinscheduler.plugin.alert.dingtalk;
*/ */
public class DingTalkParamsConstants { public class DingTalkParamsConstants {
static final String DING_TALK_PROXY_ENABLE = "isEnableProxy";
static final String DING_TALK_WEB_HOOK = "webhook";
static final String NAME_DING_TALK_WEB_HOOK = "WebHook";
static final String DING_TALK_KEYWORD = "keyword";
static final String NAME_DING_TALK_KEYWORD = "Keyword";
static final String NAME_DING_TALK_PROXY_ENABLE = "IsEnableProxy";
static final String DING_TALK_PROXY = "proxy";
static final String NAME_DING_TALK_PROXY = "Proxy";
static final String DING_TALK_PORT = "port";
static final String NAME_DING_TALK_PORT = "Port";
static final String DING_TALK_USER = "user";
static final String NAME_DING_TALK_USER = "User";
static final String DING_TALK_PASSWORD = "password";
static final String NAME_DING_TALK_PASSWORD = "Password";
private DingTalkParamsConstants() { private DingTalkParamsConstants() {
throw new IllegalStateException("Utility class"); throw new IllegalStateException("Utility class");
} }
static final String DING_TALK_WEB_HOOK = "dingtalk.webhook";
static final String NAME_DING_TALK_WEB_HOOK = "dingTalkWebHook";
static final String DING_TALK_KEYWORD = "dingtalk.keyword";
static final String NAME_DING_TALK_KEYWORD = "dingTalkKeyword";
public static final String DING_TALK_PROXY_ENABLE = "dingtalk.isEnableProxy";
static final String NAME_DING_TALK_PROXY_ENABLE = "dingTalkIsEnableProxy";
static final String DING_TALK_PROXY = "dingtalk.proxy";
static final String NAME_DING_TALK_PROXY = "dingTalkProxy";
static final String DING_TALK_PORT = "dingtalk.port";
static final String NAME_DING_TALK_PORT = "dingTalkPort";
static final String DING_TALK_USER = "dingtalk.user";
static final String NAME_DING_TALK_USER = "dingTalkUser";
static final String DING_TALK_PASSWORD = "dingtalk.password";
static final String NAME_DING_TALK_PASSWORD = "dingTalkPassword";
} }

132
dolphinscheduler-alert-plugin/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java

@ -75,51 +75,6 @@ public class DingTalkSender {
} }
public AlertResult sendDingTalkMsg(String msg, String charset) {
AlertResult alertResult;
try {
String resp = sendMsg(msg, charset);
return checkSendDingTalkSendMsgResult(resp);
} catch (Exception e) {
logger.info("send ding talk alert msg exception : {}", e.getMessage());
alertResult = new AlertResult();
alertResult.setStatus("false");
alertResult.setMessage("send ding talk alert fail.");
}
return alertResult;
}
private String sendMsg(String msg, String charset) throws IOException {
String msgToJson = textToJsonString(msg + "#" + keyword);
HttpPost httpPost = constructHttpPost(url, msgToJson, charset);
CloseableHttpClient httpClient;
if (Boolean.TRUE.equals(enableProxy)) {
httpClient = getProxyClient(proxy, port, user, password);
RequestConfig rcf = getProxyConfig(proxy, port);
httpPost.setConfig(rcf);
} else {
httpClient = getDefaultClient();
}
try {
CloseableHttpResponse response = httpClient.execute(httpPost);
String resp;
try {
HttpEntity entity = response.getEntity();
resp = EntityUtils.toString(entity, charset);
EntityUtils.consume(entity);
} finally {
response.close();
}
logger.info("Ding Talk send {}, resp: {}", msg, resp);
return resp;
} finally {
httpClient.close();
}
}
private static HttpPost constructHttpPost(String url, String msg, String charset) { private static HttpPost constructHttpPost(String url, String msg, String charset) {
HttpPost post = new HttpPost(url); HttpPost post = new HttpPost(url);
StringEntity entity = new StringEntity(msg, charset); StringEntity entity = new StringEntity(msg, charset);
@ -155,27 +110,6 @@ public class DingTalkSender {
return JSONUtils.toJsonString(items); return JSONUtils.toJsonString(items);
} }
public static class DingTalkSendMsgResponse {
private Integer errcode;
private String errmsg;
public Integer getErrcode() {
return errcode;
}
public void setErrcode(Integer errcode) {
this.errcode = errcode;
}
public String getErrmsg() {
return errmsg;
}
public void setErrmsg(String errmsg) {
this.errmsg = errmsg;
}
}
private static AlertResult checkSendDingTalkSendMsgResult(String result) { private static AlertResult checkSendDingTalkSendMsgResult(String result) {
AlertResult alertResult = new AlertResult(); AlertResult alertResult = new AlertResult();
alertResult.setStatus("false"); alertResult.setStatus("false");
@ -201,4 +135,70 @@ public class DingTalkSender {
return alertResult; return alertResult;
} }
public AlertResult sendDingTalkMsg(String title, String content) {
AlertResult alertResult;
try {
String resp = sendMsg(title, content);
return checkSendDingTalkSendMsgResult(resp);
} catch (Exception e) {
logger.info("send ding talk alert msg exception : {}", e.getMessage());
alertResult = new AlertResult();
alertResult.setStatus("false");
alertResult.setMessage("send ding talk alert fail.");
}
return alertResult;
}
private String sendMsg(String title, String content) throws IOException {
String msgToJson = textToJsonString(title + content + "#" + keyword);
HttpPost httpPost = constructHttpPost(url, msgToJson, "UTF-8");
CloseableHttpClient httpClient;
if (Boolean.TRUE.equals(enableProxy)) {
httpClient = getProxyClient(proxy, port, user, password);
RequestConfig rcf = getProxyConfig(proxy, port);
httpPost.setConfig(rcf);
} else {
httpClient = getDefaultClient();
}
try {
CloseableHttpResponse response = httpClient.execute(httpPost);
String resp;
try {
HttpEntity entity = response.getEntity();
resp = EntityUtils.toString(entity, "UTF-8");
EntityUtils.consume(entity);
} finally {
response.close();
}
logger.info("Ding Talk send title :{},content : {}, resp: {}", title, content, resp);
return resp;
} finally {
httpClient.close();
}
}
public static class DingTalkSendMsgResponse {
private Integer errcode;
private String errmsg;
public Integer getErrcode() {
return errcode;
}
public void setErrcode(Integer errcode) {
this.errcode = errcode;
}
public String getErrmsg() {
return errmsg;
}
public void setErrmsg(String errmsg) {
this.errmsg = errmsg;
}
}
} }

2
dolphinscheduler-alert-plugin/dolphinscheduler-alert-dingtalk/src/test/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSenderTest.java

@ -50,7 +50,7 @@ public class DingTalkSenderTest {
dingTalkSender.sendDingTalkMsg("keyWord+Welcome", "UTF-8"); dingTalkSender.sendDingTalkMsg("keyWord+Welcome", "UTF-8");
dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE, "true"); dingTalkConfig.put(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE, "true");
dingTalkSender = new DingTalkSender(dingTalkConfig); dingTalkSender = new DingTalkSender(dingTalkConfig);
AlertResult alertResult = dingTalkSender.sendDingTalkMsg("keyWord+Welcome", "UTF-8"); AlertResult alertResult = dingTalkSender.sendDingTalkMsg("title", "content test");
Assert.assertEquals("false",alertResult.getStatus()); Assert.assertEquals("false",alertResult.getStatus());
} }

7
dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannel.java

@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.spi.alert.AlertChannel;
import org.apache.dolphinscheduler.spi.alert.AlertData; import org.apache.dolphinscheduler.spi.alert.AlertData;
import org.apache.dolphinscheduler.spi.alert.AlertInfo; import org.apache.dolphinscheduler.spi.alert.AlertInfo;
import org.apache.dolphinscheduler.spi.alert.AlertResult; import org.apache.dolphinscheduler.spi.alert.AlertResult;
import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
import java.util.Map; import java.util.Map;
@ -38,8 +37,10 @@ public class EmailAlertChannel implements AlertChannel {
public AlertResult process(AlertInfo info) { public AlertResult process(AlertInfo info) {
AlertData alert = info.getAlertData(); AlertData alert = info.getAlertData();
String alertParams = info.getAlertParams(); Map<String, String> paramsMap = info.getAlertParams();
Map<String, String> paramsMap = PluginParamsTransfer.getPluginParamsMap(alertParams); if (null == paramsMap) {
return new AlertResult("false", "mail params is null");
}
MailSender mailSender = new MailSender(paramsMap); MailSender mailSender = new MailSender(paramsMap);
AlertResult alertResult = mailSender.sendMails(alert.getTitle(), alert.getContent()); AlertResult alertResult = mailSender.sendMails(alert.getTitle(), alert.getContent());

20
dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailParamsConstants.java

@ -33,33 +33,33 @@ public class MailParamsConstants {
public static final String NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERCCS = "receiverCcs"; public static final String NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERCCS = "receiverCcs";
public static final String MAIL_PROTOCOL = "transport.protocol"; public static final String MAIL_PROTOCOL = "transport.protocol";
public static final String NAME_MAIL_PROTOCOL = "protocol"; public static final String NAME_MAIL_PROTOCOL = "mail.protocol";
public static final String MAIL_SMTP_HOST = "smtp.host"; public static final String MAIL_SMTP_HOST = "mail.smtp.host";
public static final String NAME_MAIL_SMTP_HOST = "serverHost"; public static final String NAME_MAIL_SMTP_HOST = "serverHost";
public static final String MAIL_SMTP_PORT = "smtp.port"; public static final String MAIL_SMTP_PORT = "mail.smtp.port";
public static final String NAME_MAIL_SMTP_PORT = "serverPort"; public static final String NAME_MAIL_SMTP_PORT = "serverPort";
public static final String MAIL_SENDER = "sender"; public static final String MAIL_SENDER = "mail.sender";
public static final String NAME_MAIL_SENDER = "sender"; public static final String NAME_MAIL_SENDER = "sender";
public static final String MAIL_SMTP_AUTH = "smtp.auth"; public static final String MAIL_SMTP_AUTH = "mail.smtp.auth";
public static final String NAME_MAIL_SMTP_AUTH = "enableSmtpAuth"; public static final String NAME_MAIL_SMTP_AUTH = "enableSmtpAuth";
public static final String MAIL_USER = "user"; public static final String MAIL_USER = "mail.user";
public static final String NAME_MAIL_USER = "user"; public static final String NAME_MAIL_USER = "user";
public static final String MAIL_PASSWD = "passwd"; public static final String MAIL_PASSWD = "mail.passwd";
public static final String NAME_MAIL_PASSWD = "passwd"; public static final String NAME_MAIL_PASSWD = "passwd";
public static final String MAIL_SMTP_STARTTLS_ENABLE = "smtp.starttls.enable"; public static final String MAIL_SMTP_STARTTLS_ENABLE = "mail.smtp.starttls.enable";
public static final String NAME_MAIL_SMTP_STARTTLS_ENABLE = "starttlsEnable"; public static final String NAME_MAIL_SMTP_STARTTLS_ENABLE = "starttlsEnable";
public static final String MAIL_SMTP_SSL_ENABLE = "smtp.ssl.enable"; public static final String MAIL_SMTP_SSL_ENABLE = "mail.smtp.ssl.enable";
public static final String NAME_MAIL_SMTP_SSL_ENABLE = "sslEnable"; public static final String NAME_MAIL_SMTP_SSL_ENABLE = "sslEnable";
public static final String MAIL_SMTP_SSL_TRUST = "smtp.ssl.trust"; public static final String MAIL_SMTP_SSL_TRUST = "mail.smtp.ssl.trust";
public static final String NAME_MAIL_SMTP_SSL_TRUST = "smtpSslTrust"; public static final String NAME_MAIL_SMTP_SSL_TRUST = "smtpSslTrust";
} }

23
dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannelFactoryTest.java

@ -19,13 +19,10 @@ package org.apache.dolphinscheduler.plugin.alert.email;
import org.apache.dolphinscheduler.spi.alert.AlertChannel; import org.apache.dolphinscheduler.spi.alert.AlertChannel;
import org.apache.dolphinscheduler.spi.params.base.PluginParams; import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import java.util.List; import java.util.List;
import org.junit.After;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before;
import org.junit.Test; import org.junit.Test;
/** /**
@ -36,29 +33,13 @@ import org.junit.Test;
*/ */
public class EmailAlertChannelFactoryTest { public class EmailAlertChannelFactoryTest {
@Before
public void before() throws Exception {
}
@After
public void after() throws Exception {
}
/**
* Method: getName()
*/
@Test
public void testGetName() throws Exception {
}
/** /**
* Method: getParams() * Method: getParams()
*/ */
@Test @Test
public void testGetParams() throws Exception { public void testGetParams() {
EmailAlertChannelFactory emailAlertChannelFactory = new EmailAlertChannelFactory(); EmailAlertChannelFactory emailAlertChannelFactory = new EmailAlertChannelFactory();
List<PluginParams> params = emailAlertChannelFactory.getParams(); List<PluginParams> params = emailAlertChannelFactory.getParams();
System.out.println(JSONUtils.toJsonString(params));
Assert.assertEquals(12, params.size()); Assert.assertEquals(12, params.size());
} }
@ -66,7 +47,7 @@ public class EmailAlertChannelFactoryTest {
* Method: create() * Method: create()
*/ */
@Test @Test
public void testCreate() throws Exception { public void testCreate() {
EmailAlertChannelFactory emailAlertChannelFactory = new EmailAlertChannelFactory(); EmailAlertChannelFactory emailAlertChannelFactory = new EmailAlertChannelFactory();
AlertChannel alertChannel = emailAlertChannelFactory.create(); AlertChannel alertChannel = emailAlertChannelFactory.create();
Assert.assertNotNull(alertChannel); Assert.assertNotNull(alertChannel);

6
dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannelTest.java

@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.spi.alert.AlertResult;
import org.apache.dolphinscheduler.spi.alert.ShowType; import org.apache.dolphinscheduler.spi.alert.ShowType;
import org.apache.dolphinscheduler.spi.params.InputParam; import org.apache.dolphinscheduler.spi.params.InputParam;
import org.apache.dolphinscheduler.spi.params.PasswordParam; import org.apache.dolphinscheduler.spi.params.PasswordParam;
import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
import org.apache.dolphinscheduler.spi.params.RadioParam; import org.apache.dolphinscheduler.spi.params.RadioParam;
import org.apache.dolphinscheduler.spi.params.base.DataType; import org.apache.dolphinscheduler.spi.params.base.DataType;
import org.apache.dolphinscheduler.spi.params.base.ParamsOptions; import org.apache.dolphinscheduler.spi.params.base.ParamsOptions;
@ -34,6 +35,7 @@ import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
@ -66,7 +68,9 @@ public class EmailAlertChannelTest {
.setTitle("test"); .setTitle("test");
AlertInfo alertInfo = new AlertInfo(); AlertInfo alertInfo = new AlertInfo();
alertInfo.setAlertData(alertData); alertInfo.setAlertData(alertData);
alertInfo.setAlertParams(getEmailAlertParams()); Map<String, String> paramsMap = PluginParamsTransfer.getPluginParamsMap(getEmailAlertParams());
alertInfo.setAlertParams(paramsMap);
AlertResult alertResult = emailAlertChannel.process(alertInfo); AlertResult alertResult = emailAlertChannel.process(alertInfo);
Assert.assertNotNull(alertResult); Assert.assertNotNull(alertResult);
Assert.assertEquals("false", alertResult.getStatus()); Assert.assertEquals("false", alertResult.getStatus());

1
dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/src/test/java/org/apache/dolphinscheduler/plugin/alert/email/MailUtilsTest.java

@ -53,6 +53,7 @@ public class MailUtilsTest {
emailConfig.put(MailParamsConstants.NAME_MAIL_SENDER, "xxx1.xxx.com"); emailConfig.put(MailParamsConstants.NAME_MAIL_SENDER, "xxx1.xxx.com");
emailConfig.put(MailParamsConstants.NAME_MAIL_USER, "xxx2.xxx.com"); emailConfig.put(MailParamsConstants.NAME_MAIL_USER, "xxx2.xxx.com");
emailConfig.put(MailParamsConstants.NAME_MAIL_PASSWD, "111111"); emailConfig.put(MailParamsConstants.NAME_MAIL_PASSWD, "111111");
emailConfig.put(MailParamsConstants.NAME_MAIL_SMTP_AUTH, "true");
emailConfig.put(MailParamsConstants.NAME_MAIL_SMTP_STARTTLS_ENABLE, "true"); emailConfig.put(MailParamsConstants.NAME_MAIL_SMTP_STARTTLS_ENABLE, "true");
emailConfig.put(MailParamsConstants.NAME_MAIL_SMTP_SSL_ENABLE, "false"); emailConfig.put(MailParamsConstants.NAME_MAIL_SMTP_SSL_ENABLE, "false");
emailConfig.put(MailParamsConstants.NAME_MAIL_SMTP_SSL_TRUST, "false"); emailConfig.put(MailParamsConstants.NAME_MAIL_SMTP_SSL_TRUST, "false");

82
dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/pom.xml

@ -0,0 +1,82 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>dolphinscheduler-alert-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
<version>1.3.4-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-alert-feishu</artifactId>
<packaging>dolphinscheduler-plugin</packaging>
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-spi</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<type>jar</type>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<finalName>dolphinscheduler-alert-feishu-${project.version}</finalName>
</build>
</project>

38
dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannel.java

@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.alert.feishu;
import org.apache.dolphinscheduler.spi.alert.AlertChannel;
import org.apache.dolphinscheduler.spi.alert.AlertData;
import org.apache.dolphinscheduler.spi.alert.AlertInfo;
import org.apache.dolphinscheduler.spi.alert.AlertResult;
import java.util.Map;
public class FeiShuAlertChannel implements AlertChannel {
@Override
public AlertResult process(AlertInfo alertInfo) {
AlertData alertData = alertInfo.getAlertData();
Map<String, String> paramsMap = alertInfo.getAlertParams();
if (null == paramsMap) {
return new AlertResult("false", "fei shu params is null");
}
return new FeiShuSender(paramsMap).sendFeiShuMsg(alertData);
}
}

82
dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannelFactory.java

@ -0,0 +1,82 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.alert.feishu;
import org.apache.dolphinscheduler.spi.alert.AlertChannel;
import org.apache.dolphinscheduler.spi.alert.AlertChannelFactory;
import org.apache.dolphinscheduler.spi.params.InputParam;
import org.apache.dolphinscheduler.spi.params.PasswordParam;
import org.apache.dolphinscheduler.spi.params.RadioParam;
import org.apache.dolphinscheduler.spi.params.base.ParamsOptions;
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import org.apache.dolphinscheduler.spi.params.base.Validate;
import java.util.Arrays;
import java.util.List;
public class FeiShuAlertChannelFactory implements AlertChannelFactory {
@Override
public String getName() {
return "Feishu";
}
@Override
public List<PluginParams> getParams() {
InputParam webHookParam = InputParam.newBuilder(FeiShuParamsConstants.NAME_WEB_HOOK, FeiShuParamsConstants.WEB_HOOK)
.addValidate(Validate.newBuilder()
.setRequired(true)
.build())
.build();
RadioParam isEnableProxy =
RadioParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PROXY_ENABLE, FeiShuParamsConstants.NAME_FEI_SHU_PROXY_ENABLE)
.addParamsOptions(new ParamsOptions("YES", true, false))
.addParamsOptions(new ParamsOptions("NO", false, false))
.setValue(true)
.addValidate(Validate.newBuilder()
.setRequired(false)
.build())
.build();
InputParam proxyParam =
InputParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PROXY, FeiShuParamsConstants.FEI_SHU_PROXY)
.addValidate(Validate.newBuilder()
.setRequired(false).build())
.build();
InputParam portParam = InputParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PORT, FeiShuParamsConstants.FEI_SHU_PORT)
.addValidate(Validate.newBuilder()
.setRequired(false).build())
.build();
InputParam userParam =
InputParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_USER, FeiShuParamsConstants.FEI_SHU_USER)
.addValidate(Validate.newBuilder()
.setRequired(false).build())
.build();
PasswordParam passwordParam = PasswordParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PASSWORD, FeiShuParamsConstants.FEI_SHU_PASSWORD)
.setPlaceholder("if enable use authentication, you need input password")
.build();
return Arrays.asList(webHookParam, isEnableProxy, proxyParam, portParam, userParam, passwordParam);
}
@Override
public AlertChannel create() {
return new FeiShuAlertChannel();
}
}

26
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/MsgManager.java → dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertPlugin.java

@ -15,26 +15,16 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.alert.manager; package org.apache.dolphinscheduler.plugin.alert.feishu;
import org.apache.dolphinscheduler.dao.entity.Alert; import org.apache.dolphinscheduler.spi.DolphinSchedulerPlugin;
import org.apache.dolphinscheduler.spi.alert.AlertChannelFactory;
import org.slf4j.Logger; import com.google.common.collect.ImmutableList;
import org.slf4j.LoggerFactory;
/** public class FeiShuAlertPlugin implements DolphinSchedulerPlugin {
* SMS send manager @Override
*/ public Iterable<AlertChannelFactory> getAlertChannelFactorys() {
public class MsgManager { return ImmutableList.of(new FeiShuAlertChannelFactory());
private static final Logger logger = LoggerFactory.getLogger(MsgManager.class);
/**
* SMS send
*
* @param alert the alert
*/
public void send(Alert alert) {
logger.info("send message {}", alert);
} }
} }

49
dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuParamsConstants.java

@ -0,0 +1,49 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.alert.feishu;
public class FeiShuParamsConstants {
private FeiShuParamsConstants() {
throw new IllegalStateException("Utility class");
}
static final String WEB_HOOK = "webhook";
static final String NAME_WEB_HOOK = "webHook";
public static final String FEI_SHU_PROXY_ENABLE = "isEnableProxy";
static final String NAME_FEI_SHU_PROXY_ENABLE = "isEnableProxy";
static final String FEI_SHU_PROXY = "proxy";
static final String NAME_FEI_SHU_PROXY = "proxy";
static final String FEI_SHU_PORT = "port";
static final String NAME_FEI_SHU_PORT = "port";
static final String FEI_SHU_USER = "user";
static final String NAME_FEI_SHU_USER = "user";
static final String FEI_SHU_PASSWORD = "password";
static final String NAME_FEI_SHU_PASSWORD = "password";
}

214
dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java

@ -0,0 +1,214 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.alert.feishu;
import org.apache.dolphinscheduler.spi.alert.AlertData;
import org.apache.dolphinscheduler.spi.alert.AlertResult;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.commons.codec.binary.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonProperty;
public class FeiShuSender {
private static final Logger logger = LoggerFactory.getLogger(FeiShuSender.class);
private String url;
private Boolean enableProxy;
private String proxy;
private Integer port;
private String user;
private String password;
FeiShuSender(Map<String, String> config) {
url = config.get(FeiShuParamsConstants.NAME_WEB_HOOK);
enableProxy = Boolean.valueOf(config.get(FeiShuParamsConstants.NAME_FEI_SHU_PROXY_ENABLE));
if (Boolean.TRUE.equals(enableProxy)) {
port = Integer.parseInt(config.get(FeiShuParamsConstants.NAME_FEI_SHU_PORT));
proxy = config.get(FeiShuParamsConstants.NAME_FEI_SHU_PROXY);
user = config.get(FeiShuParamsConstants.NAME_FEI_SHU_USER);
password = config.get(FeiShuParamsConstants.NAME_FEI_SHU_PASSWORD);
}
}
private static String textToJsonString(AlertData alertData) {
Map<String, Object> items = new HashMap<>(2);
items.put("msg_type", "text");
Map<String, String> textContent = new HashMap<>();
byte[] byt = StringUtils.getBytesUtf8(formatContent(alertData));
String txt = StringUtils.newStringUtf8(byt);
textContent.put("text", txt);
items.put("content", textContent);
return JSONUtils.toJsonString(items);
}
public static AlertResult checkSendFeiShuSendMsgResult(String result) {
AlertResult alertResult = new AlertResult();
alertResult.setStatus("false");
if (org.apache.dolphinscheduler.spi.utils.StringUtils.isBlank(result)) {
alertResult.setMessage("send fei shu msg error");
logger.info("send fei shu msg error,fei shu server resp is null");
return alertResult;
}
FeiShuSendMsgResponse sendMsgResponse = JSONUtils.parseObject(result, FeiShuSendMsgResponse.class);
if (null == sendMsgResponse) {
alertResult.setMessage("send fei shu msg fail");
logger.info("send fei shu msg error,resp error");
return alertResult;
}
if (sendMsgResponse.statusCode == 0) {
alertResult.setStatus("true");
alertResult.setMessage("send fei shu msg success");
return alertResult;
}
alertResult.setMessage(String.format("alert send fei shu msg error : %s", sendMsgResponse.getStatusMessage()));
logger.info("alert send fei shu msg error : {} ,Extra : {} ", sendMsgResponse.getStatusMessage(), sendMsgResponse.getExtra());
return alertResult;
}
public static String formatContent(AlertData alertData) {
if (alertData.getContent() != null) {
List<Map> list = JSONUtils.toList(alertData.getContent(), Map.class);
if (list.isEmpty()) {
return alertData.getTitle() + alertData.getContent();
}
StringBuilder contents = new StringBuilder(100);
contents.append(String.format("`%s`%n", alertData.getTitle()));
for (Map map : list) {
Iterator<Entry<String, Object>> entries = map.entrySet().iterator();
while (entries.hasNext()) {
Entry<String, Object> entry = entries.next();
String key = entry.getKey();
String value = entry.getValue().toString();
contents.append(key + ":" + value);
contents.append("\n");
}
}
return contents.toString();
}
return null;
}
public AlertResult sendFeiShuMsg(AlertData alertData) {
AlertResult alertResult;
try {
String resp = sendMsg(alertData);
return checkSendFeiShuSendMsgResult(resp);
} catch (Exception e) {
logger.info("send fei shu alert msg exception : {}", e.getMessage());
alertResult = new AlertResult();
alertResult.setStatus("false");
alertResult.setMessage("send fei shu alert fail.");
}
return alertResult;
}
private String sendMsg(AlertData alertData) throws IOException {
String msgToJson = textToJsonString(alertData);
HttpPost httpPost = HttpRequestUtil.constructHttpPost(url, msgToJson);
CloseableHttpClient httpClient;
httpClient = HttpRequestUtil.getHttpClient(enableProxy, proxy, port, user, password);
try {
CloseableHttpResponse response = httpClient.execute(httpPost);
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
logger.error("send feishu message error, return http status code: {} ", statusCode);
}
String resp;
try {
HttpEntity entity = response.getEntity();
resp = EntityUtils.toString(entity, "utf-8");
EntityUtils.consume(entity);
} finally {
response.close();
}
logger.info("Fei Shu send title :{} ,content :{}, resp: {}", alertData.getTitle(), alertData.getContent(), resp);
return resp;
} finally {
httpClient.close();
}
}
public static class FeiShuSendMsgResponse {
@JsonProperty("Extra")
private String extra;
@JsonProperty("StatusCode")
private Integer statusCode;
@JsonProperty("StatusMessage")
private String statusMessage;
public String getExtra() {
return extra;
}
public void setExtra(String extra) {
this.extra = extra;
}
public Integer getStatusCode() {
return statusCode;
}
public void setStatusCode(Integer statusCode) {
this.statusCode = statusCode;
}
public String getStatusMessage() {
return statusMessage;
}
public void setStatusMessage(String statusMessage) {
this.statusMessage = statusMessage;
}
}
}

50
dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/HttpRequestUtil.java

@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.alert.feishu;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
public class HttpRequestUtil {
public static CloseableHttpClient getHttpClient(boolean enableProxy, String proxy, Integer port, String user, String password) {
if (enableProxy) {
HttpHost httpProxy = new HttpHost(proxy, port);
CredentialsProvider provider = new BasicCredentialsProvider();
provider.setCredentials(new AuthScope(httpProxy), new UsernamePasswordCredentials(user, password));
return HttpClients.custom().setDefaultCredentialsProvider(provider).build();
} else {
return HttpClients.createDefault();
}
}
public static HttpPost constructHttpPost(String url, String msg) {
HttpPost post = new HttpPost(url);
StringEntity entity = new StringEntity(msg, ContentType.APPLICATION_JSON);
post.setEntity(entity);
return post;
}
}

45
dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/test/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannelFactoryTest.java

@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.alert.feishu;
import org.apache.dolphinscheduler.spi.alert.AlertChannel;
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import java.util.List;
import org.junit.Assert;
import org.junit.Test;
public class FeiShuAlertChannelFactoryTest {
@Test
public void testGetParams() {
FeiShuAlertChannelFactory feiShuAlertChannelFactory = new FeiShuAlertChannelFactory();
List<PluginParams> params = feiShuAlertChannelFactory.getParams();
JSONUtils.toJsonString(params);
Assert.assertEquals(6, params.size());
}
@Test
public void testCreate() {
FeiShuAlertChannelFactory feiShuAlertChannelFactory = new FeiShuAlertChannelFactory();
AlertChannel alertChannel = feiShuAlertChannelFactory.create();
Assert.assertNotNull(alertChannel);
}
}

99
dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/src/test/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSenderTest.java

@ -0,0 +1,99 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.alert.feishu;
import org.apache.dolphinscheduler.spi.alert.AlertData;
import org.apache.dolphinscheduler.spi.alert.AlertResult;
import java.util.HashMap;
import java.util.Map;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
public class FeiShuSenderTest {
private static Map<String, String> feiShuConfig = new HashMap<>();
@Before
public void initFeiShuConfig() {
feiShuConfig.put(FeiShuParamsConstants.WEB_HOOK, "https://open.feishu.cn/open-apis/bot/v2/hook/xxxxx");
}
@Test
public void testSend() {
AlertData alertData = new AlertData();
alertData.setTitle("feishu test title");
alertData.setContent("feishu test content");
FeiShuSender feiShuSender = new FeiShuSender(feiShuConfig);
AlertResult alertResult = feiShuSender.sendFeiShuMsg(alertData);
Assert.assertEquals("false", alertResult.getStatus());
}
@Test
public void testFormatContent() {
String alertMsg = "[\n"
+ " {\n"
+ " \"owner\": \"dolphinscheduler\",\n"
+ " \"processEndTime\": \"2021-01-29 19:01:11\",\n"
+ " \"processHost\": \"10.81.129.4:5678\",\n"
+ " \"processId\": 2926,\n"
+ " \"processName\": \"3-20210129190038108\",\n"
+ " \"processStartTime\": \"2021-01-29 19:00:38\",\n"
+ " \"processState\": \"SUCCESS\",\n"
+ " \"processType\": \"START_PROCESS\",\n"
+ " \"projectId\": 2,\n"
+ " \"projectName\": \"testdelproject\",\n"
+ " \"recovery\": \"NO\",\n"
+ " \"retryTimes\": 0,\n"
+ " \"runTimes\": 1,\n"
+ " \"taskId\": 0\n"
+ " }\n"
+ "]";
AlertData alertData = new AlertData();
alertData.setTitle("");
alertData.setContent(alertMsg);
Assert.assertNotNull(FeiShuSender.formatContent(alertData));
}
@Test
public void testSendWithFormatException() {
AlertData alertData = new AlertData();
alertData.setTitle("feishu test title");
alertData.setContent("feishu test content");
FeiShuSender feiShuSender = new FeiShuSender(feiShuConfig);
String alertResult = feiShuSender.formatContent(alertData);
Assert.assertEquals(alertResult, alertData.getTitle() + alertData.getContent());
}
@Test
public void testCheckSendFeiShuSendMsgResult() {
FeiShuSender feiShuSender = new FeiShuSender(feiShuConfig);
AlertResult alertResult = feiShuSender.checkSendFeiShuSendMsgResult("");
Assert.assertFalse(Boolean.valueOf(alertResult.getStatus()));
AlertResult alertResult2 = feiShuSender.checkSendFeiShuSendMsgResult("123");
Assert.assertEquals("send fei shu msg fail",alertResult2.getMessage());
String response = "{\"StatusCode\":\"0\",\"extra\":\"extra\",\"StatusMessage\":\"StatusMessage\"}";
AlertResult alertResult3 = feiShuSender.checkSendFeiShuSendMsgResult(response);
Assert.assertTrue(Boolean.valueOf(alertResult3.getStatus()));
}
}

7
dolphinscheduler-alert-plugin/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannel.java

@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.spi.alert.AlertChannel;
import org.apache.dolphinscheduler.spi.alert.AlertData; import org.apache.dolphinscheduler.spi.alert.AlertData;
import org.apache.dolphinscheduler.spi.alert.AlertInfo; import org.apache.dolphinscheduler.spi.alert.AlertInfo;
import org.apache.dolphinscheduler.spi.alert.AlertResult; import org.apache.dolphinscheduler.spi.alert.AlertResult;
import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
import java.util.Map; import java.util.Map;
@ -33,8 +32,10 @@ public class HttpAlertChannel implements AlertChannel {
public AlertResult process(AlertInfo alertInfo) { public AlertResult process(AlertInfo alertInfo) {
AlertData alertData = alertInfo.getAlertData(); AlertData alertData = alertInfo.getAlertData();
String alertParams = alertInfo.getAlertParams(); Map<String, String> paramsMap = alertInfo.getAlertParams();
Map<String, String> paramsMap = PluginParamsTransfer.getPluginParamsMap(alertParams); if (null == paramsMap) {
return new AlertResult("false", "http params is null");
}
return new HttpSender(paramsMap).send(alertData.getContent()); return new HttpSender(paramsMap).send(alertData.getContent());
} }

7
dolphinscheduler-alert-plugin/dolphinscheduler-alert-http/src/test/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannelTest.java

@ -21,12 +21,14 @@ import org.apache.dolphinscheduler.spi.alert.AlertData;
import org.apache.dolphinscheduler.spi.alert.AlertInfo; import org.apache.dolphinscheduler.spi.alert.AlertInfo;
import org.apache.dolphinscheduler.spi.alert.AlertResult; import org.apache.dolphinscheduler.spi.alert.AlertResult;
import org.apache.dolphinscheduler.spi.params.InputParam; import org.apache.dolphinscheduler.spi.params.InputParam;
import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
import org.apache.dolphinscheduler.spi.params.base.PluginParams; import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import org.apache.dolphinscheduler.spi.params.base.Validate; import org.apache.dolphinscheduler.spi.params.base.Validate;
import org.apache.dolphinscheduler.spi.utils.JSONUtils; import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
@ -45,7 +47,7 @@ public class HttpAlertChannelTest {
alertData.setContent("Fault tolerance warning"); alertData.setContent("Fault tolerance warning");
alertInfo.setAlertData(alertData); alertInfo.setAlertData(alertData);
AlertResult alertResult = alertChannel.process(alertInfo); AlertResult alertResult = alertChannel.process(alertInfo);
Assert.assertEquals("Request types are not supported", alertResult.getMessage()); Assert.assertEquals("http params is null", alertResult.getMessage());
} }
@Test @Test
@ -56,7 +58,8 @@ public class HttpAlertChannelTest {
AlertData alertData = new AlertData(); AlertData alertData = new AlertData();
alertData.setContent("Fault tolerance warning"); alertData.setContent("Fault tolerance warning");
alertInfo.setAlertData(alertData); alertInfo.setAlertData(alertData);
alertInfo.setAlertParams(getParams()); Map<String, String> paramsMap = PluginParamsTransfer.getPluginParamsMap(getParams());
alertInfo.setAlertParams(paramsMap);
AlertResult alertResult = alertChannel.process(alertInfo); AlertResult alertResult = alertChannel.process(alertInfo);
Assert.assertEquals("true", alertResult.getStatus()); Assert.assertEquals("true", alertResult.getStatus());
} }

7
dolphinscheduler-alert-plugin/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannel.java

@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.spi.alert.AlertChannel;
import org.apache.dolphinscheduler.spi.alert.AlertData; import org.apache.dolphinscheduler.spi.alert.AlertData;
import org.apache.dolphinscheduler.spi.alert.AlertInfo; import org.apache.dolphinscheduler.spi.alert.AlertInfo;
import org.apache.dolphinscheduler.spi.alert.AlertResult; import org.apache.dolphinscheduler.spi.alert.AlertResult;
import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
import java.util.Map; import java.util.Map;
@ -33,8 +32,10 @@ public class ScriptAlertChannel implements AlertChannel {
@Override @Override
public AlertResult process(AlertInfo alertinfo) { public AlertResult process(AlertInfo alertinfo) {
AlertData alertData = alertinfo.getAlertData(); AlertData alertData = alertinfo.getAlertData();
String alertParams = alertinfo.getAlertParams(); Map<String, String> paramsMap = alertinfo.getAlertParams();
Map<String, String> paramsMap = PluginParamsTransfer.getPluginParamsMap(alertParams); if (null == paramsMap) {
return new AlertResult("false", "ding talk params is null");
}
return new ScriptSender(paramsMap).sendScriptAlert(alertData.getTitle()); return new ScriptSender(paramsMap).sendScriptAlert(alertData.getTitle());
} }
} }

2
dolphinscheduler-alert-plugin/dolphinscheduler-alert-script/src/test/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannelFactoryTest.java

@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.plugin.alert.script;
import org.apache.dolphinscheduler.spi.alert.AlertChannel; import org.apache.dolphinscheduler.spi.alert.AlertChannel;
import org.apache.dolphinscheduler.spi.params.base.PluginParams; import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import java.util.List; import java.util.List;
@ -35,7 +34,6 @@ public class ScriptAlertChannelFactoryTest {
public void testGetParams() { public void testGetParams() {
ScriptAlertChannelFactory scriptAlertChannelFactory = new ScriptAlertChannelFactory(); ScriptAlertChannelFactory scriptAlertChannelFactory = new ScriptAlertChannelFactory();
List<PluginParams> params = scriptAlertChannelFactory.getParams(); List<PluginParams> params = scriptAlertChannelFactory.getParams();
JSONUtils.toJsonString(params);
Assert.assertEquals(3, params.size()); Assert.assertEquals(3, params.size());
} }

7
dolphinscheduler-alert-plugin/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannel.java

@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.spi.alert.AlertChannel;
import org.apache.dolphinscheduler.spi.alert.AlertData; import org.apache.dolphinscheduler.spi.alert.AlertData;
import org.apache.dolphinscheduler.spi.alert.AlertInfo; import org.apache.dolphinscheduler.spi.alert.AlertInfo;
import org.apache.dolphinscheduler.spi.alert.AlertResult; import org.apache.dolphinscheduler.spi.alert.AlertResult;
import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
import java.util.Map; import java.util.Map;
@ -33,8 +32,10 @@ public class WeChatAlertChannel implements AlertChannel {
@Override @Override
public AlertResult process(AlertInfo info) { public AlertResult process(AlertInfo info) {
AlertData alertData = info.getAlertData(); AlertData alertData = info.getAlertData();
String alertParams = info.getAlertParams(); Map<String, String> paramsMap = info.getAlertParams();
Map<String, String> paramsMap = PluginParamsTransfer.getPluginParamsMap(alertParams); if (null == paramsMap) {
return new AlertResult("false", "we chat params is null");
}
return new WeChatSender(paramsMap).sendEnterpriseWeChat(alertData.getTitle(), alertData.getContent()); return new WeChatSender(paramsMap).sendEnterpriseWeChat(alertData.getTitle(), alertData.getContent());
} }

1
dolphinscheduler-alert-plugin/pom.xml

@ -35,6 +35,7 @@
<module>dolphinscheduler-alert-dingtalk</module> <module>dolphinscheduler-alert-dingtalk</module>
<module>dolphinscheduler-alert-script</module> <module>dolphinscheduler-alert-script</module>
<module>dolphinscheduler-alert-http</module> <module>dolphinscheduler-alert-http</module>
<module>dolphinscheduler-alert-feishu</module>
</modules> </modules>

7
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java

@ -29,7 +29,6 @@ import org.apache.dolphinscheduler.alert.utils.PropertyUtils;
import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.DaoFactory; import org.apache.dolphinscheduler.dao.DaoFactory;
import org.apache.dolphinscheduler.dao.PluginDao;
import org.apache.dolphinscheduler.dao.entity.Alert; import org.apache.dolphinscheduler.dao.entity.Alert;
import org.apache.dolphinscheduler.remote.NettyRemotingServer; import org.apache.dolphinscheduler.remote.NettyRemotingServer;
import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.CommandType;
@ -53,8 +52,6 @@ public class AlertServer {
*/ */
private AlertDao alertDao = DaoFactory.getDaoInstance(AlertDao.class); private AlertDao alertDao = DaoFactory.getDaoInstance(AlertDao.class);
private PluginDao pluginDao = DaoFactory.getDaoInstance(PluginDao.class);
private AlertSender alertSender; private AlertSender alertSender;
private static AlertServer instance; private static AlertServer instance;
@ -114,7 +111,7 @@ public class AlertServer {
NettyServerConfig serverConfig = new NettyServerConfig(); NettyServerConfig serverConfig = new NettyServerConfig();
serverConfig.setListenPort(ALERT_RPC_PORT); serverConfig.setListenPort(ALERT_RPC_PORT);
this.server = new NettyRemotingServer(serverConfig); this.server = new NettyRemotingServer(serverConfig);
this.server.registerProcessor(CommandType.ALERT_SEND_REQUEST, new AlertRequestProcessor(alertDao, alertPluginManager, pluginDao)); this.server.registerProcessor(CommandType.ALERT_SEND_REQUEST, new AlertRequestProcessor(alertDao, alertPluginManager));
this.server.start(); this.server.start();
} }
@ -133,7 +130,7 @@ public class AlertServer {
logger.warn("No Alert Plugin . Can not send alert info. "); logger.warn("No Alert Plugin . Can not send alert info. ");
} else { } else {
List<Alert> alerts = alertDao.listWaitExecutionAlert(); List<Alert> alerts = alertDao.listWaitExecutionAlert();
alertSender = new AlertSender(alerts, alertDao, alertPluginManager, pluginDao); alertSender = new AlertSender(alerts, alertDao, alertPluginManager);
alertSender.run(); alertSender.run();
} }
} }

55
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EmailManager.java

@ -1,55 +0,0 @@
///*
// * Licensed to the Apache Software Foundation (ASF) under one or more
// * contributor license agreements. See the NOTICE file distributed with
// * this work for additional information regarding copyright ownership.
// * The ASF licenses this file to You under the Apache License, Version 2.0
// * (the "License"); you may not use this file except in compliance with
// * the License. You may obtain a copy of the License at
// *
// * http://www.apache.org/licenses/LICENSE-2.0
// *
// * Unless required by applicable law or agreed to in writing, software
// * distributed under the License is distributed on an "AS IS" BASIS,
// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// * See the License for the specific language governing permissions and
// * limitations under the License.
// */
//package org.apache.dolphinscheduler.alert.manager;
//
//import org.apache.dolphinscheduler.alert.utils.MailUtils;
//
//import java.util.List;
//import java.util.Map;
//
///**
// * email send manager
// */
//public class EmailManager {
// /**
// * email send
// * @param receiversList the receiver list
// * @param receiversCcList the cc List
// * @param title the title
// * @param content the content
// * @param showType the showType
// * @return the send result
// */
// public Map<String,Object> send(List<String> receiversList,List<String> receiversCcList,String title,String content,String showType){
//
// return MailUtils.sendMails(receiversList, receiversCcList, title, content, showType);
// }
//
// /**
// * msg send
// * @param receiversList the receiver list
// * @param title the title
// * @param content the content
// * @param showType the showType
// * @return the send result
// */
// public Map<String,Object> send(List<String> receiversList,String title,String content,String showType){
//
// return MailUtils.sendMails(receiversList,title, content, showType);
// }
//
//}

13
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/plugin/AlertPluginManager.java

@ -31,6 +31,7 @@ import org.apache.dolphinscheduler.spi.classloader.ThreadContextClassLoader;
import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer; import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
import org.apache.dolphinscheduler.spi.params.base.PluginParams; import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
@ -47,6 +48,11 @@ public class AlertPluginManager extends AbstractDolphinPluginManager {
private final Map<String, AlertChannelFactory> alertChannelFactoryMap = new ConcurrentHashMap<>(); private final Map<String, AlertChannelFactory> alertChannelFactoryMap = new ConcurrentHashMap<>();
private final Map<String, AlertChannel> alertChannelMap = new ConcurrentHashMap<>(); private final Map<String, AlertChannel> alertChannelMap = new ConcurrentHashMap<>();
/**
* k->pluginDefineId v->pluginDefineName
*/
private final Map<Integer, String> pluginDefineMap = new HashMap<>();
public void addAlertChannelFactory(AlertChannelFactory alertChannelFactory) { public void addAlertChannelFactory(AlertChannelFactory alertChannelFactory) {
requireNonNull(alertChannelFactory, "alertChannelFactory is null"); requireNonNull(alertChannelFactory, "alertChannelFactory is null");
@ -83,6 +89,10 @@ public class AlertPluginManager extends AbstractDolphinPluginManager {
return alertChannelMap; return alertChannelMap;
} }
public String getPluginNameById(int id) {
return pluginDefineMap.get(id);
}
@Override @Override
public void installPlugin(DolphinSchedulerPlugin dolphinSchedulerPlugin) { public void installPlugin(DolphinSchedulerPlugin dolphinSchedulerPlugin) {
for (AlertChannelFactory alertChannelFactory : dolphinSchedulerPlugin.getAlertChannelFactorys()) { for (AlertChannelFactory alertChannelFactory : dolphinSchedulerPlugin.getAlertChannelFactorys()) {
@ -93,7 +103,8 @@ public class AlertPluginManager extends AbstractDolphinPluginManager {
String paramsJson = PluginParamsTransfer.transferParamsToJson(params); String paramsJson = PluginParamsTransfer.transferParamsToJson(params);
PluginDefine pluginDefine = new PluginDefine(nameEn, PluginType.ALERT.getDesc(), paramsJson); PluginDefine pluginDefine = new PluginDefine(nameEn, PluginType.ALERT.getDesc(), paramsJson);
pluginDao.addOrUpdatePluginDefine(pluginDefine); int id = pluginDao.addOrUpdatePluginDefine(pluginDefine);
pluginDefineMap.put(id, pluginDefine.getPluginName());
} }
} }
} }

9
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/processor/AlertRequestProcessor.java

@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager;
import org.apache.dolphinscheduler.alert.runner.AlertSender; import org.apache.dolphinscheduler.alert.runner.AlertSender;
import org.apache.dolphinscheduler.common.utils.Preconditions; import org.apache.dolphinscheduler.common.utils.Preconditions;
import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.PluginDao;
import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.CommandType;
import org.apache.dolphinscheduler.remote.command.alert.AlertSendRequestCommand; import org.apache.dolphinscheduler.remote.command.alert.AlertSendRequestCommand;
@ -35,18 +34,16 @@ import org.slf4j.LoggerFactory;
import io.netty.channel.Channel; import io.netty.channel.Channel;
/** /**
* alert request processor * alert request processor
*/ */
public class AlertRequestProcessor implements NettyRequestProcessor { public class AlertRequestProcessor implements NettyRequestProcessor {
private final Logger logger = LoggerFactory.getLogger(AlertRequestProcessor.class); private final Logger logger = LoggerFactory.getLogger(AlertRequestProcessor.class);
private AlertDao alertDao; private AlertDao alertDao;
private PluginDao pluginDao;
private AlertPluginManager alertPluginManager; private AlertPluginManager alertPluginManager;
public AlertRequestProcessor(AlertDao alertDao, AlertPluginManager alertPluginManager, PluginDao pluginDao) { public AlertRequestProcessor(AlertDao alertDao, AlertPluginManager alertPluginManager) {
this.alertDao = alertDao; this.alertDao = alertDao;
this.pluginDao = pluginDao;
this.alertPluginManager = alertPluginManager; this.alertPluginManager = alertPluginManager;
} }
@ -59,7 +56,7 @@ public class AlertRequestProcessor implements NettyRequestProcessor {
command.getBody(), AlertSendRequestCommand.class); command.getBody(), AlertSendRequestCommand.class);
logger.info("received command : {}", alertSendRequestCommand); logger.info("received command : {}", alertSendRequestCommand);
AlertSender alertSender = new AlertSender(alertDao, alertPluginManager, pluginDao); AlertSender alertSender = new AlertSender(alertDao, alertPluginManager);
AlertSendResponseCommand alertSendResponseCommand = alertSender.syncHandler(alertSendRequestCommand.getGroupId(), alertSendRequestCommand.getTitle(), alertSendRequestCommand.getContent()); AlertSendResponseCommand alertSendResponseCommand = alertSender.syncHandler(alertSendRequestCommand.getGroupId(), alertSendRequestCommand.getTitle(), alertSendRequestCommand.getContent());
channel.writeAndFlush(alertSendResponseCommand.convert2Command(command.getOpaque())); channel.writeAndFlush(alertSendResponseCommand.convert2Command(command.getOpaque()));

39
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java

@ -20,8 +20,8 @@ package org.apache.dolphinscheduler.alert.runner;
import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager; import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager;
import org.apache.dolphinscheduler.common.enums.AlertStatus; import org.apache.dolphinscheduler.common.enums.AlertStatus;
import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.PluginDao;
import org.apache.dolphinscheduler.dao.entity.Alert; import org.apache.dolphinscheduler.dao.entity.Alert;
import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance;
import org.apache.dolphinscheduler.remote.command.alert.AlertSendResponseCommand; import org.apache.dolphinscheduler.remote.command.alert.AlertSendResponseCommand;
@ -33,6 +33,7 @@ import org.apache.dolphinscheduler.spi.alert.AlertResult;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -46,25 +47,22 @@ public class AlertSender {
private List<Alert> alertList; private List<Alert> alertList;
private AlertDao alertDao; private AlertDao alertDao;
private PluginDao pluginDao;
private AlertPluginManager alertPluginManager; private AlertPluginManager alertPluginManager;
public AlertSender(AlertPluginManager alertPluginManager) { public AlertSender(AlertPluginManager alertPluginManager) {
this.alertPluginManager = alertPluginManager; this.alertPluginManager = alertPluginManager;
} }
public AlertSender(AlertDao alertDao, AlertPluginManager alertPluginManager, PluginDao pluginDao) { public AlertSender(AlertDao alertDao, AlertPluginManager alertPluginManager) {
super(); super();
this.alertDao = alertDao; this.alertDao = alertDao;
this.pluginDao = pluginDao;
this.alertPluginManager = alertPluginManager; this.alertPluginManager = alertPluginManager;
} }
public AlertSender(List<Alert> alertList, AlertDao alertDao, AlertPluginManager alertPluginManager, PluginDao pluginDao) { public AlertSender(List<Alert> alertList, AlertDao alertDao, AlertPluginManager alertPluginManager) {
super(); super();
this.alertList = alertList; this.alertList = alertList;
this.alertDao = alertDao; this.alertDao = alertDao;
this.pluginDao = pluginDao;
this.alertPluginManager = alertPluginManager; this.alertPluginManager = alertPluginManager;
} }
@ -75,13 +73,14 @@ public class AlertSender {
List<AlertPluginInstance> alertInstanceList = alertDao.listInstanceByAlertGroupId(alertGroupId); List<AlertPluginInstance> alertInstanceList = alertDao.listInstanceByAlertGroupId(alertGroupId);
if (CollectionUtils.isEmpty(alertInstanceList)) { if (CollectionUtils.isEmpty(alertInstanceList)) {
logger.error("send alert msg fail,no bind plugin instance."); logger.error("send alert msg fail,no bind plugin instance.");
return; alertDao.updateAlert(AlertStatus.EXECUTION_FAILURE, "no bind plugin instance", alert.getId());
continue;
} }
AlertData alertData = new AlertData(); AlertData alertData = new AlertData();
alertData.setId(alert.getId()) alertData.setId(alert.getId())
.setContent(alert.getContent()) .setContent(alert.getContent())
.setLog(alert.getLog()) .setLog(alert.getLog())
.setTitle(alert.getTitle()); .setTitle(alert.getTitle());
for (AlertPluginInstance instance : alertInstanceList) { for (AlertPluginInstance instance : alertInstanceList) {
@ -106,8 +105,8 @@ public class AlertSender {
List<AlertPluginInstance> alertInstanceList = alertDao.listInstanceByAlertGroupId(alertGroupId); List<AlertPluginInstance> alertInstanceList = alertDao.listInstanceByAlertGroupId(alertGroupId);
AlertData alertData = new AlertData(); AlertData alertData = new AlertData();
alertData.setContent(title) alertData.setContent(content)
.setTitle(content); .setTitle(title);
boolean sendResponseStatus = true; boolean sendResponseStatus = true;
List<AlertSendResponseResult> sendResponseResults = new ArrayList<>(); List<AlertSendResponseResult> sendResponseResults = new ArrayList<>();
@ -126,7 +125,7 @@ public class AlertSender {
for (AlertPluginInstance instance : alertInstanceList) { for (AlertPluginInstance instance : alertInstanceList) {
AlertResult alertResult = this.alertResultHandler(instance, alertData); AlertResult alertResult = this.alertResultHandler(instance, alertData);
AlertSendResponseResult alertSendResponseResult = new AlertSendResponseResult( AlertSendResponseResult alertSendResponseResult = new AlertSendResponseResult(
Boolean.parseBoolean(String.valueOf(alertResult.getStatus())), alertResult.getMessage()); Boolean.parseBoolean(String.valueOf(alertResult.getStatus())), alertResult.getMessage());
sendResponseStatus = sendResponseStatus && alertSendResponseResult.getStatus(); sendResponseStatus = sendResponseStatus && alertSendResponseResult.getStatus();
sendResponseResults.add(alertSendResponseResult); sendResponseResults.add(alertSendResponseResult);
} }
@ -142,7 +141,7 @@ public class AlertSender {
* @return AlertResult * @return AlertResult
*/ */
private AlertResult alertResultHandler(AlertPluginInstance instance, AlertData alertData) { private AlertResult alertResultHandler(AlertPluginInstance instance, AlertData alertData) {
String pluginName = pluginDao.getPluginDefineById(instance.getPluginDefineId()).getPluginName(); String pluginName = alertPluginManager.getPluginNameById(instance.getPluginDefineId());
AlertChannel alertChannel = alertPluginManager.getAlertChannelMap().get(pluginName); AlertChannel alertChannel = alertPluginManager.getAlertChannelMap().get(pluginName);
AlertResult alertResultExtend = new AlertResult(); AlertResult alertResultExtend = new AlertResult();
String pluginInstanceName = instance.getInstanceName(); String pluginInstanceName = instance.getInstanceName();
@ -156,8 +155,16 @@ public class AlertSender {
AlertInfo alertInfo = new AlertInfo(); AlertInfo alertInfo = new AlertInfo();
alertInfo.setAlertData(alertData); alertInfo.setAlertData(alertData);
alertInfo.setAlertParams(instance.getPluginInstanceParams()); Map<String, String> paramsMap = JSONUtils.toMap(instance.getPluginInstanceParams());
AlertResult alertResult = alertChannel.process(alertInfo); alertInfo.setAlertParams(paramsMap);
AlertResult alertResult;
try {
alertResult = alertChannel.process(alertInfo);
} catch (Exception e) {
alertResult = new AlertResult("false", e.getMessage());
logger.error("send alert error alert data id :{},", alertData.getId(), e);
}
if (alertResult == null) { if (alertResult == null) {
String message = String.format("Alert Plugin %s send error : return alertResult value is null", pluginInstanceName); String message = String.format("Alert Plugin %s send error : return alertResult value is null", pluginInstanceName);

2
dolphinscheduler-alert/src/main/resources/alert.properties

@ -21,7 +21,7 @@
#eg : Alert Server Listener port #eg : Alert Server Listener port
#alert.plugin.dir config the Alert Plugin dir . AlertServer while find and load the Alert Plugin Jar from this dir when deploy and start AlertServer on the server . #alert.plugin.dir config the Alert Plugin dir . AlertServer while find and load the Alert Plugin Jar from this dir when deploy and start AlertServer on the server .
#eg :alert.plugin.dir=/opt/soft/spi/lib/plugin/alert alert.plugin.dir=./lib/plugin/alert
#maven.local.repository=/Users/gaojun/Documents/jianguoyun/localRepository #maven.local.repository=/Users/gaojun/Documents/jianguoyun/localRepository

9
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/AlertServerTest.java

@ -34,12 +34,13 @@ import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito; import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner; import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class) @RunWith(PowerMockRunner.class)
@PrepareForTest({AlertServer.class,DaoFactory.class}) @PrepareForTest({AlertServer.class, DaoFactory.class})
public class AlertServerTest { public class AlertServerTest {
@Before @Before
@ -61,7 +62,8 @@ public class AlertServerTest {
AlertPluginManager alertPluginManager = PowerMockito.mock(AlertPluginManager.class); AlertPluginManager alertPluginManager = PowerMockito.mock(AlertPluginManager.class);
PowerMockito.whenNew(AlertPluginManager.class).withNoArguments().thenReturn(alertPluginManager); PowerMockito.whenNew(AlertPluginManager.class).withNoArguments().thenReturn(alertPluginManager);
ConcurrentHashMap alertChannelMap = new ConcurrentHashMap<>(); ConcurrentHashMap alertChannelMap = new ConcurrentHashMap<>();
alertChannelMap.put("pluginName",alertChannelMock); alertChannelMap.put("pluginName", alertChannelMock);
PowerMockito.when(alertPluginManager.getPluginNameById(Mockito.anyInt())).thenReturn("pluginName");
PowerMockito.when(alertPluginManager.getAlertChannelMap()).thenReturn(alertChannelMap); PowerMockito.when(alertPluginManager.getAlertChannelMap()).thenReturn(alertChannelMap);
DolphinPluginManagerConfig alertPluginManagerConfig = PowerMockito.mock(DolphinPluginManagerConfig.class); DolphinPluginManagerConfig alertPluginManagerConfig = PowerMockito.mock(DolphinPluginManagerConfig.class);
@ -79,7 +81,8 @@ public class AlertServerTest {
Assert.assertNotNull(alertServer); Assert.assertNotNull(alertServer);
new Thread(() -> { new Thread(() -> {
alertServer.start(); }) alertServer.start();
})
.start(); .start();
Thread.sleep(5 * Constants.ALERT_SCAN_INTERVAL); Thread.sleep(5 * Constants.ALERT_SCAN_INTERVAL);

2
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java

@ -136,7 +136,7 @@ public class EmailAlertPluginTest {
alertPluginInstance.setPluginInstanceParams(getEmailAlertParams()); alertPluginInstance.setPluginInstanceParams(getEmailAlertParams());
alertDao.getAlertPluginInstanceMapper().insert(alertPluginInstance); alertDao.getAlertPluginInstanceMapper().insert(alertPluginInstance);
AlertSender alertSender = new AlertSender(alertList, alertDao, alertPluginManager, pluginDao); AlertSender alertSender = new AlertSender(alertList, alertDao, alertPluginManager);
alertSender.run(); alertSender.run();
Alert alertResult = alertDao.getAlertMapper().selectById(alert1.getId()); Alert alertResult = alertDao.getAlertMapper().selectById(alert1.getId());

11
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/processor/AlertRequestProcessorTest.java

@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.alert.processor;
import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager; import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager;
import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.PluginDao;
import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.CommandType;
import org.apache.dolphinscheduler.remote.command.alert.AlertSendRequestCommand; import org.apache.dolphinscheduler.remote.command.alert.AlertSendRequestCommand;
@ -37,7 +36,6 @@ import io.netty.channel.Channel;
public class AlertRequestProcessorTest { public class AlertRequestProcessorTest {
private AlertDao alertDao; private AlertDao alertDao;
private PluginDao pluginDao;
private AlertPluginManager alertPluginManager; private AlertPluginManager alertPluginManager;
private AlertRequestProcessor alertRequestProcessor; private AlertRequestProcessor alertRequestProcessor;
@ -45,17 +43,16 @@ public class AlertRequestProcessorTest {
@Before @Before
public void before() { public void before() {
alertDao = PowerMockito.mock(AlertDao.class); alertDao = PowerMockito.mock(AlertDao.class);
pluginDao = PowerMockito.mock(PluginDao.class);
alertPluginManager = PowerMockito.mock(AlertPluginManager.class); alertPluginManager = PowerMockito.mock(AlertPluginManager.class);
alertRequestProcessor = new AlertRequestProcessor(alertDao,alertPluginManager,pluginDao); alertRequestProcessor = new AlertRequestProcessor(alertDao, alertPluginManager);
} }
@Test @Test
public void testProcess() { public void testProcess() {
Channel channel = PowerMockito.mock(Channel.class); Channel channel = PowerMockito.mock(Channel.class);
AlertSendRequestCommand alertSendRequestCommand = new AlertSendRequestCommand(1,"title","content"); AlertSendRequestCommand alertSendRequestCommand = new AlertSendRequestCommand(1, "title", "content");
Command reqCommand = alertSendRequestCommand.convert2Command(); Command reqCommand = alertSendRequestCommand.convert2Command();
Assert.assertEquals(CommandType.ALERT_SEND_REQUEST,reqCommand.getType()); Assert.assertEquals(CommandType.ALERT_SEND_REQUEST, reqCommand.getType());
alertRequestProcessor.process(channel,reqCommand); alertRequestProcessor.process(channel, reqCommand);
} }
} }

36
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java

@ -67,7 +67,7 @@ public class AlertSenderTest {
int alertGroupId = 1; int alertGroupId = 1;
String title = "alert mail test title"; String title = "alert mail test title";
String content = "alert mail test content"; String content = "alert mail test content";
alertSender = new AlertSender(alertDao,alertPluginManager,pluginDao); alertSender = new AlertSender(alertDao, alertPluginManager);
//1.alert instance does not exist //1.alert instance does not exist
PowerMockito.when(alertDao.listInstanceByAlertGroupId(alertGroupId)).thenReturn(null); PowerMockito.when(alertDao.listInstanceByAlertGroupId(alertGroupId)).thenReturn(null);
@ -75,7 +75,7 @@ public class AlertSenderTest {
AlertSendResponseCommand alertSendResponseCommand = alertSender.syncHandler(alertGroupId, title, content); AlertSendResponseCommand alertSendResponseCommand = alertSender.syncHandler(alertGroupId, title, content);
Assert.assertFalse(alertSendResponseCommand.getResStatus()); Assert.assertFalse(alertSendResponseCommand.getResStatus());
alertSendResponseCommand.getResResults().forEach(result -> alertSendResponseCommand.getResResults().forEach(result ->
logger.info("alert send response result, status:{}, message:{}",result.getStatus(),result.getMessage())); logger.info("alert send response result, status:{}, message:{}", result.getStatus(), result.getMessage()));
//2.alert plugin does not exist //2.alert plugin does not exist
int pluginDefineId = 1; int pluginDefineId = 1;
@ -83,30 +83,31 @@ public class AlertSenderTest {
String pluginInstanceName = "alert-instance-mail"; String pluginInstanceName = "alert-instance-mail";
List<AlertPluginInstance> alertInstanceList = new ArrayList<>(); List<AlertPluginInstance> alertInstanceList = new ArrayList<>();
AlertPluginInstance alertPluginInstance = new AlertPluginInstance( AlertPluginInstance alertPluginInstance = new AlertPluginInstance(
pluginDefineId,pluginInstanceParams,pluginInstanceName); pluginDefineId, pluginInstanceParams, pluginInstanceName);
alertInstanceList.add(alertPluginInstance); alertInstanceList.add(alertPluginInstance);
PowerMockito.when(alertDao.listInstanceByAlertGroupId(1)).thenReturn(alertInstanceList); PowerMockito.when(alertDao.listInstanceByAlertGroupId(1)).thenReturn(alertInstanceList);
String pluginName = "alert-plugin-mail"; String pluginName = "alert-plugin-mail";
PluginDefine pluginDefine = new PluginDefine(pluginName,"1",null); PluginDefine pluginDefine = new PluginDefine(pluginName, "1", null);
PowerMockito.when(pluginDao.getPluginDefineById(pluginDefineId)).thenReturn(pluginDefine); PowerMockito.when(pluginDao.getPluginDefineById(pluginDefineId)).thenReturn(pluginDefine);
alertSendResponseCommand = alertSender.syncHandler(alertGroupId, title, content); alertSendResponseCommand = alertSender.syncHandler(alertGroupId, title, content);
Assert.assertFalse(alertSendResponseCommand.getResStatus()); Assert.assertFalse(alertSendResponseCommand.getResStatus());
alertSendResponseCommand.getResResults().forEach(result -> alertSendResponseCommand.getResResults().forEach(result ->
logger.info("alert send response result, status:{}, message:{}",result.getStatus(),result.getMessage())); logger.info("alert send response result, status:{}, message:{}", result.getStatus(), result.getMessage()));
//3.alert result value is null //3.alert result value is null
AlertChannel alertChannelMock = PowerMockito.mock(AlertChannel.class); AlertChannel alertChannelMock = PowerMockito.mock(AlertChannel.class);
PowerMockito.when(alertChannelMock.process(Mockito.any())).thenReturn(null); PowerMockito.when(alertChannelMock.process(Mockito.any())).thenReturn(null);
Map<String, AlertChannel> alertChannelMap = new ConcurrentHashMap<>(); Map<String, AlertChannel> alertChannelMap = new ConcurrentHashMap<>();
alertChannelMap.put(pluginName,alertChannelMock); alertChannelMap.put(pluginName, alertChannelMock);
PowerMockito.when(alertPluginManager.getAlertChannelMap()).thenReturn(alertChannelMap); PowerMockito.when(alertPluginManager.getAlertChannelMap()).thenReturn(alertChannelMap);
PowerMockito.when(alertPluginManager.getPluginNameById(Mockito.anyInt())).thenReturn("alert-plugin-mail");
alertSendResponseCommand = alertSender.syncHandler(alertGroupId, title, content); alertSendResponseCommand = alertSender.syncHandler(alertGroupId, title, content);
Assert.assertFalse(alertSendResponseCommand.getResStatus()); Assert.assertFalse(alertSendResponseCommand.getResStatus());
alertSendResponseCommand.getResResults().forEach(result -> alertSendResponseCommand.getResResults().forEach(result ->
logger.info("alert send response result, status:{}, message:{}",result.getStatus(),result.getMessage())); logger.info("alert send response result, status:{}, message:{}", result.getStatus(), result.getMessage()));
//4.abnormal information inside the alert plug-in code //4.abnormal information inside the alert plug-in code
AlertResult alertResult = new AlertResult(); AlertResult alertResult = new AlertResult();
@ -114,27 +115,27 @@ public class AlertSenderTest {
alertResult.setMessage("Abnormal information inside the alert plug-in code"); alertResult.setMessage("Abnormal information inside the alert plug-in code");
PowerMockito.when(alertChannelMock.process(Mockito.any())).thenReturn(alertResult); PowerMockito.when(alertChannelMock.process(Mockito.any())).thenReturn(alertResult);
alertChannelMap = new ConcurrentHashMap<>(); alertChannelMap = new ConcurrentHashMap<>();
alertChannelMap.put(pluginName,alertChannelMock); alertChannelMap.put(pluginName, alertChannelMock);
PowerMockito.when(alertPluginManager.getAlertChannelMap()).thenReturn(alertChannelMap); PowerMockito.when(alertPluginManager.getAlertChannelMap()).thenReturn(alertChannelMap);
alertSendResponseCommand = alertSender.syncHandler(alertGroupId, title, content); alertSendResponseCommand = alertSender.syncHandler(alertGroupId, title, content);
Assert.assertFalse(alertSendResponseCommand.getResStatus()); Assert.assertFalse(alertSendResponseCommand.getResStatus());
alertSendResponseCommand.getResResults().forEach(result -> alertSendResponseCommand.getResResults().forEach(result ->
logger.info("alert send response result, status:{}, message:{}",result.getStatus(),result.getMessage())); logger.info("alert send response result, status:{}, message:{}", result.getStatus(), result.getMessage()));
//5.alert plugin send success //5.alert plugin send success
alertResult = new AlertResult(); alertResult = new AlertResult();
alertResult.setStatus(String.valueOf(true)); alertResult.setStatus(String.valueOf(true));
alertResult.setMessage(String.format("Alert Plugin %s send success",pluginInstanceName)); alertResult.setMessage(String.format("Alert Plugin %s send success", pluginInstanceName));
PowerMockito.when(alertChannelMock.process(Mockito.any())).thenReturn(alertResult); PowerMockito.when(alertChannelMock.process(Mockito.any())).thenReturn(alertResult);
alertChannelMap = new ConcurrentHashMap<>(); alertChannelMap = new ConcurrentHashMap<>();
alertChannelMap.put(pluginName,alertChannelMock); alertChannelMap.put(pluginName, alertChannelMock);
PowerMockito.when(alertPluginManager.getAlertChannelMap()).thenReturn(alertChannelMap); PowerMockito.when(alertPluginManager.getAlertChannelMap()).thenReturn(alertChannelMap);
alertSendResponseCommand = alertSender.syncHandler(alertGroupId, title, content); alertSendResponseCommand = alertSender.syncHandler(alertGroupId, title, content);
Assert.assertTrue(alertSendResponseCommand.getResStatus()); Assert.assertTrue(alertSendResponseCommand.getResStatus());
alertSendResponseCommand.getResResults().forEach(result -> alertSendResponseCommand.getResResults().forEach(result ->
logger.info("alert send response result, status:{}, message:{}",result.getStatus(),result.getMessage())); logger.info("alert send response result, status:{}, message:{}", result.getStatus(), result.getMessage()));
} }
@ -150,28 +151,29 @@ public class AlertSenderTest {
alert.setContent(content); alert.setContent(content);
alertList.add(alert); alertList.add(alert);
alertSender = new AlertSender(alertList,alertDao,alertPluginManager,pluginDao); alertSender = new AlertSender(alertList, alertDao, alertPluginManager);
int pluginDefineId = 1; int pluginDefineId = 1;
String pluginInstanceParams = "alert-instance-mail-params"; String pluginInstanceParams = "alert-instance-mail-params";
String pluginInstanceName = "alert-instance-mail"; String pluginInstanceName = "alert-instance-mail";
List<AlertPluginInstance> alertInstanceList = new ArrayList<>(); List<AlertPluginInstance> alertInstanceList = new ArrayList<>();
AlertPluginInstance alertPluginInstance = new AlertPluginInstance( AlertPluginInstance alertPluginInstance = new AlertPluginInstance(
pluginDefineId,pluginInstanceParams,pluginInstanceName); pluginDefineId, pluginInstanceParams, pluginInstanceName);
alertInstanceList.add(alertPluginInstance); alertInstanceList.add(alertPluginInstance);
PowerMockito.when(alertDao.listInstanceByAlertGroupId(alertGroupId)).thenReturn(alertInstanceList); PowerMockito.when(alertDao.listInstanceByAlertGroupId(alertGroupId)).thenReturn(alertInstanceList);
String pluginName = "alert-plugin-mail"; String pluginName = "alert-plugin-mail";
PluginDefine pluginDefine = new PluginDefine(pluginName,"1",null); PluginDefine pluginDefine = new PluginDefine(pluginName, "1", null);
PowerMockito.when(pluginDao.getPluginDefineById(pluginDefineId)).thenReturn(pluginDefine); PowerMockito.when(pluginDao.getPluginDefineById(pluginDefineId)).thenReturn(pluginDefine);
PowerMockito.when(alertPluginManager.getPluginNameById(1)).thenReturn("alert-instance-mail");
AlertResult alertResult = new AlertResult(); AlertResult alertResult = new AlertResult();
alertResult.setStatus(String.valueOf(true)); alertResult.setStatus(String.valueOf(true));
alertResult.setMessage(String.format("Alert Plugin %s send success",pluginInstanceName)); alertResult.setMessage(String.format("Alert Plugin %s send success", pluginInstanceName));
AlertChannel alertChannelMock = PowerMockito.mock(AlertChannel.class); AlertChannel alertChannelMock = PowerMockito.mock(AlertChannel.class);
PowerMockito.when(alertChannelMock.process(Mockito.any())).thenReturn(alertResult); PowerMockito.when(alertChannelMock.process(Mockito.any())).thenReturn(alertResult);
ConcurrentHashMap alertChannelMap = new ConcurrentHashMap<>(); ConcurrentHashMap alertChannelMap = new ConcurrentHashMap<>();
alertChannelMap.put(pluginName,alertChannelMock); alertChannelMap.put(pluginName, alertChannelMock);
PowerMockito.when(alertPluginManager.getAlertChannelMap()).thenReturn(alertChannelMap); PowerMockito.when(alertPluginManager.getAlertChannelMap()).thenReturn(alertChannelMap);
Assert.assertTrue(Boolean.parseBoolean(alertResult.getStatus())); Assert.assertTrue(Boolean.parseBoolean(alertResult.getStatus()));
alertSender.run(); alertSender.run();

42
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java

@ -96,10 +96,14 @@ public class DataSourceController extends BaseController {
@ApiImplicitParam(name = "host", value = "DATA_SOURCE_HOST", required = true, dataType = "String"), @ApiImplicitParam(name = "host", value = "DATA_SOURCE_HOST", required = true, dataType = "String"),
@ApiImplicitParam(name = "port", value = "DATA_SOURCE_PORT", required = true, dataType = "String"), @ApiImplicitParam(name = "port", value = "DATA_SOURCE_PORT", required = true, dataType = "String"),
@ApiImplicitParam(name = "database", value = "DATABASE_NAME", required = true, dataType = "String"), @ApiImplicitParam(name = "database", value = "DATABASE_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "principal", value = "DATA_SOURCE_PRINCIPAL", dataType = "String"),
@ApiImplicitParam(name = "userName", value = "USER_NAME", required = true, dataType = "String"), @ApiImplicitParam(name = "userName", value = "USER_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "password", value = "PASSWORD", dataType = "String"), @ApiImplicitParam(name = "password", value = "PASSWORD", dataType = "String"),
@ApiImplicitParam(name = "connectType", value = "CONNECT_TYPE", dataType = "DbConnectType"), @ApiImplicitParam(name = "connectType", value = "CONNECT_TYPE", dataType = "DbConnectType"),
@ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String") @ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String"),
@ApiImplicitParam(name = "javaSecurityKrb5Conf", value = "DATA_SOURCE_KERBEROS_KRB5_CONF", dataType = "String"),
@ApiImplicitParam(name = "loginUserKeytabUsername", value = "DATA_SOURCE_KERBEROS_KEYTAB_USERNAME", dataType = "String"),
@ApiImplicitParam(name = "loginUserKeytabPath", value = "DATA_SOURCE_KERBEROS_KEYTAB_PATH", dataType = "String")
}) })
@PostMapping(value = "/create") @PostMapping(value = "/create")
@ResponseStatus(HttpStatus.CREATED) @ResponseStatus(HttpStatus.CREATED)
@ -115,10 +119,14 @@ public class DataSourceController extends BaseController {
@RequestParam(value = "userName") String userName, @RequestParam(value = "userName") String userName,
@RequestParam(value = "password") String password, @RequestParam(value = "password") String password,
@RequestParam(value = "connectType") DbConnectType connectType, @RequestParam(value = "connectType") DbConnectType connectType,
@RequestParam(value = "other") String other) { @RequestParam(value = "other") String other,
@RequestParam(value = "javaSecurityKrb5Conf", required = false) String javaSecurityKrb5Conf,
@RequestParam(value = "loginUserKeytabUsername", required = false) String loginUserKeytabUsername,
@RequestParam(value = "loginUserKeytabPath", required = false) String loginUserKeytabPath) {
logger.info("login user {} create datasource name: {}, note: {}, type: {}, host: {}, port: {}, database : {}, principal: {}, userName : {}, connectType: {}, other: {}", logger.info("login user {} create datasource name: {}, note: {}, type: {}, host: {}, port: {}, database : {}, principal: {}, userName : {}, connectType: {}, other: {}",
loginUser.getUserName(), name, note, type, host, port, database, principal, userName, connectType, other); loginUser.getUserName(), name, note, type, host, port, database, principal, userName, connectType, other);
String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other); String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other,
javaSecurityKrb5Conf, loginUserKeytabUsername, loginUserKeytabPath);
return dataSourceService.createDataSource(loginUser, name, note, type, parameter); return dataSourceService.createDataSource(loginUser, name, note, type, parameter);
} }
@ -149,10 +157,14 @@ public class DataSourceController extends BaseController {
@ApiImplicitParam(name = "host", value = "DATA_SOURCE_HOST", required = true, dataType = "String"), @ApiImplicitParam(name = "host", value = "DATA_SOURCE_HOST", required = true, dataType = "String"),
@ApiImplicitParam(name = "port", value = "DATA_SOURCE_PORT", required = true, dataType = "String"), @ApiImplicitParam(name = "port", value = "DATA_SOURCE_PORT", required = true, dataType = "String"),
@ApiImplicitParam(name = "database", value = "DATABASE_NAME", required = true, dataType = "String"), @ApiImplicitParam(name = "database", value = "DATABASE_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "principal", value = "DATA_SOURCE_PRINCIPAL", dataType = "String"),
@ApiImplicitParam(name = "userName", value = "USER_NAME", required = true, dataType = "String"), @ApiImplicitParam(name = "userName", value = "USER_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "password", value = "PASSWORD", dataType = "String"), @ApiImplicitParam(name = "password", value = "PASSWORD", dataType = "String"),
@ApiImplicitParam(name = "connectType", value = "CONNECT_TYPE", dataType = "DbConnectType"), @ApiImplicitParam(name = "connectType", value = "CONNECT_TYPE", dataType = "DbConnectType"),
@ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String") @ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String"),
@ApiImplicitParam(name = "javaSecurityKrb5Conf", value = "DATA_SOURCE_KERBEROS_KRB5_CONF", dataType = "String"),
@ApiImplicitParam(name = "loginUserKeytabUsername", value = "DATA_SOURCE_KERBEROS_KEYTAB_USERNAME", dataType = "String"),
@ApiImplicitParam(name = "loginUserKeytabPath", value = "DATA_SOURCE_KERBEROS_KEYTAB_PATH", dataType = "String")
}) })
@PostMapping(value = "/update") @PostMapping(value = "/update")
@ResponseStatus(HttpStatus.OK) @ResponseStatus(HttpStatus.OK)
@ -169,10 +181,14 @@ public class DataSourceController extends BaseController {
@RequestParam(value = "userName") String userName, @RequestParam(value = "userName") String userName,
@RequestParam(value = "password") String password, @RequestParam(value = "password") String password,
@RequestParam(value = "connectType") DbConnectType connectType, @RequestParam(value = "connectType") DbConnectType connectType,
@RequestParam(value = "other") String other) { @RequestParam(value = "other") String other,
@RequestParam(value = "javaSecurityKrb5Conf", required = false) String javaSecurityKrb5Conf,
@RequestParam(value = "loginUserKeytabUsername", required = false) String loginUserKeytabUsername,
@RequestParam(value = "loginUserKeytabPath", required = false) String loginUserKeytabPath) {
logger.info("login user {} updateProcessInstance datasource name: {}, note: {}, type: {}, connectType: {}, other: {}", logger.info("login user {} updateProcessInstance datasource name: {}, note: {}, type: {}, connectType: {}, other: {}",
loginUser.getUserName(), name, note, type, connectType, other); loginUser.getUserName(), name, note, type, connectType, other);
String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other); String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other,
javaSecurityKrb5Conf, loginUserKeytabUsername, loginUserKeytabPath);
return dataSourceService.updateDataSource(id, loginUser, name, note, type, parameter); return dataSourceService.updateDataSource(id, loginUser, name, note, type, parameter);
} }
@ -274,10 +290,14 @@ public class DataSourceController extends BaseController {
@ApiImplicitParam(name = "host", value = "DATA_SOURCE_HOST", required = true, dataType = "String"), @ApiImplicitParam(name = "host", value = "DATA_SOURCE_HOST", required = true, dataType = "String"),
@ApiImplicitParam(name = "port", value = "DATA_SOURCE_PORT", required = true, dataType = "String"), @ApiImplicitParam(name = "port", value = "DATA_SOURCE_PORT", required = true, dataType = "String"),
@ApiImplicitParam(name = "database", value = "DATABASE_NAME", required = true, dataType = "String"), @ApiImplicitParam(name = "database", value = "DATABASE_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "principal", value = "DATA_SOURCE_PRINCIPAL", dataType = "String"),
@ApiImplicitParam(name = "userName", value = "USER_NAME", required = true, dataType = "String"), @ApiImplicitParam(name = "userName", value = "USER_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "password", value = "PASSWORD", dataType = "String"), @ApiImplicitParam(name = "password", value = "PASSWORD", dataType = "String"),
@ApiImplicitParam(name = "connectType", value = "CONNECT_TYPE", dataType = "DbConnectType"), @ApiImplicitParam(name = "connectType", value = "CONNECT_TYPE", dataType = "DbConnectType"),
@ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String") @ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String"),
@ApiImplicitParam(name = "javaSecurityKrb5Conf", value = "DATA_SOURCE_KERBEROS_KRB5_CONF", dataType = "String"),
@ApiImplicitParam(name = "loginUserKeytabUsername", value = "DATA_SOURCE_KERBEROS_KEYTAB_USERNAME", dataType = "String"),
@ApiImplicitParam(name = "loginUserKeytabPath", value = "DATA_SOURCE_KERBEROS_KEYTAB_PATH", dataType = "String")
}) })
@PostMapping(value = "/connect") @PostMapping(value = "/connect")
@ResponseStatus(HttpStatus.OK) @ResponseStatus(HttpStatus.OK)
@ -293,10 +313,14 @@ public class DataSourceController extends BaseController {
@RequestParam(value = "userName") String userName, @RequestParam(value = "userName") String userName,
@RequestParam(value = "password") String password, @RequestParam(value = "password") String password,
@RequestParam(value = "connectType") DbConnectType connectType, @RequestParam(value = "connectType") DbConnectType connectType,
@RequestParam(value = "other") String other) { @RequestParam(value = "other") String other,
@RequestParam(value = "javaSecurityKrb5Conf", required = false) String javaSecurityKrb5Conf,
@RequestParam(value = "loginUserKeytabUsername", required = false) String loginUserKeytabUsername,
@RequestParam(value = "loginUserKeytabPath", required = false) String loginUserKeytabPath) {
logger.info("login user {}, connect datasource: {}, note: {}, type: {}, connectType: {}, other: {}", logger.info("login user {}, connect datasource: {}, note: {}, type: {}, connectType: {}, other: {}",
loginUser.getUserName(), name, note, type, connectType, other); loginUser.getUserName(), name, note, type, connectType, other);
String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other); String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other,
javaSecurityKrb5Conf, loginUserKeytabUsername, loginUserKeytabPath);
return dataSourceService.checkConnection(type, parameter); return dataSourceService.checkConnection(type, parameter);
} }

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java

@ -133,7 +133,7 @@ public enum Status {
QUERY_TASK_INSTANCE_LOG_ERROR(10103, "view task instance log error", "查询任务实例日志错误"), QUERY_TASK_INSTANCE_LOG_ERROR(10103, "view task instance log error", "查询任务实例日志错误"),
DOWNLOAD_TASK_INSTANCE_LOG_FILE_ERROR(10104, "download task instance log file error", "下载任务日志文件错误"), DOWNLOAD_TASK_INSTANCE_LOG_FILE_ERROR(10104, "download task instance log file error", "下载任务日志文件错误"),
CREATE_PROCESS_DEFINITION(10105, "create process definition", "创建工作流错误"), CREATE_PROCESS_DEFINITION(10105, "create process definition", "创建工作流错误"),
VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR(10106, "verify process definition name unique error", "工作流定义名称已存在"), VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR(10106, "verify process definition name unique error", "工作流定义名称验证错误"),
UPDATE_PROCESS_DEFINITION_ERROR(10107, "update process definition error", "更新工作流定义错误"), UPDATE_PROCESS_DEFINITION_ERROR(10107, "update process definition error", "更新工作流定义错误"),
RELEASE_PROCESS_DEFINITION_ERROR(10108, "release process definition error", "上线工作流错误"), RELEASE_PROCESS_DEFINITION_ERROR(10108, "release process definition error", "上线工作流错误"),
QUERY_DATAIL_OF_PROCESS_DEFINITION_ERROR(10109, "query datail of process definition error", "查询工作流详细信息错误"), QUERY_DATAIL_OF_PROCESS_DEFINITION_ERROR(10109, "query datail of process definition error", "查询工作流详细信息错误"),
@ -199,6 +199,7 @@ public enum Status {
FORCE_TASK_SUCCESS_ERROR(10165, "force task success error", "强制成功任务实例错误"), FORCE_TASK_SUCCESS_ERROR(10165, "force task success error", "强制成功任务实例错误"),
TASK_INSTANCE_STATE_OPERATION_ERROR(10166, "the status of task instance {0} is {1},Cannot perform force success operation", "任务实例[{0}]的状态是[{1}],无法执行强制成功操作"), TASK_INSTANCE_STATE_OPERATION_ERROR(10166, "the status of task instance {0} is {1},Cannot perform force success operation", "任务实例[{0}]的状态是[{1}],无法执行强制成功操作"),
DATASOURCE_TYPE_NOT_EXIST(10167, "data source type not exist", "数据源类型不存在"), DATASOURCE_TYPE_NOT_EXIST(10167, "data source type not exist", "数据源类型不存在"),
PROCESS_DEFINITION_NAME_EXIST(10168, "process definition name {0} already exists", "工作流定义名称[{0}]已存在"),
UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found", "UDF函数不存在"), UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found", "UDF函数不存在"),
UDF_FUNCTION_EXISTS(20002, "UDF function already exists", "UDF函数已存在"), UDF_FUNCTION_EXISTS(20002, "UDF function already exists", "UDF函数已存在"),
@ -290,6 +291,7 @@ public enum Status {
QUERY_ALL_ALERT_PLUGIN_INSTANCE_ERROR(110009, "query all alert plugin instance error", "查询所有告警实例失败"), QUERY_ALL_ALERT_PLUGIN_INSTANCE_ERROR(110009, "query all alert plugin instance error", "查询所有告警实例失败"),
PLUGIN_INSTANCE_ALREADY_EXIT(110010,"plugin instance already exit","该告警插件实例已存在"), PLUGIN_INSTANCE_ALREADY_EXIT(110010,"plugin instance already exit","该告警插件实例已存在"),
LIST_PAGING_ALERT_PLUGIN_INSTANCE_ERROR(110011,"query plugin instance page error","分页查询告警实例失败"), LIST_PAGING_ALERT_PLUGIN_INSTANCE_ERROR(110011,"query plugin instance page error","分页查询告警实例失败"),
DELETE_ALERT_PLUGIN_INSTANCE_ERROR_HAS_ALERT_GROUP_ASSOCIATED(110012,"failed to delete the alert instance, there is an alarm group associated with this alert instance","删除告警实例失败,存在与此告警实例关联的警报组")
; ;

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LocaleChangeInterceptor.java

@ -17,7 +17,6 @@
package org.apache.dolphinscheduler.api.interceptor; package org.apache.dolphinscheduler.api.interceptor;
import org.apache.dolphinscheduler.api.service.BaseService;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import java.util.Locale; import java.util.Locale;
@ -30,12 +29,13 @@ import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.lang.Nullable; import org.springframework.lang.Nullable;
import org.springframework.util.StringUtils; import org.springframework.util.StringUtils;
import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; import org.springframework.web.servlet.handler.HandlerInterceptorAdapter;
import org.springframework.web.util.WebUtils;
public class LocaleChangeInterceptor extends HandlerInterceptorAdapter { public class LocaleChangeInterceptor extends HandlerInterceptorAdapter {
@Override @Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) { public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) {
Cookie cookie = BaseService.getCookie(request, Constants.LOCALE_LANGUAGE); Cookie cookie = WebUtils.getCookie(request, Constants.LOCALE_LANGUAGE);
if (cookie != null) { if (cookie != null) {
// Proceed in cookie // Proceed in cookie
return true; return true;

21
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java

@ -113,27 +113,6 @@ public class BaseService {
return false; return false;
} }
/**
* get cookie info by name
*
* @param request request
* @param name 'sessionId'
* @return get cookie info
*/
public static Cookie getCookie(HttpServletRequest request, String name) {
Cookie[] cookies = request.getCookies();
if (cookies != null && cookies.length > 0) {
for (Cookie cookie : cookies) {
if (StringUtils.isNotEmpty(name) && name.equalsIgnoreCase(cookie.getName())) {
return cookie;
}
}
}
return null;
}
/** /**
* create tenant dir if not exists * create tenant dir if not exists
* *

9
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java

@ -267,6 +267,9 @@ public class DataSourceService extends BaseService {
map.put(HOST, host); map.put(HOST, host);
map.put(PORT, port); map.put(PORT, port);
map.put(PRINCIPAL, datasourceForm.getPrincipal()); map.put(PRINCIPAL, datasourceForm.getPrincipal());
map.put(Constants.KERBEROS_KRB5_CONF_PATH, datasourceForm.getJavaSecurityKrb5Conf());
map.put(Constants.KERBEROS_KEY_TAB_USERNAME, datasourceForm.getLoginUserKeytabUsername());
map.put(Constants.KERBEROS_KEY_TAB_PATH, datasourceForm.getLoginUserKeytabPath());
map.put(DATABASE, database); map.put(DATABASE, database);
map.put(USER_NAME, datasourceForm.getUser()); map.put(USER_NAME, datasourceForm.getUser());
map.put(OTHER, otherMap); map.put(OTHER, otherMap);
@ -424,7 +427,8 @@ public class DataSourceService extends BaseService {
*/ */
public String buildParameter(DbType type, String host, public String buildParameter(DbType type, String host,
String port, String database, String principal, String userName, String port, String database, String principal, String userName,
String password, DbConnectType connectType, String other) { String password, DbConnectType connectType, String other,
String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath) {
String address = buildAddress(type, host, port, connectType); String address = buildAddress(type, host, port, connectType);
Map<String, Object> parameterMap = new LinkedHashMap<String, Object>(6); Map<String, Object> parameterMap = new LinkedHashMap<String, Object>(6);
@ -467,6 +471,9 @@ public class DataSourceService extends BaseService {
if (CommonUtils.getKerberosStartupState() if (CommonUtils.getKerberosStartupState()
&& (type == DbType.HIVE || type == DbType.SPARK)) { && (type == DbType.HIVE || type == DbType.SPARK)) {
parameterMap.put(Constants.PRINCIPAL, principal); parameterMap.put(Constants.PRINCIPAL, principal);
parameterMap.put(Constants.KERBEROS_KRB5_CONF_PATH, javaSecurityKrb5Conf);
parameterMap.put(Constants.KERBEROS_KEY_TAB_USERNAME, loginUserKeytabUsername);
parameterMap.put(Constants.KERBEROS_KEY_TAB_PATH, loginUserKeytabPath);
} }
Map<String, String> map = JSONUtils.toMap(other); Map<String, String> map = JSONUtils.toMap(other);

12
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java

@ -256,7 +256,7 @@ public class ProcessInstanceService extends BaseService {
List<ProcessInstance> processInstances = processInstanceList.getRecords(); List<ProcessInstance> processInstances = processInstanceList.getRecords();
for (ProcessInstance processInstance : processInstances) { for (ProcessInstance processInstance : processInstances) {
processInstance.setDuration(DateUtils.differSec(processInstance.getStartTime(), processInstance.getEndTime())); processInstance.setDuration(DateUtils.format2Duration(processInstance.getStartTime(), processInstance.getEndTime()));
User executor = usersService.queryUser(processInstance.getExecutorId()); User executor = usersService.queryUser(processInstance.getExecutorId());
if (null != executor) { if (null != executor) {
processInstance.setExecutorName(executor.getUserName()); processInstance.setExecutorName(executor.getUserName());
@ -428,10 +428,9 @@ public class ProcessInstanceService extends BaseService {
return result; return result;
} }
Date schedule = null; Date schedule = null;
schedule = processInstance.getScheduleTime();
if (scheduleTime != null) { if (scheduleTime != null) {
schedule = DateUtils.getScheduleDate(scheduleTime); schedule = DateUtils.getScheduleDate(scheduleTime);
} else {
schedule = processInstance.getScheduleTime();
} }
processInstance.setScheduleTime(schedule); processInstance.setScheduleTime(schedule);
processInstance.setLocations(locations); processInstance.setLocations(locations);
@ -460,13 +459,18 @@ public class ProcessInstanceService extends BaseService {
if (tenant != null) { if (tenant != null) {
processInstance.setTenantCode(tenant.getTenantCode()); processInstance.setTenantCode(tenant.getTenantCode());
} }
// get the processinstancejson before saving,and then save the name and taskid
String oldJson = processInstance.getProcessInstanceJson();
if (StringUtils.isNotEmpty(oldJson)) {
processInstanceJson = processService.changeJson(processData,oldJson);
}
processInstance.setProcessInstanceJson(processInstanceJson); processInstance.setProcessInstanceJson(processInstanceJson);
processInstance.setGlobalParams(globalParams); processInstance.setGlobalParams(globalParams);
} }
int update = processService.updateProcessInstance(processInstance); int update = processService.updateProcessInstance(processInstance);
int updateDefine = 1; int updateDefine = 1;
if (Boolean.TRUE.equals(syncDefine) && StringUtils.isNotEmpty(processInstanceJson)) { if (Boolean.TRUE.equals(syncDefine)) {
processDefinition.setProcessDefinitionJson(processInstanceJson); processDefinition.setProcessDefinitionJson(processInstanceJson);
processDefinition.setGlobalParams(originDefParams); processDefinition.setGlobalParams(originDefParams);
processDefinition.setLocations(locations); processDefinition.setLocations(locations);

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java

@ -131,7 +131,7 @@ public class TaskInstanceService extends BaseService {
List<TaskInstance> taskInstanceList = taskInstanceIPage.getRecords(); List<TaskInstance> taskInstanceList = taskInstanceIPage.getRecords();
for (TaskInstance taskInstance : taskInstanceList) { for (TaskInstance taskInstance : taskInstanceList) {
taskInstance.setDuration(DateUtils.differSec(taskInstance.getStartTime(), taskInstance.getEndTime())); taskInstance.setDuration(DateUtils.format2Duration(taskInstance.getStartTime(), taskInstance.getEndTime()));
User executor = usersService.queryUser(taskInstance.getExecutorId()); User executor = usersService.queryUser(taskInstance.getExecutorId());
if (null != executor) { if (null != executor) {
taskInstance.setExecutorName(executor.getUserName()); taskInstance.setExecutorName(executor.getUserName());

12
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java

@ -153,19 +153,21 @@ public class WorkerGroupService extends BaseService {
} }
} }
// available workerGroup list
List<String> availableWorkerGroupList = new ArrayList<>();
for (String workerGroup : workerGroupList) { for (String workerGroup : workerGroupList) {
String workerGroupPath = workerPath + "/" + workerGroup; String workerGroupPath = workerPath + "/" + workerGroup;
List<String> childrenNodes = zookeeperCachedOperator.getChildrenKeys(workerGroupPath); List<String> childrenNodes = zookeeperCachedOperator.getChildrenKeys(workerGroupPath);
String timeStamp = "";
for (int i = 0; i < childrenNodes.size(); i++) {
String ip = childrenNodes.get(i);
childrenNodes.set(i, ip.substring(0, ip.lastIndexOf(":")));
timeStamp = ip.substring(ip.lastIndexOf(":"));
}
if (CollectionUtils.isNotEmpty(childrenNodes)) { if (CollectionUtils.isNotEmpty(childrenNodes)) {
availableWorkerGroupList.add(workerGroup);
WorkerGroup wg = new WorkerGroup(); WorkerGroup wg = new WorkerGroup();
wg.setName(workerGroup); wg.setName(workerGroup);
if (isPaging) { if (isPaging) {
wg.setIpList(childrenNodes); wg.setIpList(childrenNodes);
String registeredIpValue = zookeeperCachedOperator.get(workerGroupPath + "/" + childrenNodes.get(0)); String registeredIpValue = zookeeperCachedOperator.get(workerGroupPath + "/" + childrenNodes.get(0) + timeStamp);
wg.setCreateTime(DateUtils.stringToDate(registeredIpValue.split(",")[6])); wg.setCreateTime(DateUtils.stringToDate(registeredIpValue.split(",")[6]));
wg.setUpdateTime(DateUtils.stringToDate(registeredIpValue.split(",")[7])); wg.setUpdateTime(DateUtils.stringToDate(registeredIpValue.split(",")[7]));
} }

88
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java

@ -24,16 +24,25 @@ import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.vo.AlertPluginInstanceVO; import org.apache.dolphinscheduler.api.vo.AlertPluginInstanceVO;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance;
import org.apache.dolphinscheduler.dao.entity.PluginDefine; import org.apache.dolphinscheduler.dao.entity.PluginDefine;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper;
import org.apache.dolphinscheduler.dao.mapper.AlertPluginInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.AlertPluginInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.PluginDefineMapper; import org.apache.dolphinscheduler.dao.mapper.PluginDefineMapper;
import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import org.apache.commons.collections4.MapUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@ -56,6 +65,9 @@ public class AlertPluginInstanceServiceImpl extends BaseService implements Alert
@Autowired @Autowired
private PluginDefineMapper pluginDefineMapper; private PluginDefineMapper pluginDefineMapper;
@Autowired
private AlertGroupMapper alertGroupMapper;
/** /**
* creat alert plugin instance * creat alert plugin instance
* *
@ -67,7 +79,8 @@ public class AlertPluginInstanceServiceImpl extends BaseService implements Alert
@Override @Override
public Map<String, Object> create(User loginUser, int pluginDefineId, String instanceName, String pluginInstanceParams) { public Map<String, Object> create(User loginUser, int pluginDefineId, String instanceName, String pluginInstanceParams) {
AlertPluginInstance alertPluginInstance = new AlertPluginInstance(); AlertPluginInstance alertPluginInstance = new AlertPluginInstance();
alertPluginInstance.setPluginInstanceParams(pluginInstanceParams); String paramsMapJson = parsePluginParamsMap(pluginInstanceParams);
alertPluginInstance.setPluginInstanceParams(paramsMapJson);
alertPluginInstance.setInstanceName(instanceName); alertPluginInstance.setInstanceName(instanceName);
alertPluginInstance.setPluginDefineId(pluginDefineId); alertPluginInstance.setPluginDefineId(pluginDefineId);
@ -82,7 +95,9 @@ public class AlertPluginInstanceServiceImpl extends BaseService implements Alert
if (i > 0) { if (i > 0) {
putMsg(result, Status.SUCCESS); putMsg(result, Status.SUCCESS);
return result;
} }
putMsg(result, Status.SAVE_ERROR);
return result; return result;
} }
@ -98,7 +113,8 @@ public class AlertPluginInstanceServiceImpl extends BaseService implements Alert
public Map<String, Object> update(User loginUser, int pluginInstanceId, String instanceName, String pluginInstanceParams) { public Map<String, Object> update(User loginUser, int pluginInstanceId, String instanceName, String pluginInstanceParams) {
AlertPluginInstance alertPluginInstance = new AlertPluginInstance(); AlertPluginInstance alertPluginInstance = new AlertPluginInstance();
alertPluginInstance.setPluginInstanceParams(pluginInstanceParams); String paramsMapJson = parsePluginParamsMap(pluginInstanceParams);
alertPluginInstance.setPluginInstanceParams(paramsMapJson);
alertPluginInstance.setInstanceName(instanceName); alertPluginInstance.setInstanceName(instanceName);
alertPluginInstance.setId(pluginInstanceId); alertPluginInstance.setId(pluginInstanceId);
Map<String, Object> result = new HashMap<>(); Map<String, Object> result = new HashMap<>();
@ -106,8 +122,9 @@ public class AlertPluginInstanceServiceImpl extends BaseService implements Alert
if (i > 0) { if (i > 0) {
putMsg(result, Status.SUCCESS); putMsg(result, Status.SUCCESS);
return result;
} }
putMsg(result, Status.SAVE_ERROR);
return result; return result;
} }
@ -121,6 +138,13 @@ public class AlertPluginInstanceServiceImpl extends BaseService implements Alert
@Override @Override
public Map<String, Object> delete(User loginUser, int id) { public Map<String, Object> delete(User loginUser, int id) {
Map<String, Object> result = new HashMap<>(); Map<String, Object> result = new HashMap<>();
//check if there is an associated alert group
boolean hasAssociatedAlertGroup = checkHasAssociatedAlertGroup(String.valueOf(id));
if (hasAssociatedAlertGroup) {
putMsg(result, Status.DELETE_ALERT_PLUGIN_INSTANCE_ERROR_HAS_ALERT_GROUP_ASSOCIATED);
return result;
}
int i = alertPluginInstanceMapper.deleteById(id); int i = alertPluginInstanceMapper.deleteById(id);
if (i > 0) { if (i > 0) {
putMsg(result, Status.SUCCESS); putMsg(result, Status.SUCCESS);
@ -188,21 +212,73 @@ public class AlertPluginInstanceServiceImpl extends BaseService implements Alert
if (CollectionUtils.isEmpty(pluginDefineList)) { if (CollectionUtils.isEmpty(pluginDefineList)) {
return null; return null;
} }
Map<Integer, String> pluginDefineMap = pluginDefineList.stream().collect(Collectors.toMap(PluginDefine::getId, PluginDefine::getPluginName)); Map<Integer, PluginDefine> pluginDefineMap = pluginDefineList.stream().collect(Collectors.toMap(PluginDefine::getId, Function.identity()));
List<AlertPluginInstanceVO> alertPluginInstanceVOS = new ArrayList<>(); List<AlertPluginInstanceVO> alertPluginInstanceVOS = new ArrayList<>();
alertPluginInstances.forEach(alertPluginInstance -> { alertPluginInstances.forEach(alertPluginInstance -> {
AlertPluginInstanceVO alertPluginInstanceVO = new AlertPluginInstanceVO(); AlertPluginInstanceVO alertPluginInstanceVO = new AlertPluginInstanceVO();
alertPluginInstanceVO.setAlertPluginName(pluginDefineMap.get(alertPluginInstance.getPluginDefineId()));
alertPluginInstanceVO.setCreateTime(alertPluginInstance.getCreateTime()); alertPluginInstanceVO.setCreateTime(alertPluginInstance.getCreateTime());
alertPluginInstanceVO.setUpdateTime(alertPluginInstance.getUpdateTime()); alertPluginInstanceVO.setUpdateTime(alertPluginInstance.getUpdateTime());
alertPluginInstanceVO.setPluginDefineId(alertPluginInstance.getPluginDefineId()); alertPluginInstanceVO.setPluginDefineId(alertPluginInstance.getPluginDefineId());
alertPluginInstanceVO.setInstanceName(alertPluginInstance.getInstanceName()); alertPluginInstanceVO.setInstanceName(alertPluginInstance.getInstanceName());
alertPluginInstanceVO.setId(alertPluginInstance.getId()); alertPluginInstanceVO.setId(alertPluginInstance.getId());
PluginDefine pluginDefine = pluginDefineMap.get(alertPluginInstance.getPluginDefineId());
//FIXME When the user removes the plug-in, this will happen. At this time, maybe we should add a new field to indicate that the plug-in has expired?
if (null == pluginDefine) {
return;
}
alertPluginInstanceVO.setAlertPluginName(pluginDefine.getPluginName());
//todo List pages do not recommend returning this parameter //todo List pages do not recommend returning this parameter
alertPluginInstanceVO.setPluginInstanceParams(alertPluginInstance.getPluginInstanceParams()); String pluginParamsMapString = alertPluginInstance.getPluginInstanceParams();
String uiPluginParams = parseToPluginUiParams(pluginParamsMapString, pluginDefine.getPluginParams());
alertPluginInstanceVO.setPluginInstanceParams(uiPluginParams);
alertPluginInstanceVOS.add(alertPluginInstanceVO); alertPluginInstanceVOS.add(alertPluginInstanceVO);
}); });
return alertPluginInstanceVOS; return alertPluginInstanceVOS;
} }
/**
* Get the parameters actually needed by the plugin
*
* @param pluginParams Complete parameters(include ui)
* @return k, v(json string)
*/
private String parsePluginParamsMap(String pluginParams) {
Map<String, String> paramsMap = PluginParamsTransfer.getPluginParamsMap(pluginParams);
return JSONUtils.toJsonString(paramsMap);
}
/**
* parseToPluginUiParams
*
* @param pluginParamsMapString k-v data
* @param pluginUiParams Complete parameters(include ui)
* @return Complete parameters list(include ui)
*/
private String parseToPluginUiParams(String pluginParamsMapString, String pluginUiParams) {
Map<String, String> paramsMap = JSONUtils.toMap(pluginParamsMapString);
if (MapUtils.isEmpty(paramsMap)) {
return null;
}
List<PluginParams> pluginParamsList = JSONUtils.toList(pluginUiParams, PluginParams.class);
List<PluginParams> newPluginParamsList = new ArrayList<>(pluginParamsList.size());
pluginParamsList.forEach(pluginParams -> {
pluginParams.setValue(paramsMap.get(pluginParams.getName()));
newPluginParamsList.add(pluginParams);
});
return JSONUtils.toJsonString(newPluginParamsList);
}
private boolean checkHasAssociatedAlertGroup(String id) {
List<String> idsList = alertGroupMapper.queryInstanceIdsList();
if (CollectionUtils.isEmpty(idsList)) {
return false;
}
Optional<String> first = idsList.stream().filter(k -> null != k && Arrays.asList(k.split(",")).contains(id)).findFirst();
return first.isPresent();
}
} }

10
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java

@ -429,11 +429,13 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
// check whether the new process define name exist // check whether the new process define name exist
ProcessDefinition definition = processDefineMapper.verifyByDefineName(project.getId(), name); ProcessDefinition definition = processDefineMapper.verifyByDefineName(project.getId(), name);
if (definition != null) { if (definition != null) {
putMsg(result, Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR, name); putMsg(result, Status.PROCESS_DEFINITION_NAME_EXIST, name);
return result; return result;
} }
} }
// get the processdefinitionjson before saving,and then save the name and taskid
String oldJson = processDefine.getProcessDefinitionJson();
processDefinitionJson = processService.changeJson(processData,oldJson);
Date now = new Date(); Date now = new Date();
processDefine.setId(id); processDefine.setId(id);
@ -495,7 +497,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
if (processDefinition == null) { if (processDefinition == null) {
putMsg(result, Status.SUCCESS); putMsg(result, Status.SUCCESS);
} else { } else {
putMsg(result, Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR, name); putMsg(result, Status.PROCESS_DEFINITION_NAME_EXIST, name);
} }
return result; return result;
} }
@ -1325,7 +1327,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
List<ProcessInstance> processInstanceList = processInstanceService.queryByProcessDefineId(processId, limit); List<ProcessInstance> processInstanceList = processInstanceService.queryByProcessDefineId(processId, limit);
for (ProcessInstance processInstance : processInstanceList) { for (ProcessInstance processInstance : processInstanceList) {
processInstance.setDuration(DateUtils.differSec(processInstance.getStartTime(), processInstance.getEndTime())); processInstance.setDuration(DateUtils.format2Duration(processInstance.getStartTime(), processInstance.getEndTime()));
} }
if (limit > processInstanceList.size()) { if (limit > processInstanceList.size()) {

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java

@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.util.WebUtils;
/** /**
* session service implement * session service implement
@ -60,7 +61,7 @@ public class SessionServiceImpl extends BaseService implements SessionService {
String sessionId = request.getHeader(Constants.SESSION_ID); String sessionId = request.getHeader(Constants.SESSION_ID);
if (StringUtils.isBlank(sessionId)) { if (StringUtils.isBlank(sessionId)) {
Cookie cookie = getCookie(request, Constants.SESSION_ID); Cookie cookie = WebUtils.getCookie(request, Constants.SESSION_ID);
if (cookie != null) { if (cookie != null) {
sessionId = cookie.getValue(); sessionId = cookie.getValue();

4
dolphinscheduler-api/src/main/resources/i18n/messages.properties

@ -125,6 +125,10 @@ TENANT_CODE=os tenant code
QUEUE_NAME=queue name QUEUE_NAME=queue name
PASSWORD=password PASSWORD=password
DATA_SOURCE_OTHER=jdbc connection params, format:{"key1":"value1",...} DATA_SOURCE_OTHER=jdbc connection params, format:{"key1":"value1",...}
DATA_SOURCE_PRINCIPAL=principal
DATA_SOURCE_KERBEROS_KRB5_CONF=the kerberos authentication parameter java.security.krb5.conf
DATA_SOURCE_KERBEROS_KEYTAB_USERNAME=the kerberos authentication parameter login.user.keytab.username
DATA_SOURCE_KERBEROS_KEYTAB_PATH=the kerberos authentication parameter login.user.keytab.path
PROJECT_TAG=project related operation PROJECT_TAG=project related operation
CREATE_PROJECT_NOTES=create project CREATE_PROJECT_NOTES=create project
PROJECT_DESC=project description PROJECT_DESC=project description

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save