Browse Source

Merge branch 'dev' into dev

pull/3/MERGE
BoYiZhang 4 years ago committed by GitHub
parent
commit
3823a081bd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 4
      ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py
  2. 4
      ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/params.py
  3. 2
      docker/build/Dockerfile
  4. 66
      docker/build/README.md
  5. 66
      docker/build/README_zh_CN.md
  6. 14
      docker/build/conf/dolphinscheduler/alert.properties.tpl
  7. 12
      docker/build/conf/dolphinscheduler/common.properties.tpl
  8. 15
      docker/build/conf/dolphinscheduler/datasource.properties.tpl
  9. 2
      docker/build/conf/dolphinscheduler/logback/logback-alert.xml
  10. 2
      docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
  11. 23
      docker/build/startup-init-conf.sh
  12. 34
      docker/build/startup.sh
  13. 40
      docker/docker-swarm/docker-compose.yml
  14. 52
      docker/docker-swarm/docker-stack.yml
  15. 2
      docker/kubernetes/dolphinscheduler/Chart.yaml
  16. 30
      docker/kubernetes/dolphinscheduler/README.md
  17. 25
      docker/kubernetes/dolphinscheduler/requirements.yaml
  18. 13
      docker/kubernetes/dolphinscheduler/templates/NOTES.txt
  19. 8
      docker/kubernetes/dolphinscheduler/templates/_helpers.tpl
  20. 1
      docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml
  21. 35
      docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml
  22. 98
      docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml
  23. 93
      docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
  24. 17
      docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml
  25. 93
      docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
  26. 163
      docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
  27. 87
      docker/kubernetes/dolphinscheduler/values.yaml
  28. 30
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java
  29. 8
      dolphinscheduler-alert/src/main/resources/alert.properties
  30. 4
      dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java
  31. 8
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
  32. 7
      dolphinscheduler-server/src/main/resources/config/install_config.conf
  33. 2
      dolphinscheduler-ui/package.json
  34. 5
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/startingParam/index.vue
  35. 6
      dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue
  36. 6
      dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue
  37. 25
      e2e/src/test/java/org/apache/dolphinscheduler/data/project/ProcessInstanceData.java
  38. 2
      e2e/src/test/java/org/apache/dolphinscheduler/data/project/ProjectData.java
  39. 2
      e2e/src/test/java/org/apache/dolphinscheduler/data/project/WorkflowDefineData.java
  40. 31
      e2e/src/test/java/org/apache/dolphinscheduler/locator/project/ProcessInstanceLocator.java
  41. 9
      e2e/src/test/java/org/apache/dolphinscheduler/locator/project/ProjectLocator.java
  42. 4
      e2e/src/test/java/org/apache/dolphinscheduler/locator/project/TimingLocator.java
  43. 2
      e2e/src/test/java/org/apache/dolphinscheduler/locator/project/WorkflowDefineLocator.java
  44. 226
      e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateWorkflowPage.java
  45. 40
      e2e/src/test/java/org/apache/dolphinscheduler/page/project/ProcessInstancePage.java
  46. 29
      e2e/src/test/java/org/apache/dolphinscheduler/page/project/ProjectPage.java
  47. 4
      e2e/src/test/java/org/apache/dolphinscheduler/page/project/TimingPage.java
  48. 226
      e2e/src/test/java/org/apache/dolphinscheduler/page/project/WorkflowDefinePage.java
  49. 6
      e2e/src/test/java/org/apache/dolphinscheduler/testcase/testDeleteData/TestDeleteProject.java
  50. 26
      e2e/src/test/java/org/apache/dolphinscheduler/testcase/testDeleteData/TestDeleteWorkflow.java
  51. 35
      e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestProcessInstance.java
  52. 12
      e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestProject.java
  53. 10
      e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestRunWorkflow.java
  54. 36
      e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestWorkflowDefine.java
  55. 29
      e2e/testng.xml
  56. 2
      script/dolphinscheduler-daemon.sh
  57. 3
      sql/dolphinscheduler-postgre.sql
  58. 1
      sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_ddl.sql

4
ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py

@ -76,8 +76,8 @@ else:
dolphin_alert_map = {}
wechat_push_url = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token'
wechat_token_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret'
wechat_team_send_msg = '{\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"}'
wechat_user_send_msg = '{\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}}'
wechat_team_send_msg = '{\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"}'
wechat_user_send_msg = '{\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}}'
dolphin_alert_map['enterprise.wechat.push.ur'] = wechat_push_url
dolphin_alert_map['enterprise.wechat.token.url'] = wechat_token_url

4
ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/params.py

@ -77,8 +77,8 @@ else:
dolphin_alert_map = {}
wechat_push_url = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token'
wechat_token_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret'
wechat_team_send_msg = '{\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"}'
wechat_user_send_msg = '{\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}}'
wechat_team_send_msg = '{\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"}'
wechat_user_send_msg = '{\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}}'
dolphin_alert_config_map = config['configurations']['dolphin-alert']

2
docker/build/Dockerfile

@ -42,7 +42,7 @@ ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz
RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ /opt/dolphinscheduler/
ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
#4. install pg
#4. install database, if use mysql as your backend database, the `mysql-client` package should be installed
RUN apk add postgresql postgresql-contrib
#5. modify nginx
RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \

66
docker/build/README.md

@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org
#### You can start a dolphinscheduler instance
```
$ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \
-e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \
-p 8888:8888 \
dolphinscheduler all
```
@ -25,14 +25,14 @@ The default postgres user `root`, postgres password `root` and database `dolphin
The default zookeeper is created in the `startup.sh`.
#### Or via Environment Variables **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`POSTGRESQL_DATABASE`** **`ZOOKEEPER_QUORUM`**
#### Or via Environment Variables **`DATABASE_HOST`** **`DATABASE_PORT`** **`DATABASE_DATABASE`** **`ZOOKEEPER_QUORUM`**
You can specify **existing postgres service**. Example:
```
$ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-p 8888:8888 \
dolphinscheduler all
```
@ -42,7 +42,7 @@ You can specify **existing zookeeper service**. Example:
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \
-p 8888:8888 \
dolphinscheduler all
```
@ -56,8 +56,8 @@ You can start a standalone dolphinscheduler server.
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler master-server
```
@ -66,8 +66,8 @@ dolphinscheduler master-server
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler worker-server
```
@ -75,8 +75,8 @@ dolphinscheduler worker-server
```
$ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-p 12345:12345 \
dolphinscheduler api-server
```
@ -85,8 +85,8 @@ dolphinscheduler api-server
```
$ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler alert-server
```
@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \
dolphinscheduler frontend
```
**Note**: You must be specify `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server.
**Note**: You must be specify `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server.
## How to build a docker image
@ -124,33 +124,51 @@ Please read `./docker/build/hooks/build` `./docker/build/hooks/build.bat` script
The Dolphin Scheduler image uses several environment variables which are easy to miss. While none of the variables are required, they may significantly aid you in using the image.
**`POSTGRESQL_HOST`**
**`DATABASE_TYPE`**
This environment variable sets the host for PostgreSQL. The default value is `127.0.0.1`.
This environment variable sets the type for database. The default value is `postgresql`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`POSTGRESQL_PORT`**
**`DATABASE_DRIVER`**
This environment variable sets the port for PostgreSQL. The default value is `5432`.
This environment variable sets the type for database. The default value is `org.postgresql.Driver`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`POSTGRESQL_USERNAME`**
**`DATABASE_HOST`**
This environment variable sets the username for PostgreSQL. The default value is `root`.
This environment variable sets the host for database. The default value is `127.0.0.1`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`POSTGRESQL_PASSWORD`**
**`DATABASE_PORT`**
This environment variable sets the password for PostgreSQL. The default value is `root`.
This environment variable sets the port for database. The default value is `5432`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`POSTGRESQL_DATABASE`**
**`DATABASE_USERNAME`**
This environment variable sets the database for PostgreSQL. The default value is `dolphinscheduler`.
This environment variable sets the username for database. The default value is `root`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`DATABASE_PASSWORD`**
This environment variable sets the password for database. The default value is `root`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`DATABASE_DATABASE`**
This environment variable sets the database for database. The default value is `dolphinscheduler`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`DATABASE_PARAMS`**
This environment variable sets the database for database. The default value is `characterEncoding=utf8`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.

66
docker/build/README_zh_CN.md

@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org
#### 你可以运行一个dolphinscheduler实例
```
$ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \
-e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \
-p 8888:8888 \
dolphinscheduler all
```
@ -25,14 +25,14 @@ dolphinscheduler all
同时,默认的`Zookeeper`也会在`startup.sh`脚本中被创建。
#### 或者通过环境变量 **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`ZOOKEEPER_QUORUM`** 使用已存在的服务
#### 或者通过环境变量 **`DATABASE_HOST`** **`DATABASE_PORT`** **`ZOOKEEPER_QUORUM`** 使用已存在的服务
你可以指定一个已经存在的 **`Postgres`** 服务. 如下:
```
$ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-p 8888:8888 \
dolphinscheduler all
```
@ -42,7 +42,7 @@ dolphinscheduler all
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \
-p 8888:8888 \
dolphinscheduler all
```
@ -56,8 +56,8 @@ dolphinscheduler all
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler master-server
```
@ -66,8 +66,8 @@ dolphinscheduler master-server
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler worker-server
```
@ -75,8 +75,8 @@ dolphinscheduler worker-server
```
$ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-p 12345:12345 \
dolphinscheduler api-server
```
@ -85,8 +85,8 @@ dolphinscheduler api-server
```
$ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler alert-server
```
@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \
dolphinscheduler frontend
```
**注意**: 当你运行dolphinscheduler中的部分服务时,你必须指定这些环境变量 `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM`
**注意**: 当你运行dolphinscheduler中的部分服务时,你必须指定这些环境变量 `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM`
## 如何构建一个docker镜像
@ -124,33 +124,51 @@ c:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat
Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些变量不是必须的,但是可以帮助你更容易配置镜像并根据你的需求定义相应的服务配置。
**`POSTGRESQL_HOST`**
**`DATABASE_TYPE`**
配置`PostgreSQL`的`HOST`, 默认值 `127.0.0.1`
配置`database`的`TYPE`, 默认值 `postgresql`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`POSTGRESQL_PORT`**
**`DATABASE_DRIVER`**
配置`PostgreSQL`的`PORT`, 默认值 `5432`
配置`database`的`DRIVER`, 默认值 `org.postgresql.Driver`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`POSTGRESQL_USERNAME`**
**`DATABASE_HOST`**
配置`PostgreSQL`的`USERNAME`, 默认值 `root`
配置`database`的`HOST`, 默认值 `127.0.0.1`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`POSTGRESQL_PASSWORD`**
**`DATABASE_PORT`**
配置`PostgreSQL`的`PASSWORD`, 默认值 `root`
配置`database`的`PORT`, 默认值 `5432`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`POSTGRESQL_DATABASE`**
**`DATABASE_USERNAME`**
配置`PostgreSQL`的`DATABASE`, 默认值 `dolphinscheduler`
配置`database`的`USERNAME`, 默认值 `root`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`DATABASE_PASSWORD`**
配置`database`的`PASSWORD`, 默认值 `root`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`DATABASE_DATABASE`**
配置`database`的`DATABASE`, 默认值 `dolphinscheduler`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`DATABASE_PARAMS`**
配置`database`的`PARAMS`, 默认值 `characterEncoding=utf8`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。

14
docker/build/conf/dolphinscheduler/alert.properties.tpl

@ -35,16 +35,22 @@ mail.smtp.ssl.trust=${MAIL_SMTP_SSL_TRUST}
#xls file path,need create if not exist
xls.file.path=${XLS_FILE_PATH}
# plugins dir
plugin.dir=${ALERT_PLUGIN_DIR}
# Enterprise WeChat configuration
enterprise.wechat.enable=${ENTERPRISE_WECHAT_ENABLE}
enterprise.wechat.corp.id=${ENTERPRISE_WECHAT_CORP_ID}
enterprise.wechat.secret=${ENTERPRISE_WECHAT_SECRET}
enterprise.wechat.agent.id=${ENTERPRISE_WECHAT_AGENT_ID}
enterprise.wechat.users=${ENTERPRISE_WECHAT_USERS}
enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret
enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token
enterprise.wechat.team.send.msg={\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"}
enterprise.wechat.user.send.msg={\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}}
enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid={corpId}&corpsecret={secret}
enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={token}
enterprise.wechat.team.send.msg={\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"}
enterprise.wechat.user.send.msg={\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}}

12
docker/build/conf/dolphinscheduler/common.properties.tpl

@ -25,13 +25,13 @@ dolphinscheduler.env.path=${DOLPHINSCHEDULER_ENV_PATH}
data.basedir.path=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH}
# resource upload startup type : HDFS,S3,NONE
resource.storage.type=NONE
resource.storage.type=${RESOURCE_STORAGE_TYPE}
#============================================================================
# HDFS
#============================================================================
# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended
#resource.upload.path=/dolphinscheduler
resource.upload.path=${RESOURCE_UPLOAD_PATH}
# whether kerberos starts
#hadoop.security.authentication.startup.state=false
@ -58,16 +58,16 @@ kerberos.expire.time=7
# S3
#============================================================================
# if resource.storage.type=S3,the value like: s3a://dolphinscheduler ; if resource.storage.type=HDFS, When namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
fs.defaultFS=hdfs://mycluster:8020
fs.defaultFS=${FS_DEFAULT_FS}
# if resource.storage.type=S3,s3 endpoint
#fs.s3a.endpoint=http://192.168.199.91:9010
fs.s3a.endpoint=${FS_S3A_ENDPOINT}
# if resource.storage.type=S3,s3 access key
#fs.s3a.access.key=A3DXS30FO22544RE
fs.s3a.access.key=${FS_S3A_ACCESS_KEY}
# if resource.storage.type=S3,s3 secret key
#fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK
fs.s3a.secret.key=${FS_S3A_SECRET_KEY}
# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty TODO
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx

15
docker/build/conf/dolphinscheduler/datasource.properties.tpl

@ -15,16 +15,11 @@
# limitations under the License.
#
# mysql
#spring.datasource.driver-class-name=com.mysql.jdbc.Driver
#spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
# postgre
spring.datasource.driver-class-name=org.postgresql.Driver
spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8
spring.datasource.username=${POSTGRESQL_USERNAME}
spring.datasource.password=${POSTGRESQL_PASSWORD}
# db
spring.datasource.driver-class-name=${DATABASE_DRIVER}
spring.datasource.url=jdbc:${DATABASE_TYPE}://${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DATABASE}?${DATABASE_PARAMS}
spring.datasource.username=${DATABASE_USERNAME}
spring.datasource.password=${DATABASE_PASSWORD}
## base spring data source configuration todo need to remove
#spring.datasource.type=com.alibaba.druid.pool.DruidDataSource

2
docker/build/conf/dolphinscheduler/logback/logback-alert.xml

@ -46,7 +46,7 @@
<root level="INFO">
<appender-ref ref="STDOUT"/>
<appender-ref ref="APILOGFILE"/>
<appender-ref ref="ALERTLOGFILE"/>
</root>
</configuration>

2
docker/build/conf/dolphinscheduler/zookeeper.properties.tpl

@ -19,7 +19,7 @@
zookeeper.quorum=${ZOOKEEPER_QUORUM}
# dolphinscheduler root directory
#zookeeper.dolphinscheduler.root=/dolphinscheduler
zookeeper.dolphinscheduler.root=${ZOOKEEPER_ROOT}
# dolphinscheduler failover directory
#zookeeper.session.timeout=60000

23
docker/build/startup-init-conf.sh

@ -24,22 +24,33 @@ echo "init env variables"
#============================================================================
# Database Source
#============================================================================
export POSTGRESQL_HOST=${POSTGRESQL_HOST:-"127.0.0.1"}
export POSTGRESQL_PORT=${POSTGRESQL_PORT:-"5432"}
export POSTGRESQL_USERNAME=${POSTGRESQL_USERNAME:-"root"}
export POSTGRESQL_PASSWORD=${POSTGRESQL_PASSWORD:-"root"}
export POSTGRESQL_DATABASE=${POSTGRESQL_DATABASE:-"dolphinscheduler"}
export DATABASE_HOST=${DATABASE_HOST:-"127.0.0.1"}
export DATABASE_PORT=${DATABASE_PORT:-"5432"}
export DATABASE_USERNAME=${DATABASE_USERNAME:-"root"}
export DATABASE_PASSWORD=${DATABASE_PASSWORD:-"root"}
export DATABASE_DATABASE=${DATABASE_DATABASE:-"dolphinscheduler"}
export DATABASE_TYPE=${DATABASE_TYPE:-"postgresql"}
export DATABASE_DRIVER=${DATABASE_DRIVER:-"org.postgresql.Driver"}
export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"}
#============================================================================
# System
#============================================================================
export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"}
export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"}
export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""}
export RESOURCE_STORAGE_TYPE=${RESOURCE_STORAGE_TYPE:-"NONE"}
export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/ds"}
export FS_DEFAULT_FS=${FS_DEFAULT_FS:-"s3a://xxxx"}
export FS_S3A_ENDPOINT=${FS_S3A_ENDPOINT:-"s3.xxx.amazonaws.com"}
export FS_S3A_ACCESS_KEY=${FS_S3A_ACCESS_KEY:-"xxxxxxx"}
export FS_S3A_SECRET_KEY=${FS_S3A_SECRET_KEY:-"xxxxxxx"}
#============================================================================
# Zookeeper
#============================================================================
export ZOOKEEPER_QUORUM=${ZOOKEEPER_QUORUM:-"127.0.0.1:2181"}
export ZOOKEEPER_ROOT=${ZOOKEEPER_ROOT:-"/dolphinscheduler"}
#============================================================================
# Master Server
@ -67,6 +78,8 @@ export WORKER_GROUP=${WORKER_GROUP:-"default"}
#============================================================================
# Alert Server
#============================================================================
# alert plugin dir
export ALERT_PLUGIN_DIR=${ALERT_PLUGIN_DIR:-"/opt/dolphinscheduler"}
# XLS FILE
export XLS_FILE_PATH=${XLS_FILE_PATH:-"/tmp/xls"}
# mail

34
docker/build/startup.sh

@ -22,25 +22,33 @@ DOLPHINSCHEDULER_BIN=${DOLPHINSCHEDULER_HOME}/bin
DOLPHINSCHEDULER_SCRIPT=${DOLPHINSCHEDULER_HOME}/script
DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs
# start postgresql
initPostgreSQL() {
echo "test postgresql service"
while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do
# start database
initDatabase() {
echo "test ${DATABASE_TYPE} service"
while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
counter=$((counter+1))
if [ $counter == 30 ]; then
echo "Error: Couldn't connect to postgresql."
echo "Error: Couldn't connect to ${DATABASE_TYPE}."
exit 1
fi
echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter."
echo "Trying to connect to ${DATABASE_TYPE} at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
sleep 5
done
echo "connect postgresql service"
v=$(sudo -u postgres PGPASSWORD=${POSTGRESQL_PASSWORD} psql -h ${POSTGRESQL_HOST} -p ${POSTGRESQL_PORT} -U ${POSTGRESQL_USERNAME} -d dolphinscheduler -tAc "select 1")
echo "connect ${DATABASE_TYPE} service"
if [ ${DATABASE_TYPE} = "mysql" ]; then
v=$(mysql -h${DATABASE_HOST} -P${DATABASE_PORT} -u${DATABASE_USERNAME} --password=${DATABASE_PASSWORD} -D ${DATABASE_DATABASE} -e "select 1" 2>&1)
if [ "$(echo '${v}' | grep 'ERROR' | wc -l)" -eq 1 ]; then
echo "Error: Can't connect to database...${v}"
exit 1
fi
else
v=$(sudo -u postgres PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "select 1")
if [ "$(echo '${v}' | grep 'FATAL' | wc -l)" -eq 1 ]; then
echo "Error: Can't connect to database...${v}"
exit 1
fi
fi
echo "import sql data"
${DOLPHINSCHEDULER_SCRIPT}/create-dolphinscheduler.sh
@ -123,7 +131,7 @@ LOGFILE=/var/log/nginx/access.log
case "$1" in
(all)
initZK
initPostgreSQL
initDatabase
initMasterServer
initWorkerServer
initApiServer
@ -134,25 +142,25 @@ case "$1" in
;;
(master-server)
initZK
initPostgreSQL
initDatabase
initMasterServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-master.log
;;
(worker-server)
initZK
initPostgreSQL
initDatabase
initWorkerServer
initLoggerServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log
;;
(api-server)
initZK
initPostgreSQL
initDatabase
initApiServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log
;;
(alert-server)
initPostgreSQL
initDatabase
initAlertServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-alert.log
;;

40
docker/docker-swarm/docker-compose.yml

@ -56,11 +56,11 @@ services:
- 12345:12345
environment:
TZ: Asia/Shanghai
POSTGRESQL_HOST: dolphinscheduler-postgresql
POSTGRESQL_PORT: 5432
POSTGRESQL_USERNAME: root
POSTGRESQL_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler
DATABASE_HOST: dolphinscheduler-postgresql
DATABASE_PORT: 5432
DATABASE_USERNAME: root
DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
@ -119,11 +119,11 @@ services:
ENTERPRISE_WECHAT_SECRET: ""
ENTERPRISE_WECHAT_AGENT_ID: ""
ENTERPRISE_WECHAT_USERS: ""
POSTGRESQL_HOST: dolphinscheduler-postgresql
POSTGRESQL_PORT: 5432
POSTGRESQL_USERNAME: root
POSTGRESQL_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler
DATABASE_HOST: dolphinscheduler-postgresql
DATABASE_PORT: 5432
DATABASE_USERNAME: root
DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
interval: 30s
@ -152,11 +152,11 @@ services:
MASTER_TASK_COMMIT_INTERVAL: "1000"
MASTER_MAX_CPULOAD_AVG: "100"
MASTER_RESERVED_MEMORY: "0.1"
POSTGRESQL_HOST: dolphinscheduler-postgresql
POSTGRESQL_PORT: 5432
POSTGRESQL_USERNAME: root
POSTGRESQL_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler
DATABASE_HOST: dolphinscheduler-postgresql
DATABASE_PORT: 5432
DATABASE_USERNAME: root
DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
@ -188,11 +188,11 @@ services:
WORKER_RESERVED_MEMORY: "0.1"
WORKER_GROUP: "default"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
POSTGRESQL_HOST: dolphinscheduler-postgresql
POSTGRESQL_PORT: 5432
POSTGRESQL_USERNAME: root
POSTGRESQL_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler
DATABASE_HOST: dolphinscheduler-postgresql
DATABASE_PORT: 5432
DATABASE_USERNAME: root
DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]

52
docker/docker-swarm/docker-stack.yml

@ -20,13 +20,13 @@ services:
dolphinscheduler-postgresql:
image: bitnami/postgresql:latest
ports:
- 5432:5432
environment:
TZ: Asia/Shanghai
POSTGRESQL_USERNAME: root
POSTGRESQL_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler
ports:
- 5432:5432
volumes:
- dolphinscheduler-postgresql:/bitnami/postgresql
networks:
@ -37,12 +37,12 @@ services:
dolphinscheduler-zookeeper:
image: bitnami/zookeeper:latest
ports:
- 2181:2181
environment:
TZ: Asia/Shanghai
ALLOW_ANONYMOUS_LOGIN: "yes"
ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
ports:
- 2181:2181
volumes:
- dolphinscheduler-zookeeper:/bitnami/zookeeper
networks:
@ -54,16 +54,16 @@ services:
dolphinscheduler-api:
image: apache/dolphinscheduler:latest
command: ["api-server"]
ports:
- 12345:12345
environment:
TZ: Asia/Shanghai
POSTGRESQL_HOST: dolphinscheduler-postgresql
POSTGRESQL_PORT: 5432
POSTGRESQL_USERNAME: root
POSTGRESQL_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler
DATABASE_HOST: dolphinscheduler-postgresql
DATABASE_PORT: 5432
DATABASE_USERNAME: root
DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
ports:
- 12345:12345
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
interval: 30
@ -120,11 +120,11 @@ services:
ENTERPRISE_WECHAT_SECRET: ""
ENTERPRISE_WECHAT_AGENT_ID: ""
ENTERPRISE_WECHAT_USERS: ""
POSTGRESQL_HOST: dolphinscheduler-postgresql
POSTGRESQL_PORT: 5432
POSTGRESQL_USERNAME: root
POSTGRESQL_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler
DATABASE_HOST: dolphinscheduler-postgresql
DATABASE_PORT: 5432
DATABASE_USERNAME: root
DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
interval: 30
@ -153,11 +153,11 @@ services:
MASTER_TASK_COMMIT_INTERVAL: "1000"
MASTER_MAX_CPULOAD_AVG: "100"
MASTER_RESERVED_MEMORY: "0.1"
POSTGRESQL_HOST: dolphinscheduler-postgresql
POSTGRESQL_PORT: 5432
POSTGRESQL_USERNAME: root
POSTGRESQL_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler
DATABASE_HOST: dolphinscheduler-postgresql
DATABASE_PORT: 5432
DATABASE_USERNAME: root
DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
@ -188,11 +188,11 @@ services:
WORKER_RESERVED_MEMORY: "0.1"
WORKER_GROUP: "default"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
POSTGRESQL_HOST: dolphinscheduler-postgresql
POSTGRESQL_PORT: 5432
POSTGRESQL_USERNAME: root
POSTGRESQL_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler
DATABASE_HOST: dolphinscheduler-postgresql
DATABASE_PORT: 5432
DATABASE_USERNAME: root
DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]

2
docker/kubernetes/dolphinscheduler/Chart.yaml

@ -49,4 +49,4 @@ dependencies:
- name: zookeeper
version: 5.x.x
repository: https://charts.bitnami.com/bitnami
condition: redis.enabled
condition: zookeeper.enabled

30
docker/kubernetes/dolphinscheduler/README.md

@ -46,7 +46,7 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `image.repository` | Docker image repository for the Dolphins Scheduler | `dolphinscheduler` |
| `image.tag` | Docker image version for the Dolphins Scheduler | `1.2.1` |
| `image.imagePullPolicy` | Image pull policy. One of Always, Never, IfNotPresent | `IfNotPresent` |
| `imagePullSecrets` | ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images | `[]` |
| `image.pullSecres` | PullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images | `[]` |
| | | |
| `postgresql.enabled` | If not exists external PostgreSQL, by default, the Dolphins Scheduler will use a internal PostgreSQL | `true` |
| `postgresql.postgresqlUsername` | The username for internal PostgreSQL | `root` |
@ -55,11 +55,14 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `postgresql.persistence.enabled` | Set `postgresql.persistence.enabled` to `true` to mount a new volume for internal PostgreSQL | `false` |
| `postgresql.persistence.size` | `PersistentVolumeClaim` Size | `20Gi` |
| `postgresql.persistence.storageClass` | PostgreSQL data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `externalDatabase.type` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database type will use it. | `postgresql` |
| `externalDatabase.driver` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database driver will use it. | `org.postgresql.Driver` |
| `externalDatabase.host` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database host will use it. | `localhost` |
| `externalDatabase.port` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database port will use it. | `5432` |
| `externalDatabase.username` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database username will use it. | `root` |
| `externalDatabase.password` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database password will use it. | `root` |
| `externalDatabase.database` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database database will use it. | `dolphinscheduler` |
| `externalDatabase.params` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database params will use it. | `characterEncoding=utf8` |
| | | |
| `zookeeper.enabled` | If not exists external Zookeeper, by default, the Dolphin Scheduler will use a internal Zookeeper | `true` |
| `zookeeper.taskQueue` | Specify task queue for `master` and `worker` | `zookeeper` |
@ -68,12 +71,25 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `zookeeper.persistence.storageClass` | Zookeeper data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `externalZookeeper.taskQueue` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify task queue for `master` and `worker` | `zookeeper` |
| `externalZookeeper.zookeeperQuorum` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify Zookeeper quorum | `127.0.0.1:2181` |
| `externalZookeeper.zookeeperRoot` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify Zookeeper root path for `master` and `worker` | `dolphinscheduler` |
| | | |
| `common.configmap.DOLPHINSCHEDULER_ENV_PATH` | Extra env file path. | `/tmp/dolphinscheduler/env` |
| `common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH` | File uploaded path of DS. | `/tmp/dolphinscheduler/files` |
| `common.configmap.RESOURCE_STORAGE_TYPE` | Resource Storate type, support type are: S3、HDFS、NONE. | `NONE` |
| `common.configmap.RESOURCE_UPLOAD_PATH` | The base path of resource. | `/ds` |
| `common.configmap.FS_DEFAULT_FS` | The default fs of resource, for s3 is the `s3a` prefix and bucket name. | `s3a://xxxx` |
| `common.configmap.FS_S3A_ENDPOINT` | If the resource type is `S3`, you should fill this filed, it's the endpoint of s3. | `s3.xxx.amazonaws.com` |
| `common.configmap.FS_S3A_ACCESS_KEY` | The access key for your s3 bucket. | `xxxxxxx` |
| `common.configmap.FS_S3A_SECRET_KEY` | The secret key for your s3 bucket. | `xxxxxxx` |
| `master.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` |
| | | |
| `master.replicas` | Replicas is the desired number of replicas of the given Template | `3` |
| `master.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
| `master.tolerations` | If specified, the pod's tolerations | `{}` |
| `master.affinity` | If specified, the pod's scheduling constraints | `{}` |
| `master.jvmOptions` | The JVM options for master server. | `""` |
| `master.resources` | The `resource` limit and request config for master server. | `{}` |
| `master.annotations` | The `annotations` for master server. | `{}` |
| `master.configmap.MASTER_EXEC_THREADS` | Master execute thread num | `100` |
| `master.configmap.MASTER_EXEC_TASK_NUM` | Master execute task number in parallel | `20` |
| `master.configmap.MASTER_HEARTBEAT_INTERVAL` | Master heartbeat interval | `10` |
@ -103,6 +119,9 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `worker.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
| `worker.tolerations` | If specified, the pod's tolerations | `{}` |
| `worker.affinity` | If specified, the pod's scheduling constraints | `{}` |
| `worker.jvmOptions` | The JVM options for worker server. | `""` |
| `worker.resources` | The `resource` limit and request config for worker server. | `{}` |
| `worker.annotations` | The `annotations` for worker server. | `{}` |
| `worker.configmap.WORKER_EXEC_THREADS` | Worker execute thread num | `100` |
| `worker.configmap.WORKER_HEARTBEAT_INTERVAL` | Worker heartbeat interval | `10` |
| `worker.configmap.WORKER_FETCH_TASK_NUM` | Submit the number of tasks at a time | `3` |
@ -139,6 +158,10 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `alert.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
| `alert.tolerations` | If specified, the pod's tolerations | `{}` |
| `alert.affinity` | If specified, the pod's scheduling constraints | `{}` |
| `alert.jvmOptions` | The JVM options for alert server. | `""` |
| `alert.resources` | The `resource` limit and request config for alert server. | `{}` |
| `alert.annotations` | The `annotations` for alert server. | `{}` |
| `alert.configmap.ALERT_PLUGIN_DIR` | Alert plugin path. | `/opt/dolphinscheduler/alert/plugin` |
| `alert.configmap.XLS_FILE_PATH` | XLS file path | `/tmp/xls` |
| `alert.configmap.MAIL_SERVER_HOST` | Mail `SERVER HOST ` | `nil` |
| `alert.configmap.MAIL_SERVER_PORT` | Mail `SERVER PORT` | `nil` |
@ -177,6 +200,9 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `api.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
| `api.tolerations` | If specified, the pod's tolerations | `{}` |
| `api.affinity` | If specified, the pod's scheduling constraints | `{}` |
| `api.jvmOptions` | The JVM options for api server. | `""` |
| `api.resources` | The `resource` limit and request config for api server. | `{}` |
| `api.annotations` | The `annotations` for api server. | `{}` |
| `api.livenessProbe.enabled` | Turn on and off liveness probe | `true` |
| `api.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` |
| `api.livenessProbe.periodSeconds` | How often to perform the probe | `30` |
@ -201,6 +227,8 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `frontend.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
| `frontend.tolerations` | If specified, the pod's tolerations | `{}` |
| `frontend.affinity` | If specified, the pod's scheduling constraints | `{}` |
| `frontend.resources` | The `resource` limit and request config for frontend server. | `{}` |
| `frontend.annotations` | The `annotations` for frontend server. | `{}` |
| `frontend.livenessProbe.enabled` | Turn on and off liveness probe | `true` |
| `frontend.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` |
| `frontend.livenessProbe.periodSeconds` | How often to perform the probe | `30` |

25
docker/kubernetes/dolphinscheduler/requirements.yaml

@ -1,25 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
dependencies:
- name: postgresql
version: 8.x.x
repository: https://charts.bitnami.com/bitnami
condition: postgresql.enabled
- name: zookeeper
version: 5.x.x
repository: https://charts.bitnami.com/bitnami
condition: redis.enabled

13
docker/kubernetes/dolphinscheduler/templates/NOTES.txt

@ -29,16 +29,3 @@
kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-frontend 8888:8888
{{- end }}
2. Get the Dolphinscheduler URL by running:
{{- if .Values.ingress.enabled }}
export HOSTNAME=$(kubectl get ingress --namespace {{ .Release.Namespace }} {{ template "dolphinscheduler.fullname" . }} -o jsonpath='{.spec.rules[0].host}')
echo "Dolphinscheduler URL: http://$HOSTNAME/"
{{- else }}
kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-frontend 8888:8888
{{- end }}

8
docker/kubernetes/dolphinscheduler/templates/_helpers.tpl

@ -96,6 +96,14 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
{{- printf "%s/%s:%s" (include "dolphinscheduler.image.registry" .) .Values.image.repository .Values.image.tag -}}
{{- end -}}
{{/*
Create a default image pull secrects.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
*/}}
{{- define "dolphinscheduler.image.pullSecrets" -}}
{{- default nil .Values.image.pullSecrets -}}
{{- end -}}
{{/*
Create a default fully qualified postgresql name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).

1
docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml

@ -24,6 +24,7 @@ metadata:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
data:
ALERT_PLUGIN_DIR: {{ .Values.alert.configmap.ALERT_PLUGIN_DIR | quote }}
XLS_FILE_PATH: {{ .Values.alert.configmap.XLS_FILE_PATH | quote }}
MAIL_SERVER_HOST: {{ .Values.alert.configmap.MAIL_SERVER_HOST | quote }}
MAIL_SERVER_PORT: {{ .Values.alert.configmap.MAIL_SERVER_PORT | quote }}

35
docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml

@ -0,0 +1,35 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if .Values.common.configmap }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "dolphinscheduler.fullname" . }}-common
labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-common
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
data:
DOLPHINSCHEDULER_ENV_PATH: {{ .Values.common.configmap.DOLPHINSCHEDULER_ENV_PATH | quote }}
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ .Values.common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH | quote }}
RESOURCE_STORAGE_TYPE: {{ .Values.common.configmap.RESOURCE_STORAGE_TYPE | quote }}
RESOURCE_UPLOAD_PATH: {{ .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }}
FS_DEFAULT_FS: {{ .Values.common.configmap.FS_DEFAULT_FS | quote }}
FS_S3A_ENDPOINT: {{ .Values.common.configmap.FS_S3A_ENDPOINT | quote }}
FS_S3A_ACCESS_KEY: {{ .Values.common.configmap.FS_S3A_ACCESS_KEY | quote }}
FS_S3A_SECRET_KEY: {{ .Values.common.configmap.FS_S3A_SECRET_KEY | quote }}
{{- end }}

98
docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml

@ -43,6 +43,10 @@ spec:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: alert
{{- if .Values.alert.annotations }}
annotations:
{{- toYaml .Values.alert.annotations | nindent 8 }}
{{- end }}
spec:
{{- if .Values.alert.affinity }}
affinity: {{- toYaml .Values.alert.affinity | nindent 8 }}
@ -54,33 +58,37 @@ spec:
tolerations: {{- toYaml . | nindent 8 }}
{{- end }}
initContainers:
- name: init-postgresql
- name: init-database
image: busybox:1.31.0
command:
- /bin/sh
- -ec
- |
while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do
while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
counter=$((counter+1))
if [ $counter == 5 ]; then
echo "Error: Couldn't connect to postgresql."
echo "Error: Couldn't connect to database."
exit 1
fi
echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter."
echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
sleep 60
done
env:
- name: POSTGRESQL_HOST
- name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- name: POSTGRESQL_PORT
- name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
value: {{ .Values.externalDatabase.port }}
value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
{{- if .Values.image.pullSecrets }}
imagePullSecrets:
- name: {{ include "dolphinscheduler.image.pullSecrets" . }}
{{- end }}
containers:
- name: {{ include "dolphinscheduler.fullname" . }}-alert
@ -89,8 +97,15 @@ spec:
- "alert-server"
imagePullPolicy: {{ .Values.image.pullPolicy }}
env:
- name: DOLPHINSCHEDULER_OPTS
value: {{ default "" .Values.alert.jvmOptions }}
- name: TZ
value: {{ .Values.timezone }}
- name: ALERT_PLUGIN_DIR
valueFrom:
configMapKeyRef:
key: ALERT_PLUGIN_DIR
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: XLS_FILE_PATH
valueFrom:
configMapKeyRef:
@ -161,25 +176,37 @@ spec:
configMapKeyRef:
key: ENTERPRISE_WECHAT_USERS
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: POSTGRESQL_HOST
- name: DATABASE_TYPE
{{- if .Values.postgresql.enabled }}
value: "postgresql"
{{- else }}
value: {{ .Values.externalDatabase.type | quote }}
{{- end }}
- name: DATABASE_DRIVER
{{- if .Values.postgresql.enabled }}
value: "org.postgresql.Driver"
{{- else }}
value: {{ .Values.externalDatabase.driver | quote }}
{{- end }}
- name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- name: POSTGRESQL_PORT
- name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
value: {{ .Values.externalDatabase.port }}
value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
- name: POSTGRESQL_USERNAME
- name: DATABASE_USERNAME
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlUsername }}
{{- else }}
value: {{ .Values.externalDatabase.username | quote }}
{{- end }}
- name: POSTGRESQL_PASSWORD
- name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
{{- if .Values.postgresql.enabled }}
@ -189,12 +216,57 @@ spec:
name: {{ printf "%s-%s" .Release.Name "externaldb" }}
key: db-password
{{- end }}
- name: POSTGRESQL_DATABASE
- name: DATABASE_DATABASE
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlDatabase }}
{{- else }}
value: {{ .Values.externalDatabase.database | quote }}
{{- end }}
- name: DATABASE_PARAMS
{{- if .Values.postgresql.enabled }}
value: "characterEncoding=utf8"
{{- else }}
value: {{ .Values.externalDatabase.params | quote }}
{{- end }}
- name: RESOURCE_STORAGE_TYPE
valueFrom:
configMapKeyRef:
key: RESOURCE_STORAGE_TYPE
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: RESOURCE_UPLOAD_PATH
valueFrom:
configMapKeyRef:
key: RESOURCE_UPLOAD_PATH
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_DEFAULT_FS
valueFrom:
configMapKeyRef:
key: FS_DEFAULT_FS
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_ENDPOINT
valueFrom:
configMapKeyRef:
key: FS_S3A_ENDPOINT
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_ACCESS_KEY
valueFrom:
configMapKeyRef:
key: FS_S3A_ACCESS_KEY
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_SECRET_KEY
valueFrom:
configMapKeyRef:
key: FS_S3A_SECRET_KEY
name: {{ include "dolphinscheduler.fullname" . }}-common
{{- if .Values.alert.resources }}
resources:
limits:
memory: {{ .Values.alert.resources.limits.memory }}
cpu: {{ .Values.alert.resources.limits.cpu }}
requests:
memory: {{ .Values.alert.resources.requests.memory }}
cpu: {{ .Values.alert.resources.requests.cpu }}
{{- end }}
{{- if .Values.alert.livenessProbe.enabled }}
livenessProbe:
exec:

93
docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml

@ -43,6 +43,10 @@ spec:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: api
{{- if .Values.alert.annotations }}
annotations:
{{- toYaml .Values.alert.annotations | nindent 8 }}
{{- end }}
spec:
{{- if .Values.api.affinity }}
affinity: {{- toYaml .Values.api.affinity | nindent 8 }}
@ -54,33 +58,37 @@ spec:
tolerations: {{- toYaml . | nindent 8 }}
{{- end }}
initContainers:
- name: init-postgresql
- name: init-database
image: busybox:1.31.0
command:
- /bin/sh
- -ec
- |
while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do
while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
counter=$((counter+1))
if [ $counter == 5 ]; then
echo "Error: Couldn't connect to postgresql."
echo "Error: Couldn't connect to database."
exit 1
fi
echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter."
echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
sleep 60
done
env:
- name: POSTGRESQL_HOST
- name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- name: POSTGRESQL_PORT
- name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
value: {{ .Values.externalDatabase.port }}
value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
{{- if .Values.image.pullSecrets }}
imagePullSecrets:
- name: {{ include "dolphinscheduler.image.pullSecrets" . }}
{{- end }}
containers:
- name: {{ include "dolphinscheduler.fullname" . }}-api
@ -92,27 +100,41 @@ spec:
name: tcp-port
imagePullPolicy: {{ .Values.image.pullPolicy }}
env:
- name: DOLPHINSCHEDULER_OPTS
value: {{ default "" .Values.api.jvmOptions }}
- name: TZ
value: {{ .Values.timezone }}
- name: POSTGRESQL_HOST
- name: DATABASE_TYPE
{{- if .Values.postgresql.enabled }}
value: "postgresql"
{{- else }}
value: {{ .Values.externalDatabase.type | quote }}
{{- end }}
- name: DATABASE_DRIVER
{{- if .Values.postgresql.enabled }}
value: "org.postgresql.Driver"
{{- else }}
value: {{ .Values.externalDatabase.driver | quote }}
{{- end }}
- name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- name: POSTGRESQL_PORT
- name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
value: {{ .Values.externalDatabase.port }}
value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
- name: POSTGRESQL_USERNAME
- name: DATABASE_USERNAME
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlUsername }}
{{- else }}
value: {{ .Values.externalDatabase.username | quote }}
{{- end }}
- name: POSTGRESQL_PASSWORD
- name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
{{- if .Values.postgresql.enabled }}
@ -122,18 +144,63 @@ spec:
name: {{ printf "%s-%s" .Release.Name "externaldb" }}
key: db-password
{{- end }}
- name: POSTGRESQL_DATABASE
- name: DATABASE_DATABASE
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlDatabase }}
{{- else }}
value: {{ .Values.externalDatabase.database | quote }}
{{- end }}
- name: DATABASE_PARAMS
{{- if .Values.postgresql.enabled }}
value: "characterEncoding=utf8"
{{- else }}
value: {{ .Values.externalDatabase.params | quote }}
{{- end }}
- name: ZOOKEEPER_QUORUM
{{- if .Values.zookeeper.enabled }}
value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}"
{{- else }}
value: {{ .Values.externalZookeeper.zookeeperQuorum }}
{{- end }}
- name: RESOURCE_STORAGE_TYPE
valueFrom:
configMapKeyRef:
key: RESOURCE_STORAGE_TYPE
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: RESOURCE_UPLOAD_PATH
valueFrom:
configMapKeyRef:
key: RESOURCE_UPLOAD_PATH
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_DEFAULT_FS
valueFrom:
configMapKeyRef:
key: FS_DEFAULT_FS
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_ENDPOINT
valueFrom:
configMapKeyRef:
key: FS_S3A_ENDPOINT
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_ACCESS_KEY
valueFrom:
configMapKeyRef:
key: FS_S3A_ACCESS_KEY
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_SECRET_KEY
valueFrom:
configMapKeyRef:
key: FS_S3A_SECRET_KEY
name: {{ include "dolphinscheduler.fullname" . }}-common
{{- if .Values.api.resources }}
resources:
limits:
memory: {{ .Values.api.resources.limits.memory | quote }}
cpu: {{ .Values.api.resources.limits.cpu | quote }}
requests:
memory: {{ .Values.api.resources.requests.memory | quote }}
cpu: {{ .Values.api.resources.requests.cpu | quote }}
{{- end }}
{{- if .Values.api.livenessProbe.enabled }}
livenessProbe:
tcpSocket:

17
docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml

@ -43,6 +43,10 @@ spec:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: frontend
{{- if .Values.alert.annotations }}
annotations:
{{- toYaml .Values.alert.annotations | nindent 8 }}
{{- end }}
spec:
{{- if .Values.frontend.affinity }}
affinity: {{- toYaml .Values.frontend.affinity | nindent 8 }}
@ -53,6 +57,10 @@ spec:
{{- if .Values.frontend.tolerations }}
tolerations: {{- toYaml . | nindent 8 }}
{{- end }}
{{- if .Values.image.pullSecrets }}
imagePullSecrets:
- name: {{ include "dolphinscheduler.image.pullSecrets" . }}
{{- end }}
containers:
- name: {{ include "dolphinscheduler.fullname" . }}-frontend
image: {{ include "dolphinscheduler.image.repository" . | quote }}
@ -69,6 +77,15 @@ spec:
value: '{{ include "dolphinscheduler.fullname" . }}-api'
- name: FRONTEND_API_SERVER_PORT
value: "12345"
{{- if .Values.frontend.resources }}
resources:
limits:
memory: {{ .Values.frontend.resources.limits.memory | quote }}
cpu: {{ .Values.frontend.resources.limits.cpu | quote }}
requests:
memory: {{ .Values.frontend.resources.requests.memory | quote }}
cpu: {{ .Values.frontend.resources.requests.cpu | quote }}
{{- end }}
{{- if .Values.frontend.livenessProbe.enabled }}
livenessProbe:
tcpSocket:

93
docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml

@ -40,6 +40,10 @@ spec:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: master
{{- if .Values.alert.annotations }}
annotations:
{{- toYaml .Values.alert.annotations | nindent 8 }}
{{- end }}
spec:
{{- if .Values.master.affinity }}
affinity: {{- toYaml .Values.master.affinity | nindent 8 }}
@ -75,33 +79,37 @@ spec:
{{- else }}
value: {{ .Values.externalZookeeper.zookeeperQuorum }}
{{- end }}
- name: init-postgresql
- name: init-database
image: busybox:1.31.0
command:
- /bin/sh
- -ec
- |
while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do
while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
counter=$((counter+1))
if [ $counter == 5 ]; then
echo "Error: Couldn't connect to postgresql."
echo "Error: Couldn't connect to database."
exit 1
fi
echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter."
echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
sleep 60
done
env:
- name: POSTGRESQL_HOST
- name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- name: POSTGRESQL_PORT
- name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
value: {{ .Values.externalDatabase.port }}
value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
{{- if .Values.image.pullSecrets }}
imagePullSecrets:
- name: {{ include "dolphinscheduler.image.pullSecrets" . }}
{{- end }}
containers:
- name: {{ include "dolphinscheduler.fullname" . }}-master
@ -113,6 +121,8 @@ spec:
name: "master-port"
imagePullPolicy: {{ .Values.image.pullPolicy }}
env:
- name: DOLPHINSCHEDULER_OPTS
value: {{ default "" .Values.master.jvmOptions }}
- name: TZ
value: {{ .Values.timezone }}
- name: MASTER_EXEC_THREADS
@ -160,25 +170,37 @@ spec:
configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-master
key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
- name: POSTGRESQL_HOST
- name: DATABASE_TYPE
{{- if .Values.postgresql.enabled }}
value: "postgresql"
{{- else }}
value: {{ .Values.externalDatabase.type | quote }}
{{- end }}
- name: DATABASE_DRIVER
{{- if .Values.postgresql.enabled }}
value: "org.postgresql.Driver"
{{- else }}
value: {{ .Values.externalDatabase.driver | quote }}
{{- end }}
- name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- name: POSTGRESQL_PORT
- name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
value: {{ .Values.externalDatabase.port }}
value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
- name: POSTGRESQL_USERNAME
- name: DATABASE_USERNAME
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlUsername }}
{{- else }}
value: {{ .Values.externalDatabase.username | quote }}
{{- end }}
- name: POSTGRESQL_PASSWORD
- name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
{{- if .Values.postgresql.enabled }}
@ -188,18 +210,63 @@ spec:
name: {{ printf "%s-%s" .Release.Name "externaldb" }}
key: db-password
{{- end }}
- name: POSTGRESQL_DATABASE
- name: DATABASE_DATABASE
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlDatabase }}
{{- else }}
value: {{ .Values.externalDatabase.database | quote }}
{{- end }}
- name: DATABASE_PARAMS
{{- if .Values.postgresql.enabled }}
value: "characterEncoding=utf8"
{{- else }}
value: {{ .Values.externalDatabase.params | quote }}
{{- end }}
- name: ZOOKEEPER_QUORUM
{{- if .Values.zookeeper.enabled }}
value: {{ template "dolphinscheduler.zookeeper.quorum" . }}
{{- else }}
value: {{ .Values.externalZookeeper.zookeeperQuorum }}
{{- end }}
- name: RESOURCE_STORAGE_TYPE
valueFrom:
configMapKeyRef:
key: RESOURCE_STORAGE_TYPE
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: RESOURCE_UPLOAD_PATH
valueFrom:
configMapKeyRef:
key: RESOURCE_UPLOAD_PATH
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_DEFAULT_FS
valueFrom:
configMapKeyRef:
key: FS_DEFAULT_FS
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_ENDPOINT
valueFrom:
configMapKeyRef:
key: FS_S3A_ENDPOINT
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_ACCESS_KEY
valueFrom:
configMapKeyRef:
key: FS_S3A_ACCESS_KEY
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_SECRET_KEY
valueFrom:
configMapKeyRef:
key: FS_S3A_SECRET_KEY
name: {{ include "dolphinscheduler.fullname" . }}-common
{{- if .Values.master.resources }}
resources:
limits:
memory: {{ .Values.master.resources.limits.memory | quote }}
cpu: {{ .Values.master.resources.limits.cpu | quote }}
requests:
memory: {{ .Values.master.resources.requests.memory | quote }}
cpu: {{ .Values.master.resources.requests.cpu | quote }}
{{- end }}
{{- if .Values.master.livenessProbe.enabled }}
livenessProbe:
exec:

163
docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml

@ -40,6 +40,10 @@ spec:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: worker
{{- if .Values.alert.annotations }}
annotations:
{{- toYaml .Values.alert.annotations | nindent 8 }}
{{- end }}
spec:
{{- if .Values.worker.affinity }}
affinity: {{- toYaml .Values.worker.affinity | nindent 8 }}
@ -75,33 +79,37 @@ spec:
{{- else }}
value: {{ .Values.externalZookeeper.zookeeperQuorum }}
{{- end }}
- name: init-postgresql
- name: init-database
image: busybox:1.31.0
command:
- /bin/sh
- -ec
- |
while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do
while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
counter=$((counter+1))
if [ $counter == 5 ]; then
echo "Error: Couldn't connect to postgresql."
echo "Error: Couldn't connect to database."
exit 1
fi
echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter."
echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
sleep 60
done
env:
- name: POSTGRESQL_HOST
- name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- name: POSTGRESQL_PORT
- name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
value: {{ .Values.externalDatabase.port }}
value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
{{- if .Values.image.pullSecrets }}
imagePullSecrets:
- name: {{ include "dolphinscheduler.image.pullSecrets" . }}
{{- end }}
containers:
- name: {{ include "dolphinscheduler.fullname" . }}-worker
@ -115,6 +123,8 @@ spec:
name: "logs-port"
imagePullPolicy: {{ .Values.image.pullPolicy }}
env:
- name: DOLPHINSCHEDULER_OPTS
value: {{ default "" .Values.worker.jvmOptions }}
- name: TZ
value: {{ .Values.timezone }}
- name: WORKER_EXEC_THREADS
@ -157,25 +167,37 @@ spec:
configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-master
key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
- name: POSTGRESQL_HOST
- name: DATABASE_TYPE
{{- if .Values.postgresql.enabled }}
value: "postgresql"
{{- else }}
value: {{ .Values.externalDatabase.type | quote }}
{{- end }}
- name: DATABASE_DRIVER
{{- if .Values.postgresql.enabled }}
value: "org.postgresql.Driver"
{{- else }}
value: {{ .Values.externalDatabase.driver | quote }}
{{- end }}
- name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- name: POSTGRESQL_PORT
- name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
value: {{ .Values.externalDatabase.port }}
value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
- name: POSTGRESQL_USERNAME
- name: DATABASE_USERNAME
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlUsername }}
{{- else }}
value: {{ .Values.externalDatabase.username | quote }}
{{- end }}
- name: POSTGRESQL_PASSWORD
- name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
{{- if .Values.postgresql.enabled }}
@ -185,18 +207,133 @@ spec:
name: {{ printf "%s-%s" .Release.Name "externaldb" }}
key: db-password
{{- end }}
- name: POSTGRESQL_DATABASE
- name: DATABASE_DATABASE
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlDatabase }}
{{- else }}
value: {{ .Values.externalDatabase.database | quote }}
{{- end }}
- name: DATABASE_PARAMS
{{- if .Values.postgresql.enabled }}
value: "characterEncoding=utf8"
{{- else }}
value: {{ .Values.externalDatabase.params | quote }}
{{- end }}
- name: ZOOKEEPER_QUORUM
{{- if .Values.zookeeper.enabled }}
value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}"
{{- else }}
value: {{ .Values.externalZookeeper.zookeeperQuorum }}
{{- end }}
- name: RESOURCE_STORAGE_TYPE
valueFrom:
configMapKeyRef:
key: RESOURCE_STORAGE_TYPE
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: RESOURCE_UPLOAD_PATH
valueFrom:
configMapKeyRef:
key: RESOURCE_UPLOAD_PATH
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_DEFAULT_FS
valueFrom:
configMapKeyRef:
key: FS_DEFAULT_FS
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_ENDPOINT
valueFrom:
configMapKeyRef:
key: FS_S3A_ENDPOINT
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_ACCESS_KEY
valueFrom:
configMapKeyRef:
key: FS_S3A_ACCESS_KEY
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_SECRET_KEY
valueFrom:
configMapKeyRef:
key: FS_S3A_SECRET_KEY
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: XLS_FILE_PATH
valueFrom:
configMapKeyRef:
key: XLS_FILE_PATH
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SERVER_HOST
valueFrom:
configMapKeyRef:
key: MAIL_SERVER_HOST
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SERVER_PORT
valueFrom:
configMapKeyRef:
key: MAIL_SERVER_PORT
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SENDER
valueFrom:
configMapKeyRef:
key: MAIL_SENDER
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_USER
valueFrom:
configMapKeyRef:
key: MAIL_USER
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_PASSWD
valueFrom:
configMapKeyRef:
key: MAIL_PASSWD
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SMTP_STARTTLS_ENABLE
valueFrom:
configMapKeyRef:
key: MAIL_SMTP_STARTTLS_ENABLE
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SMTP_SSL_ENABLE
valueFrom:
configMapKeyRef:
key: MAIL_SMTP_SSL_ENABLE
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SMTP_SSL_TRUST
valueFrom:
configMapKeyRef:
key: MAIL_SMTP_SSL_TRUST
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: ENTERPRISE_WECHAT_ENABLE
valueFrom:
configMapKeyRef:
key: ENTERPRISE_WECHAT_ENABLE
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: ENTERPRISE_WECHAT_CORP_ID
valueFrom:
configMapKeyRef:
key: ENTERPRISE_WECHAT_CORP_ID
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: ENTERPRISE_WECHAT_SECRET
valueFrom:
configMapKeyRef:
key: ENTERPRISE_WECHAT_SECRET
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: ENTERPRISE_WECHAT_AGENT_ID
valueFrom:
configMapKeyRef:
key: ENTERPRISE_WECHAT_AGENT_ID
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: ENTERPRISE_WECHAT_USERS
valueFrom:
configMapKeyRef:
key: ENTERPRISE_WECHAT_USERS
name: {{ include "dolphinscheduler.fullname" . }}-alert
{{- if .Values.worker.resources }}
resources:
limits:
memory: {{ .Values.worker.resources.limits.memory | quote }}
cpu: {{ .Values.worker.resources.limits.cpu | quote }}
requests:
memory: {{ .Values.worker.resources.requests.memory | quote }}
cpu: {{ .Values.worker.resources.requests.cpu | quote }}
{{- end }}
{{- if .Values.worker.livenessProbe.enabled }}
livenessProbe:
exec:

87
docker/kubernetes/dolphinscheduler/values.yaml

@ -29,10 +29,9 @@ image:
repository: "dolphinscheduler"
tag: "latest"
pullPolicy: "IfNotPresent"
pullSecrets: []
imagePullSecrets: []
# If not exists external postgresql, by default, Dolphinscheduler's database will use it.
# If not exists external database, by default, Dolphinscheduler's database will use it.
postgresql:
enabled: true
postgresqlUsername: "root"
@ -43,21 +42,24 @@ postgresql:
size: "20Gi"
storageClass: "-"
# If exists external postgresql, and set postgresql.enable value to false.
# If postgresql.enable is false, Dolphinscheduler's database will use it.
# If exists external database, and set postgresql.enable value to false.
# external database will be used, otherwise Dolphinscheduler's database will be used.
externalDatabase:
type: "postgresql"
driver: "org.postgresql.Driver"
host: "localhost"
port: "5432"
username: "root"
password: "root"
database: "dolphinscheduler"
# multi params should join with & char
params: "characterEncoding=utf8"
# If not exists external zookeeper, by default, Dolphinscheduler's zookeeper will use it.
zookeeper:
enabled: true
taskQueue: "zookeeper"
config:
ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
config: null
service:
port: "2181"
persistence:
@ -70,6 +72,18 @@ zookeeper:
externalZookeeper:
taskQueue: "zookeeper"
zookeeperQuorum: "127.0.0.1:2181"
zookeeperRoot: "/dolphinscheduler"
common:
configmap:
DOLPHINSCHEDULER_ENV_PATH: "/tmp/dolphinscheduler/env"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler/files"
RESOURCE_STORAGE_TYPE: "NONE"
RESOURCE_UPLOAD_PATH: "/ds"
FS_DEFAULT_FS: "s3a://xxxx"
FS_S3A_ENDPOINT: "s3.xxx.amazonaws.com"
FS_S3A_ACCESS_KEY: "xxxxxxx"
FS_S3A_SECRET_KEY: "xxxxxxx"
master:
podManagementPolicy: "Parallel"
@ -85,6 +99,18 @@ master:
# If specified, the pod's scheduling constraints.
# More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
affinity: {}
# The jvm options for java instance startup
jvmOptions: ""
resources: {}
# limits:
# memory: "18Gi"
# cpu: "4"
# requests:
# memory: "2Gi"
# cpu: "500m"
# You can use annotations to attach arbitrary non-identifying metadata to objects.
# Clients such as tools and libraries can retrieve this metadata.
annotations: {}
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
configmap:
@ -137,6 +163,18 @@ worker:
# If specified, the pod's scheduling constraints.
# More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
affinity: {}
# The jvm options for java instance startup
jvmOptions: ""
resources: {}
# limits:
# memory: "18Gi"
# cpu: "4"
# requests:
# memory: "2Gi"
# cpu: "500m"
# You can use annotations to attach arbitrary non-identifying metadata to objects.
# Clients such as tools and libraries can retrieve this metadata.
annotations: {}
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
livenessProbe:
@ -213,9 +251,22 @@ alert:
# If specified, the pod's scheduling constraints.
# More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
affinity: {}
# The jvm options for java instance startup
jvmOptions: ""
resources: {}
# limits:
# memory: "4Gi"
# cpu: "1"
# requests:
# memory: "2Gi"
# cpu: "500m"
# You can use annotations to attach arbitrary non-identifying metadata to objects.
# Clients such as tools and libraries can retrieve this metadata.
annotations: {}
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
configmap:
ALERT_PLUGIN_DIR: "/opt/dolphinscheduler/alert/plugin"
XLS_FILE_PATH: "/tmp/xls"
MAIL_SERVER_HOST: ""
MAIL_SERVER_PORT: ""
@ -275,6 +326,18 @@ api:
# If specified, the pod's scheduling constraints.
# More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
affinity: {}
# The jvm options for java instance startup
jvmOptions: ""
resources: {}
# limits:
# memory: "4Gi"
# cpu: "2"
# requests:
# memory: "2Gi"
# cpu: "500m"
# You can use annotations to attach arbitrary non-identifying metadata to objects.
# Clients such as tools and libraries can retrieve this metadata.
annotations: {}
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
livenessProbe:
@ -322,6 +385,16 @@ frontend:
# If specified, the pod's scheduling constraints.
# More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
affinity: {}
resources: {}
# limits:
# memory: "256Mi"
# cpu: "1"
# requests:
# memory: "256Mi"
# cpu: "500m"
# You can use annotations to attach arbitrary non-identifying metadata to objects.
# Clients such as tools and libraries can retrieve this metadata.
annotations: {}
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
livenessProbe:

30
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java

@ -48,8 +48,8 @@ public class EnterpriseWeChatUtils {
private static final String ENTERPRISE_WE_CHAT_TOKEN_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TOKEN_URL);
private static final String ENTERPRISE_WE_CHAT_TOKEN_URL_REPLACE = ENTERPRISE_WE_CHAT_TOKEN_URL == null ? null : ENTERPRISE_WE_CHAT_TOKEN_URL
.replaceAll("\\$corpId", ENTERPRISE_WE_CHAT_CORP_ID)
.replaceAll("\\$secret", ENTERPRISE_WE_CHAT_SECRET);
.replaceAll("\\{corpId\\}", ENTERPRISE_WE_CHAT_CORP_ID)
.replaceAll("\\{secret\\}", ENTERPRISE_WE_CHAT_SECRET);
private static final String ENTERPRISE_WE_CHAT_PUSH_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_PUSH_URL);
@ -120,9 +120,9 @@ public class EnterpriseWeChatUtils {
* @return Enterprise WeChat send message
*/
public static String makeTeamSendMsg(String toParty, String agentId, String msg) {
return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\$toParty", toParty)
.replaceAll("\\$agentId", agentId)
.replaceAll("\\$msg", msg);
return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty\\}", toParty)
.replaceAll("\\{agentId\\}", agentId)
.replaceAll("\\{msg\\}", msg);
}
/**
@ -135,9 +135,9 @@ public class EnterpriseWeChatUtils {
*/
public static String makeTeamSendMsg(Collection<String> toParty, String agentId, String msg) {
String listParty = FuncUtils.mkString(toParty, "|");
return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\$toParty", listParty)
.replaceAll("\\$agentId", agentId)
.replaceAll("\\$msg", msg);
return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty\\}", listParty)
.replaceAll("\\{agentId\\}", agentId)
.replaceAll("\\{msg\\}", msg);
}
/**
@ -149,9 +149,9 @@ public class EnterpriseWeChatUtils {
* @return Enterprise WeChat send message
*/
public static String makeUserSendMsg(String toUser, String agentId, String msg) {
return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\$toUser", toUser)
.replaceAll("\\$agentId", agentId)
.replaceAll("\\$msg", msg);
return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\{toUser\\}", toUser)
.replaceAll("\\{agentId\\}", agentId)
.replaceAll("\\{msg\\}", msg);
}
/**
@ -164,9 +164,9 @@ public class EnterpriseWeChatUtils {
*/
public static String makeUserSendMsg(Collection<String> toUser, String agentId, String msg) {
String listUser = FuncUtils.mkString(toUser, "|");
return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\$toUser", listUser)
.replaceAll("\\$agentId", agentId)
.replaceAll("\\$msg", msg);
return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\{toUser\\}", listUser)
.replaceAll("\\{agentId\\}", agentId)
.replaceAll("\\{msg\\}", msg);
}
/**
@ -179,7 +179,7 @@ public class EnterpriseWeChatUtils {
* @throws IOException the IOException
*/
public static String sendEnterpriseWeChat(String charset, String data, String token) throws IOException {
String enterpriseWeChatPushUrlReplace = ENTERPRISE_WE_CHAT_PUSH_URL.replaceAll("\\$token", token);
String enterpriseWeChatPushUrlReplace = ENTERPRISE_WE_CHAT_PUSH_URL.replaceAll("\\{token\\}", token);
CloseableHttpClient httpClient = HttpClients.createDefault();
try {

8
dolphinscheduler-alert/src/main/resources/alert.properties

@ -41,10 +41,10 @@ enterprise.wechat.enable=false
#enterprise.wechat.secret=xxxxxxx
#enterprise.wechat.agent.id=xxxxxxx
#enterprise.wechat.users=xxxxxxx
#enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret
#enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token
#enterprise.wechat.team.send.msg={\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"}
#enterprise.wechat.user.send.msg={\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}}
#enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid={corpId}&corpsecret={secret}
#enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={token}
#enterprise.wechat.team.send.msg={\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"}
#enterprise.wechat.user.send.msg={\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}}
plugin.dir=/Users/xx/your/path/to/plugin/dir

4
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java

@ -55,8 +55,8 @@ public class EnterpriseWeChatUtilsTest {
private static final String enterpriseWechatUsers="LiGang,journey";
private static final String msg = "hello world";
private static final String enterpriseWechatTeamSendMsg = "{\\\"toparty\\\":\\\"$toParty\\\",\\\"agentid\\\":\\\"$agentId\\\",\\\"msgtype\\\":\\\"text\\\",\\\"text\\\":{\\\"content\\\":\\\"$msg\\\"},\\\"safe\\\":\\\"0\\\"}";
private static final String enterpriseWechatUserSendMsg = "{\\\"touser\\\":\\\"$toUser\\\",\\\"agentid\\\":\\\"$agentId\\\",\\\"msgtype\\\":\\\"markdown\\\",\\\"markdown\\\":{\\\"content\\\":\\\"$msg\\\"}}";
private static final String enterpriseWechatTeamSendMsg = "{\\\"toparty\\\":\\\"{toParty}\\\",\\\"agentid\\\":\\\"{agentId}\\\",\\\"msgtype\\\":\\\"text\\\",\\\"text\\\":{\\\"content\\\":\\\"{msg}\\\"},\\\"safe\\\":\\\"0\\\"}";
private static final String enterpriseWechatUserSendMsg = "{\\\"touser\\\":\\\"{toUser}\\\",\\\"agentid\\\":\\\"{agentId}\\\",\\\"msgtype\\\":\\\"markdown\\\",\\\"markdown\\\":{\\\"content\\\":\\\"{msg}\\\"}}";
@Before
public void init(){

8
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java

@ -46,7 +46,7 @@ import java.util.concurrent.TimeUnit;
public class MasterSchedulerService extends Thread {
/**
* logger of MasterSchedulerThread
* logger of MasterSchedulerService
*/
private static final Logger logger = LoggerFactory.getLogger(MasterSchedulerService.class);
@ -80,7 +80,7 @@ public class MasterSchedulerService extends Thread {
/**
* constructor of MasterSchedulerThread
* constructor of MasterSchedulerService
*/
@PostConstruct
public void init(){
@ -91,7 +91,7 @@ public class MasterSchedulerService extends Thread {
@Override
public void start(){
super.setName("MasterSchedulerThread");
super.setName("MasterSchedulerService");
super.start();
}
@ -109,7 +109,7 @@ public class MasterSchedulerService extends Thread {
}
/**
* run of MasterSchedulerThread
* run of MasterSchedulerService
*/
@Override
public void run() {

7
dolphinscheduler-server/src/main/resources/config/install_config.conf

@ -27,12 +27,13 @@ dbhost="192.168.xx.xx:3306"
# db username
username="xx"
# db password
# NOTICE: if there are special characters, please use the \ to escape, for example, `[` escape to `\[`
password="xx"
# database name
dbname="dolphinscheduler"
# db passwprd
# NOTICE: if there are special characters, please use the \ to escape, for example, `[` escape to `\[`
password="xx"
# zk cluster
zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181"

2
dolphinscheduler-ui/package.json

@ -63,7 +63,7 @@
"html-loader": "^0.5.5",
"html-webpack-plugin": "^3.2.0",
"mini-css-extract-plugin": "^0.8.2",
"node-sass": "^4.14.0",
"node-sass": "^4.14.1",
"postcss-loader": "^3.0.0",
"progress-bar-webpack-plugin": "^1.12.1",
"rimraf": "^2.6.2",

5
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/startingParam/index.vue

@ -76,14 +76,9 @@
return '-'
},
_getNotifyGroupList () {
let notifyGroupListS = _.cloneDeep(this.store.state.dag.notifyGroupListS) || []
if (!notifyGroupListS.length) {
this.store.dispatch('dag/getNotifyGroupList').then(res => {
this.notifyGroupList = res
})
} else {
this.notifyGroupList = notifyGroupListS
}
},
_getWorkerGroupList () {
let stateWorkerGroupsList = this.store.state.security.workerGroupsListAll || []

6
dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue

@ -243,16 +243,10 @@
},
_getNotifyGroupList () {
return new Promise((resolve, reject) => {
let notifyGroupListS = _.cloneDeep(this.store.state.dag.notifyGroupListS) || []
if (!notifyGroupListS.length) {
this.store.dispatch('dag/getNotifyGroupList').then(res => {
this.notifyGroupList = res
resolve()
})
} else {
this.notifyGroupList = notifyGroupListS
resolve()
}
})
},
_getReceiver () {

6
dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue

@ -280,8 +280,6 @@
_getNotifyGroupList () {
return new Promise((resolve, reject) => {
let notifyGroupListS = _.cloneDeep(this.store.state.dag.notifyGroupListS) || []
if (!notifyGroupListS.length) {
this.store.dispatch('dag/getNotifyGroupList').then(res => {
this.notifyGroupList = res
if (this.notifyGroupList.length) {
@ -290,10 +288,6 @@
reject(new Error(0))
}
})
} else {
this.notifyGroupList = notifyGroupListS
resolve()
}
})
},
ok () {

25
e2e/src/test/java/org/apache/dolphinscheduler/data/project/ProcessInstanceData.java

@ -0,0 +1,25 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.data.project;
public class ProcessInstanceData {
//Process Instance page title
public static final String PROCESS_INSTANCE_TITLE = "工作流实例 - DolphinScheduler";
public static final String RERUN_TYPE= "重跑";
}

2
e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreateProjectData.java → e2e/src/test/java/org/apache/dolphinscheduler/data/project/ProjectData.java

@ -16,7 +16,7 @@
*/
package org.apache.dolphinscheduler.data.project;
public class CreateProjectData {
public class ProjectData {
// create project name
public static final String PROJECT_NAME = "selenium_project_1";
// create project description

2
e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreateWorkflowData.java → e2e/src/test/java/org/apache/dolphinscheduler/data/project/WorkflowDefineData.java

@ -16,7 +16,7 @@
*/
package org.apache.dolphinscheduler.data.project;
public class CreateWorkflowData {
public class WorkflowDefineData {
/**
* create workflow data
*/

31
e2e/src/test/java/org/apache/dolphinscheduler/locator/project/ProcessInstanceLocator.java

@ -0,0 +1,31 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.locator.project;
import org.openqa.selenium.By;
public class ProcessInstanceLocator {
// jump Process Instance page
//click Process Instance name
public static final By CLICK_PROCESS_INSTANCE_NAME = By.xpath("//div[3]/div/ul/li[2]");
// click rerun button
public static final By CLICK_RERUN_BUTTON = By.xpath("//tr[2]/td[14]/div[1]/button[2]");
//assert rerun type
public static final By RUNNING_TYPE = By.xpath("//tr[2]/td[5]/span");
}

9
e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateProjectLocator.java → e2e/src/test/java/org/apache/dolphinscheduler/locator/project/ProjectLocator.java

@ -18,17 +18,12 @@ package org.apache.dolphinscheduler.locator.project;
import org.openqa.selenium.By;
public class CreateProjectLocator {
public class ProjectLocator {
//click project manage
// public static final By PROJECT_MANAGE = By.xpath("//div[@class='clearfix list'][2]");
public static final By PROJECT_MANAGE = By.xpath("//div[2]/div[2]/div/a/span");
// public static final By SECURITY_MANAGE = By.xpath("//div[@class='m-top']/div/div[2]/dev[@class='clearfix list'][6]");
public static final By SECURITY_MANAGE = By.xpath("//div[2]/div[6]/div/a/span");
//click create project button
public static final By CREATE_PROJECT_BUTTON = By.xpath("//div[2]/div/div[1]/button/span");
public static final By CREATE_PROJECT_BUTTON = By.xpath("//button/span");
//input project name
public static final By PROJECT_NAME = By.xpath("//div[2]/div/div/div[2]/div/input");

4
e2e/src/test/java/org/apache/dolphinscheduler/locator/project/TimingLocator.java

@ -45,9 +45,9 @@ public class TimingLocator {
public static final By CLICK_CREATE_BUTTON = By.xpath("//div[12]/button[2]/span");
//edit timing
public static final By CLICK_TIMING_MANAGEMENT_BUTTON = By.xpath("//div[2]/div/div[3]/div[1]/div/table/tr[2]/td[10]/button[6]");
public static final By CLICK_TIMING_MANAGEMENT_BUTTON = By.xpath("//tr[2]/td[10]/button[6]");
public static final By CLICK_EDIT_TIMING_BUTTON = By.xpath("//div[2]/div[3]/div/div[2]/div[1]/table/tr[2]/td[10]/button[1]");
public static final By CLICK_EDIT_TIMING_BUTTON = By.xpath("//tr[2]/td[10]/button[1]/i");
//online timing
public static final By CLICK_ONLINE_TIMING_BUTTON = By.xpath("//td[10]/button[2]");

2
e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateWorkflowLocator.java → e2e/src/test/java/org/apache/dolphinscheduler/locator/project/WorkflowDefineLocator.java

@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.locator.project;
import org.openqa.selenium.By;
public class CreateWorkflowLocator {
public class WorkflowDefineLocator {
/**
* jump workflow define page

226
e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateWorkflowPage.java

@ -1,226 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.page.project;
import org.apache.dolphinscheduler.common.PageCommon;
import org.apache.dolphinscheduler.constant.TestConstant;
import org.apache.dolphinscheduler.data.project.CreateWorkflowData;
import org.apache.dolphinscheduler.locator.project.CreateWorkflowLocator;
import org.apache.dolphinscheduler.locator.project.RunWorkflowLocator;
import org.openqa.selenium.WebDriver;
public class CreateWorkflowPage extends PageCommon {
public CreateWorkflowPage(WebDriver driver) {
super(driver);
}
/**
* jump create workflow page
*/
public boolean jumpWorkflowPage() throws InterruptedException {
// click project name
clickElement(CreateWorkflowLocator.CLICK_PROJECT_NAME);
Thread.sleep(TestConstant.ONE_THOUSAND);
System.out.println("Click on workflow define to jump to workflow define page");
// click workflow define
clickElement(CreateWorkflowLocator.CLICK_WORKFLOW_DEFINE);
return ifTitleContains(CreateWorkflowData.WORKFLOW_TITLE);
}
public boolean createWorkflow() throws InterruptedException {
System.out.println("Click create workflow button");
// click create workflow button
clickElement(CreateWorkflowLocator.CLICK_CREATE_WORKFLOW_BUTTON);
System.out.println("drag shell task");
//drag shell_task
dragAndDrop(CreateWorkflowLocator.MOUSE_DOWN_AT_SHELL,CreateWorkflowLocator.MOUSE_MOVE_SHELL_AT_DAG);
//input shell task _name
sendInput(CreateWorkflowLocator.INPUT_SHELL_TASK_NAME , CreateWorkflowData.SHELL_TASK_NAME);
//click stop run type
clickElement(CreateWorkflowLocator.CLICK_STOP_RUN_TYPE);
//click normal run type
clickElement(CreateWorkflowLocator.CLICK_NORMAL_RUN_TYPE);
//input shell task description
sendInput(CreateWorkflowLocator.INPUT_SHELL_TASK_DESCRIPTION , CreateWorkflowData.SHELL_TASK_DESCRIPTION);
//select task priority
clickElement(CreateWorkflowLocator.CLICK_TASK_PRIORITY);
clickElement(CreateWorkflowLocator.SELECT_TASK_PRIORITY);
//select work group
clickElement(CreateWorkflowLocator.CLICK_WORK_GROUP);
clickElement(CreateWorkflowLocator.SELECT_WORK_GROUP);
//select number of failed retries
clickElement(CreateWorkflowLocator.SELECT_FAIL_RETRIES_NUMBER);
//select failed retry interval
clickElement(CreateWorkflowLocator.SELECT_FAIL_RETRIES_INTERVAL);
//click timeout alarm
clickElement(CreateWorkflowLocator.CLICK_TIMEOUT_ALARM);
//select timeout fail
clickElement(CreateWorkflowLocator.SELECT_TIMEOUT_FAIL);
//cancel timeout alarm
clickElement(CreateWorkflowLocator.CANCEL_TIMEOUT_ALARM);
//select timeout alarm
clickElement(CreateWorkflowLocator.SELECT_TIMEOUT_ALARM);
//clear timeout
clearInput(CreateWorkflowLocator.SELECT_TIMEOUT);
clearInput(CreateWorkflowLocator.SELECT_TIMEOUT);
//input timeout
sendInput(CreateWorkflowLocator.SELECT_TIMEOUT, CreateWorkflowData.INPUT_TIMEOUT);
//click codeMirror and input script
inputCodeMirror(CreateWorkflowLocator.CLICK_CODE_MIRROR, CreateWorkflowLocator.INPUT_SCRIPT, CreateWorkflowData.SHELL_SCRIPT);
scrollToElementBottom(CreateWorkflowLocator.SCROLL_BOTTOM);
//click custom parameters
clickElement(CreateWorkflowLocator.CLICK_CUSTOM_PARAMETERS);
//input custom parameters
sendInput(CreateWorkflowLocator.INPUT_CUSTOM_PARAMETERS, CreateWorkflowData.INPUT_CUSTOM_PARAMETERS);
//input custom parameters value
sendInput(CreateWorkflowLocator.INPUT_CUSTOM_PARAMETERS_VALUE, CreateWorkflowData.INPUT_CUSTOM_PARAMETERS_VALUE);
//click add custom parameters
clickElement(CreateWorkflowLocator.CLICK_ADD_CUSTOM_PARAMETERS);
scrollToElementBottom(CreateWorkflowLocator.SCROLL_BOTTOM);
//input add custom parameters
sendInput(CreateWorkflowLocator.INPUT_ADD_CUSTOM_PARAMETERS, CreateWorkflowData.INPUT_ADD_CUSTOM_PARAMETERS);
//input add custom parameters value
sendInput(CreateWorkflowLocator.INPUT_ADD_CUSTOM_PARAMETERS_VALUE, CreateWorkflowData.INPUT_ADD_CUSTOM_PARAMETERS_VALUE);
//click delete custom parameters
clickElement(CreateWorkflowLocator.CLICK_DELETE_CUSTOM_PARAMETERS);
//click submit button
clickElement(CreateWorkflowLocator.CLICK_SUBMIT_BUTTON);
Thread.sleep(TestConstant.ONE_THOUSAND);
System.out.println("Task node set up successfully");
System.out.println("move to Dag Element ");
moveToDragElement(CreateWorkflowLocator.MOUSE_MOVE_SHELL_AT_DAG,-300,-100);
return ifTitleContains(CreateWorkflowData.CREATE_WORKFLOW_TITLE);
}
/**
* save workflow
*/
public boolean saveWorkflow() throws InterruptedException {
System.out.println("start to save workflow ");
//click save workflow button
clickElement(CreateWorkflowLocator.CLICK_SAVE_WORKFLOW_BUTTON);
//input workflow name
sendInput(CreateWorkflowLocator.INPUT_WORKFLOW_NAME, CreateWorkflowData.INPUT_WORKFLOW_NAME);
//input workflow description
sendInput(CreateWorkflowLocator.INPUT_WORKFLOW_DESCRIPTION, CreateWorkflowData.INPUT_WORKFLOW_DESCRIPTION);
//select tenant
clickElement(CreateWorkflowLocator.CLICK_TENANT);
clickElement(CreateWorkflowLocator.SELECT_TENANT);
//click workflow timeout alarm
clickElement(CreateWorkflowLocator.CLICK_WORKFLOW_TIMEOUT_ALARM);
clearInput(CreateWorkflowLocator.INPUT_WORKFLOW_TIMEOUT);
//input workflow timeout
sendInput(CreateWorkflowLocator.INPUT_WORKFLOW_TIMEOUT, CreateWorkflowData.INPUT_WORKFLOW_TIMEOUT);
//click workflow global parameters
clickElement(CreateWorkflowLocator.CLICK_WORKFLOW_GLOBAL_PARAMETERS);
//input workflow global parameters
sendInput(CreateWorkflowLocator.INPUT_WORKFLOW_GLOBAL_PARAMETERS, CreateWorkflowData.INPUT_WORKFLOW_GLOBAL_PARAMETERS);
//input workflow global parameters value
sendInput(CreateWorkflowLocator.INPUT_WORKFLOW_GLOBAL_PARAMETERS_VALUES, CreateWorkflowData.INPUT_WORKFLOW_GLOBAL_PARAMETERS_VALUES);
//click to add workflow global parameters
clickElement(CreateWorkflowLocator.CLICK_ADD_WORKFLOW_GLOBAL_PARAMETERS);
//input to add workflow global parameters
sendInput(CreateWorkflowLocator.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS, CreateWorkflowData.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS);
//input to add workflow global parameters value
sendInput(CreateWorkflowLocator.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS_VALUES, CreateWorkflowData.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS_VALUES);
//delete workflow global parameters value
clickElement(CreateWorkflowLocator.CLICK_DELETE_WORKFLOW_GLOBAL_PARAMETERS);
Thread.sleep(TestConstant.ONE_THOUSAND);
//click add button
System.out.println("submit workflow");
clickButton(CreateWorkflowLocator.CLICK_ADD_BUTTON);
return ifTitleContains(CreateWorkflowData.CREATE_WORKFLOW_TITLE);
}
public boolean onlineWorkflow() throws InterruptedException {
clickElement(CreateWorkflowLocator.CLICK_WORKFLOW_DEFINE);
// click online button
System.out.println("Click online workflow button");
clickButton(CreateWorkflowLocator.CLICK_ONLINE_WORKFLOW_BUTTON);
return ifTitleContains(CreateWorkflowData.WORKFLOW_TITLE);
}
public boolean offlineWorkflow() throws InterruptedException {
clickElement(CreateWorkflowLocator.CLICK_WORKFLOW_DEFINE);
// click offline button
System.out.println("offline workflow");
Thread.sleep(500);
clickButton(CreateWorkflowLocator.CLICK_OFFLINE_WORKFLOW_BUTTON);
return ifTitleContains(CreateWorkflowData.WORKFLOW_TITLE);
}
public boolean deleteWorkflow() throws InterruptedException {
//click delete workflow
Thread.sleep(500);
clickButton(CreateWorkflowLocator.DELETE_WORKFLOW_BOTTOM);
//click confirm delete project
clickButton(CreateWorkflowLocator.CONFIRM_DELETE_WORKFLOW_BOTTOM);
// Whether to enter the specified page after submit
return ifTitleContains(CreateWorkflowData.WORKFLOW_TITLE);
}
}

40
e2e/src/test/java/org/apache/dolphinscheduler/page/project/ProcessInstancePage.java

@ -0,0 +1,40 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.page.project;
import org.apache.dolphinscheduler.common.PageCommon;
import org.apache.dolphinscheduler.locator.project.ProcessInstanceLocator;
import org.openqa.selenium.WebDriver;
import org.apache.dolphinscheduler.data.project.ProcessInstanceData;
public class ProcessInstancePage extends PageCommon {
public ProcessInstancePage(WebDriver driver) {
super(driver);
}
/**
* rerun workflow page
*/
public boolean rerunWorkflowPage() throws InterruptedException {
clickTopElement(ProcessInstanceLocator.CLICK_PROCESS_INSTANCE_NAME);
clickTopElement(ProcessInstanceLocator.CLICK_RERUN_BUTTON);
return ifTitleContains(ProcessInstanceData.PROCESS_INSTANCE_TITLE);
}
}

29
e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateProjectPage.java → e2e/src/test/java/org/apache/dolphinscheduler/page/project/ProjectPage.java

@ -18,12 +18,12 @@ package org.apache.dolphinscheduler.page.project;
import org.apache.dolphinscheduler.common.PageCommon;
import org.apache.dolphinscheduler.constant.TestConstant;
import org.apache.dolphinscheduler.data.project.CreateProjectData;
import org.apache.dolphinscheduler.locator.project.CreateProjectLocator;
import org.apache.dolphinscheduler.data.project.ProjectData;
import org.apache.dolphinscheduler.locator.project.ProjectLocator;
import org.openqa.selenium.WebDriver;
public class CreateProjectPage extends PageCommon {
public CreateProjectPage(WebDriver driver) {
public class ProjectPage extends PageCommon {
public ProjectPage(WebDriver driver) {
super(driver);
}
@ -31,9 +31,9 @@ public class CreateProjectPage extends PageCommon {
* jump to ProjectManagePage
*/
public boolean jumpProjectManagePage() throws InterruptedException {
clickTopElement(CreateProjectLocator.PROJECT_MANAGE);
clickTopElement(ProjectLocator.PROJECT_MANAGE);
Thread.sleep(TestConstant.ONE_THOUSAND);
return ifTitleContains(CreateProjectData.PROJECT_TITLE);
return ifTitleContains(ProjectData.PROJECT_TITLE);
}
/**
@ -42,17 +42,18 @@ public class CreateProjectPage extends PageCommon {
* @return Whether to enter the specified page after create project
*/
public boolean createProject() throws InterruptedException {
clickElement(CreateProjectLocator.CREATE_PROJECT_BUTTON);
Thread.sleep(500);
clickElement(ProjectLocator.CREATE_PROJECT_BUTTON);
// input create project data
sendInput(CreateProjectLocator.PROJECT_NAME, CreateProjectData.PROJECT_NAME);
sendInput(CreateProjectLocator.PROJECT_DESCRIPTION, CreateProjectData.DESCRIPTION);
sendInput(ProjectLocator.PROJECT_NAME, ProjectData.PROJECT_NAME);
sendInput(ProjectLocator.PROJECT_DESCRIPTION, ProjectData.DESCRIPTION);
// click submit button
clickButton(CreateProjectLocator.SUBMIT_BUTTON);
clickButton(ProjectLocator.SUBMIT_BUTTON);
// Whether to enter the specified page after submit
return ifTitleContains(CreateProjectData.PROJECT_TITLE);
return ifTitleContains(ProjectData.PROJECT_TITLE);
}
/**
@ -62,12 +63,12 @@ public class CreateProjectPage extends PageCommon {
*/
public boolean deleteProject() throws InterruptedException {
//click delete project
clickElement(CreateProjectLocator.DELETE_PROJECT_BUTTON);
clickElement(ProjectLocator.DELETE_PROJECT_BUTTON);
//click confirm delete project
clickElement(CreateProjectLocator.CONFIRM_DELETE_PROJECT_BUTTON);
clickElement(ProjectLocator.CONFIRM_DELETE_PROJECT_BUTTON);
// Whether to enter the specified page after submit
return ifTitleContains(CreateProjectData.PROJECT_TITLE);
return ifTitleContains(ProjectData.PROJECT_TITLE);
}
}

4
e2e/src/test/java/org/apache/dolphinscheduler/page/project/TimingPage.java

@ -64,11 +64,11 @@ public class TimingPage extends PageCommon {
public boolean editTiming() throws InterruptedException {
// click timing button
System.out.println("Click timing management button");
Thread.sleep(1000);
clickButton(TimingLocator.CLICK_TIMING_MANAGEMENT_BUTTON);
Thread.sleep(500);
Thread.sleep(1000);
System.out.println("Click edit timing button");
clickButton(TimingLocator.CLICK_EDIT_TIMING_BUTTON);
Thread.sleep(500);
System.out.println("Click execution timing button");
clickButton(TimingLocator.CLICK_EXECUTION_TIMING_BUTTON);
Thread.sleep(1000);

226
e2e/src/test/java/org/apache/dolphinscheduler/page/project/WorkflowDefinePage.java

@ -0,0 +1,226 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.page.project;
import org.apache.dolphinscheduler.common.PageCommon;
import org.apache.dolphinscheduler.constant.TestConstant;
import org.apache.dolphinscheduler.data.project.WorkflowDefineData;
import org.apache.dolphinscheduler.locator.project.WorkflowDefineLocator;
import org.openqa.selenium.WebDriver;
public class WorkflowDefinePage extends PageCommon {
public WorkflowDefinePage(WebDriver driver) {
super(driver);
}
/**
* jump create workflow page
*/
public boolean jumpWorkflowPage() throws InterruptedException {
// click project name
clickElement(WorkflowDefineLocator.CLICK_PROJECT_NAME);
Thread.sleep(TestConstant.ONE_THOUSAND);
System.out.println("Click on workflow define to jump to workflow define page");
// click workflow define
clickElement(WorkflowDefineLocator.CLICK_WORKFLOW_DEFINE);
return ifTitleContains(WorkflowDefineData.WORKFLOW_TITLE);
}
public boolean createWorkflow() throws InterruptedException {
System.out.println("Click create workflow button");
// click create workflow button
clickElement(WorkflowDefineLocator.CLICK_CREATE_WORKFLOW_BUTTON);
System.out.println("drag shell task");
//drag shell_task
dragAndDrop(WorkflowDefineLocator.MOUSE_DOWN_AT_SHELL, WorkflowDefineLocator.MOUSE_MOVE_SHELL_AT_DAG);
//input shell task _name
sendInput(WorkflowDefineLocator.INPUT_SHELL_TASK_NAME , WorkflowDefineData.SHELL_TASK_NAME);
//click stop run type
clickElement(WorkflowDefineLocator.CLICK_STOP_RUN_TYPE);
//click normal run type
clickElement(WorkflowDefineLocator.CLICK_NORMAL_RUN_TYPE);
//input shell task description
sendInput(WorkflowDefineLocator.INPUT_SHELL_TASK_DESCRIPTION , WorkflowDefineData.SHELL_TASK_DESCRIPTION);
//select task priority
clickElement(WorkflowDefineLocator.CLICK_TASK_PRIORITY);
clickElement(WorkflowDefineLocator.SELECT_TASK_PRIORITY);
//select work group
clickElement(WorkflowDefineLocator.CLICK_WORK_GROUP);
clickElement(WorkflowDefineLocator.SELECT_WORK_GROUP);
//select number of failed retries
clickElement(WorkflowDefineLocator.SELECT_FAIL_RETRIES_NUMBER);
//select failed retry interval
clickElement(WorkflowDefineLocator.SELECT_FAIL_RETRIES_INTERVAL);
//click timeout alarm
clickElement(WorkflowDefineLocator.CLICK_TIMEOUT_ALARM);
//select timeout fail
clickElement(WorkflowDefineLocator.SELECT_TIMEOUT_FAIL);
//cancel timeout alarm
clickElement(WorkflowDefineLocator.CANCEL_TIMEOUT_ALARM);
//select timeout alarm
clickElement(WorkflowDefineLocator.SELECT_TIMEOUT_ALARM);
//clear timeout
clearInput(WorkflowDefineLocator.SELECT_TIMEOUT);
clearInput(WorkflowDefineLocator.SELECT_TIMEOUT);
//input timeout
sendInput(WorkflowDefineLocator.SELECT_TIMEOUT, WorkflowDefineData.INPUT_TIMEOUT);
//click codeMirror and input script
inputCodeMirror(WorkflowDefineLocator.CLICK_CODE_MIRROR, WorkflowDefineLocator.INPUT_SCRIPT, WorkflowDefineData.SHELL_SCRIPT);
scrollToElementBottom(WorkflowDefineLocator.SCROLL_BOTTOM);
//click custom parameters
clickElement(WorkflowDefineLocator.CLICK_CUSTOM_PARAMETERS);
//input custom parameters
sendInput(WorkflowDefineLocator.INPUT_CUSTOM_PARAMETERS, WorkflowDefineData.INPUT_CUSTOM_PARAMETERS);
//input custom parameters value
sendInput(WorkflowDefineLocator.INPUT_CUSTOM_PARAMETERS_VALUE, WorkflowDefineData.INPUT_CUSTOM_PARAMETERS_VALUE);
//click add custom parameters
clickElement(WorkflowDefineLocator.CLICK_ADD_CUSTOM_PARAMETERS);
scrollToElementBottom(WorkflowDefineLocator.SCROLL_BOTTOM);
//input add custom parameters
sendInput(WorkflowDefineLocator.INPUT_ADD_CUSTOM_PARAMETERS, WorkflowDefineData.INPUT_ADD_CUSTOM_PARAMETERS);
//input add custom parameters value
sendInput(WorkflowDefineLocator.INPUT_ADD_CUSTOM_PARAMETERS_VALUE, WorkflowDefineData.INPUT_ADD_CUSTOM_PARAMETERS_VALUE);
//click delete custom parameters
clickElement(WorkflowDefineLocator.CLICK_DELETE_CUSTOM_PARAMETERS);
//click submit button
clickElement(WorkflowDefineLocator.CLICK_SUBMIT_BUTTON);
Thread.sleep(TestConstant.ONE_THOUSAND);
System.out.println("Task node set up successfully");
System.out.println("move to Dag Element ");
moveToDragElement(WorkflowDefineLocator.MOUSE_MOVE_SHELL_AT_DAG,-300,-100);
return ifTitleContains(WorkflowDefineData.CREATE_WORKFLOW_TITLE);
}
/**
* save workflow
*/
public boolean saveWorkflow() throws InterruptedException {
System.out.println("start to save workflow ");
//click save workflow button
clickElement(WorkflowDefineLocator.CLICK_SAVE_WORKFLOW_BUTTON);
//input workflow name
sendInput(WorkflowDefineLocator.INPUT_WORKFLOW_NAME, WorkflowDefineData.INPUT_WORKFLOW_NAME);
//input workflow description
sendInput(WorkflowDefineLocator.INPUT_WORKFLOW_DESCRIPTION, WorkflowDefineData.INPUT_WORKFLOW_DESCRIPTION);
//select tenant
clickElement(WorkflowDefineLocator.CLICK_TENANT);
clickElement(WorkflowDefineLocator.SELECT_TENANT);
//click workflow timeout alarm
clickElement(WorkflowDefineLocator.CLICK_WORKFLOW_TIMEOUT_ALARM);
clearInput(WorkflowDefineLocator.INPUT_WORKFLOW_TIMEOUT);
//input workflow timeout
sendInput(WorkflowDefineLocator.INPUT_WORKFLOW_TIMEOUT, WorkflowDefineData.INPUT_WORKFLOW_TIMEOUT);
//click workflow global parameters
clickElement(WorkflowDefineLocator.CLICK_WORKFLOW_GLOBAL_PARAMETERS);
//input workflow global parameters
sendInput(WorkflowDefineLocator.INPUT_WORKFLOW_GLOBAL_PARAMETERS, WorkflowDefineData.INPUT_WORKFLOW_GLOBAL_PARAMETERS);
//input workflow global parameters value
sendInput(WorkflowDefineLocator.INPUT_WORKFLOW_GLOBAL_PARAMETERS_VALUES, WorkflowDefineData.INPUT_WORKFLOW_GLOBAL_PARAMETERS_VALUES);
//click to add workflow global parameters
clickElement(WorkflowDefineLocator.CLICK_ADD_WORKFLOW_GLOBAL_PARAMETERS);
//input to add workflow global parameters
sendInput(WorkflowDefineLocator.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS, WorkflowDefineData.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS);
//input to add workflow global parameters value
sendInput(WorkflowDefineLocator.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS_VALUES, WorkflowDefineData.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS_VALUES);
//delete workflow global parameters value
clickElement(WorkflowDefineLocator.CLICK_DELETE_WORKFLOW_GLOBAL_PARAMETERS);
Thread.sleep(TestConstant.ONE_THOUSAND);
//click add button
System.out.println("submit workflow");
clickButton(WorkflowDefineLocator.CLICK_ADD_BUTTON);
return ifTitleContains(WorkflowDefineData.CREATE_WORKFLOW_TITLE);
}
public boolean onlineWorkflow() throws InterruptedException {
clickElement(WorkflowDefineLocator.CLICK_WORKFLOW_DEFINE);
// click online button
System.out.println("Click online workflow button");
clickButton(WorkflowDefineLocator.CLICK_ONLINE_WORKFLOW_BUTTON);
return ifTitleContains(WorkflowDefineData.WORKFLOW_TITLE);
}
public boolean offlineWorkflow() throws InterruptedException {
clickElement(WorkflowDefineLocator.CLICK_WORKFLOW_DEFINE);
// click offline button
System.out.println("offline workflow");
Thread.sleep(500);
clickButton(WorkflowDefineLocator.CLICK_OFFLINE_WORKFLOW_BUTTON);
return ifTitleContains(WorkflowDefineData.WORKFLOW_TITLE);
}
public boolean deleteWorkflow() throws InterruptedException {
//click delete workflow
Thread.sleep(500);
clickElement(WorkflowDefineLocator.CLICK_WORKFLOW_DEFINE);
clickButton(WorkflowDefineLocator.DELETE_WORKFLOW_BOTTOM);
//click confirm delete project
clickButton(WorkflowDefineLocator.CONFIRM_DELETE_WORKFLOW_BOTTOM);
// Whether to enter the specified page after submit
return ifTitleContains(WorkflowDefineData.WORKFLOW_TITLE);
}
}

6
e2e/src/test/java/org/apache/dolphinscheduler/testcase/testDeleteData/TestDeleteProject.java

@ -17,15 +17,15 @@
package org.apache.dolphinscheduler.testcase.testDeleteData;
import org.apache.dolphinscheduler.base.BaseTest;
import org.apache.dolphinscheduler.page.project.CreateProjectPage;
import org.apache.dolphinscheduler.page.project.ProjectPage;
import org.testng.annotations.Test;
public class TestDeleteProject extends BaseTest {
private CreateProjectPage createProjectPage;
private ProjectPage createProjectPage;
@Test(groups={"functionTests"},dependsOnGroups = { "login","project"},description = "TestDeleteProject")
public void testDeleteProject() throws InterruptedException {
createProjectPage = new CreateProjectPage(driver);
createProjectPage = new ProjectPage(driver);
//jump to project manage page
System.out.println("jump to the project manage page to delete project");
createProjectPage.jumpProjectManagePage();

26
e2e/src/test/java/org/apache/dolphinscheduler/testcase/testDeleteData/TestDeleteWorkflow.java

@ -17,27 +17,29 @@
package org.apache.dolphinscheduler.testcase.testDeleteData;
import org.apache.dolphinscheduler.base.BaseTest;
import org.apache.dolphinscheduler.page.project.CreateProjectPage;
import org.apache.dolphinscheduler.page.project.CreateWorkflowPage;
import org.apache.dolphinscheduler.page.project.ProjectPage;
import org.apache.dolphinscheduler.page.project.WorkflowDefinePage;
import org.testng.annotations.Test;
public class TestDeleteWorkflow extends BaseTest {
private CreateWorkflowPage createWorkflowPage;
private CreateProjectPage createProjectPage;
private WorkflowDefinePage createWorkflowPage;
private ProjectPage createProjectPage;
/**
* offline workflow
* @throws InterruptedException
*/
@Test(groups={"functionTests"},dependsOnGroups = { "login","workflow"},description = "TestDeleteWorkflow")
public void testDeleteWorkflow() throws InterruptedException {
createProjectPage = new CreateProjectPage(driver);
//jump to project manage page
// System.out.println("jump to the project manage page to delete workflow");
// createProjectPage.jumpProjectManagePage();
createWorkflowPage = new CreateWorkflowPage(driver);
// createWorkflowPage.jumpWorkflowPage();
public void testOfflineWorkflow() throws InterruptedException {
createWorkflowPage = new WorkflowDefinePage(driver);
System.out.println("start offline workflow");
assert createWorkflowPage.offlineWorkflow();
System.out.println("end offline workflow");
System.out.println("===================================");
}
@Test(groups={"functionTests"},dependsOnGroups = { "login","workflow"},description = "TestDeleteWorkflow")
public void testDeleteWorkflow() throws InterruptedException {
System.out.println("start delete workflow");
assert createWorkflowPage.deleteWorkflow();
System.out.println("end delete workflow");

35
e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestProcessInstance.java

@ -0,0 +1,35 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.testcase.testProject;
import org.apache.dolphinscheduler.base.BaseTest;
import org.apache.dolphinscheduler.page.project.ProcessInstancePage;
import org.testng.annotations.Test;
public class TestProcessInstance extends BaseTest {
private ProcessInstancePage processInstancePage;
@Test(groups={"functionTests","processInstance"},dependsOnGroups = { "login","workflow" },description = "testRerun")
public void testRerun() throws InterruptedException {
processInstancePage = new ProcessInstancePage(driver);
//rerun workflow
System.out.println("start rerun workflow");
processInstancePage.rerunWorkflowPage();
System.out.println("end rerun workflow");
System.out.println("===================================");
}
}

12
e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestCreateProject.java → e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestProject.java

@ -17,21 +17,21 @@
package org.apache.dolphinscheduler.testcase.testProject;
import org.apache.dolphinscheduler.base.BaseTest;
import org.apache.dolphinscheduler.page.project.CreateProjectPage;
import org.apache.dolphinscheduler.page.project.ProjectPage;
import org.testng.annotations.Test;
public class TestCreateProject extends BaseTest {
private CreateProjectPage createProjectPage;
public class TestProject extends BaseTest {
private ProjectPage projectPage;
@Test(groups={"functionTests","project"},dependsOnGroups = { "login" },description = "CreateProjectTest")
public void testCreateProject() throws InterruptedException {
createProjectPage = new CreateProjectPage(driver);
projectPage = new ProjectPage(driver);
// enter user manage page
System.out.println("jump to the projectManage page to create project ");
createProjectPage.jumpProjectManagePage();
projectPage.jumpProjectManagePage();
//assert user manage page
System.out.println("start create project");
assert createProjectPage.createProject();
assert projectPage.createProject();
System.out.println("end create project");
System.out.println("===================================");
}

10
e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestRunWorkflow.java

@ -17,14 +17,14 @@
package org.apache.dolphinscheduler.testcase.testProject;
import org.apache.dolphinscheduler.base.BaseTest;
import org.apache.dolphinscheduler.page.project.CreateProjectPage;
import org.apache.dolphinscheduler.page.project.CreateWorkflowPage;
import org.apache.dolphinscheduler.page.project.ProjectPage;
import org.apache.dolphinscheduler.page.project.WorkflowDefinePage;
import org.apache.dolphinscheduler.page.project.RunWorkflowPage;
import org.testng.annotations.Test;
public class TestRunWorkflow extends BaseTest {
private CreateWorkflowPage createWorkflowPage;
private CreateProjectPage createProjectPage;
private WorkflowDefinePage createWorkflowPage;
private ProjectPage projectPage;
private RunWorkflowPage runWorkflowPage;
@ -32,7 +32,7 @@ public class TestRunWorkflow extends BaseTest {
public void testRunWorkflow() throws InterruptedException {
runWorkflowPage = new RunWorkflowPage(driver);
createProjectPage = new CreateProjectPage(driver);
projectPage = new ProjectPage(driver);
System.out.println("start run workflow");
assert runWorkflowPage.runWorkflow();
System.out.println("end run workflow");

36
e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestCreateWorkflow.java → e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestWorkflowDefine.java

@ -17,32 +17,40 @@
package org.apache.dolphinscheduler.testcase.testProject;
import org.apache.dolphinscheduler.base.BaseTest;
import org.apache.dolphinscheduler.page.project.CreateProjectPage;
import org.apache.dolphinscheduler.page.project.CreateWorkflowPage;
import org.apache.dolphinscheduler.page.project.ProjectPage;
import org.apache.dolphinscheduler.page.project.WorkflowDefinePage;
import org.testng.annotations.Test;
public class TestCreateWorkflow extends BaseTest {
private CreateWorkflowPage createWorkflowPage;
private CreateProjectPage createProjectPage;
public class TestWorkflowDefine extends BaseTest {
private WorkflowDefinePage workflowDefinePage;
private ProjectPage projectPage;
@Test(groups={"functionTests","workflow"},dependsOnGroups = { "login" },description = "TestCreateWorkflow")
/**
* test Create WorkflowDefine
* @throws InterruptedException
*/
@Test(groups={"functionTests","workflow"},dependsOnGroups = { "login" },description = "TestWorkflowDefine")
public void testCreateWorkflow() throws InterruptedException {
createProjectPage = new CreateProjectPage(driver);
projectPage = new ProjectPage(driver);
System.out.println("jump to the projectManage page to create workflow");
createProjectPage.jumpProjectManagePage();
projectPage.jumpProjectManagePage();
createWorkflowPage = new CreateWorkflowPage(driver);
workflowDefinePage = new WorkflowDefinePage(driver);
System.out.println("Click on the project name to jump to the project homepage");
createWorkflowPage.jumpWorkflowPage();
workflowDefinePage.jumpWorkflowPage();
System.out.println("start create workflow");
assert createWorkflowPage.createWorkflow();
assert createWorkflowPage.saveWorkflow();
assert workflowDefinePage.createWorkflow();
assert workflowDefinePage.saveWorkflow();
System.out.println("end create workflow");
System.out.println("===================================");
}
@Test(groups={"functionTests","workflow"},dependsOnGroups = { "login" },description = "TestOnlineWorkflow")
public void testOnlineWorkflow() throws InterruptedException {
workflowDefinePage = new WorkflowDefinePage(driver);
System.out.println("start online workflow");
assert createWorkflowPage.onlineWorkflow();
assert workflowDefinePage.onlineWorkflow();
System.out.println("end online workflow");
System.out.println("===================================");
}

29
e2e/testng.xml

@ -26,6 +26,7 @@
<classes>
<class name="org.apache.dolphinscheduler.testcase.TestLogin"></class>
<class name="org.apache.dolphinscheduler.testcase.testSecurity.TestTenantManage"></class>
<class name="org.apache.dolphinscheduler.testcase.testSecurity.TestAlertManage"></class>
<class name="org.apache.dolphinscheduler.testcase.testSecurity.TestQueueManage"></class>
<class name="org.apache.dolphinscheduler.testcase.testSecurity.TestTokenManage"></class>
@ -37,8 +38,19 @@
</methods>
</class>
<class name="org.apache.dolphinscheduler.testcase.testProject.TestCreateProject"></class>
<class name="org.apache.dolphinscheduler.testcase.testProject.TestCreateWorkflow"></class>
<class name="org.apache.dolphinscheduler.testcase.testProject.TestProject">
<methods>
<include name="createProject" />
</methods>
</class>
<class name="org.apache.dolphinscheduler.testcase.testProject.TestWorkflowDefine">
<methods>
<include name="testCreateWorkflow" />
<include name="testOnlineWorkflow" />
</methods>
</class>
<class name="org.apache.dolphinscheduler.testcase.testProject.TestRunWorkflow"></class>
<class name="org.apache.dolphinscheduler.testcase.testProject.TestTiming">
@ -50,8 +62,19 @@
<include name="testDeleteTiming" />
</methods>
</class>
<class name="org.apache.dolphinscheduler.testcase.testProject.TestProcessInstance">
<methods>
<include name="testRerun" />
</methods>
</class>
<class name="org.apache.dolphinscheduler.testcase.testDeleteData.TestDeleteWorkflow">
<methods>
<include name="testOfflineWorkflow" />
<include name="testDeleteWorkflow" />
</methods>
</class>
<class name="org.apache.dolphinscheduler.testcase.testDeleteData.TestDeleteWorkflow"></class>
<class name="org.apache.dolphinscheduler.testcase.testDeleteData.TestDeleteProject"></class>
<class name="org.apache.dolphinscheduler.testcase.testDeleteData.TestDeleteAlert"></class>
<class name="org.apache.dolphinscheduler.testcase.testDeleteData.TestDeleteToken"></class>

2
script/dolphinscheduler-daemon.sh

@ -46,7 +46,7 @@ export DOLPHINSCHEDULER_LOG_DIR=$DOLPHINSCHEDULER_HOME/logs
export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf
export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/*
export DOLPHINSCHEDULER_OPTS="-server -Xmx16g -Xms1g -Xss512k -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=10m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-"-server -Xmx16g -Xms1g -Xss512k -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=10m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"}
export STOP_TIMEOUT=5
if [ ! -d "$DOLPHINSCHEDULER_LOG_DIR" ]; then

3
sql/dolphinscheduler-postgre.sql

@ -627,6 +627,7 @@ CREATE TABLE t_ds_user (
state int DEFAULT 1 ,
PRIMARY KEY (id)
);
comment on column t_ds_user.state is 'state 0:disable 1:enable';
--
-- Table structure for table t_ds_version
@ -751,7 +752,7 @@ ALTER TABLE t_ds_worker_server ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_worker_
-- Records of t_ds_user?user : admin , password : dolphinscheduler123
INSERT INTO t_ds_user(user_name,user_password,user_type,email,phone,tenant_id,create_time,update_time) VALUES ('admin', '7ad2410b2f4c074479a8937a28a22b8f', '0', 'xxx@qq.com', 'xx', '0', '2018-03-27 15:48:50', '2018-10-24 17:40:22');
INSERT INTO t_ds_user(user_name,user_password,user_type,email,phone,tenant_id,state,create_time,update_time) VALUES ('admin', '7ad2410b2f4c074479a8937a28a22b8f', '0', 'xxx@qq.com', 'xx', '0', 1, '2018-03-27 15:48:50', '2018-10-24 17:40:22');
-- Records of t_ds_alertgroup,dolphinscheduler warning group
INSERT INTO t_ds_alertgroup(group_name,group_type,description,create_time,update_time) VALUES ('dolphinscheduler warning group', '0', 'dolphinscheduler warning group','2018-11-29 10:20:39', '2018-11-29 10:20:39');

1
sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_ddl.sql

@ -296,6 +296,7 @@ BEGIN
AND COLUMN_NAME ='state')
THEN
ALTER TABLE t_ds_user ADD COLUMN state int DEFAULT 1;
comment on column t_ds_user.state is 'state 0:disable 1:enable';
END IF;
END;
$$ LANGUAGE plpgsql;

Loading…
Cancel
Save