break60 4 years ago
parent
commit
e654a59d52
  1. 4
      ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py
  2. 4
      ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/params.py
  3. 2
      docker/build/Dockerfile
  4. 66
      docker/build/README.md
  5. 66
      docker/build/README_zh_CN.md
  6. 14
      docker/build/conf/dolphinscheduler/alert.properties.tpl
  7. 12
      docker/build/conf/dolphinscheduler/common.properties.tpl
  8. 15
      docker/build/conf/dolphinscheduler/datasource.properties.tpl
  9. 2
      docker/build/conf/dolphinscheduler/logback/logback-alert.xml
  10. 2
      docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
  11. 23
      docker/build/startup-init-conf.sh
  12. 40
      docker/build/startup.sh
  13. 40
      docker/docker-swarm/docker-compose.yml
  14. 52
      docker/docker-swarm/docker-stack.yml
  15. 2
      docker/kubernetes/dolphinscheduler/Chart.yaml
  16. 40
      docker/kubernetes/dolphinscheduler/README.md
  17. 25
      docker/kubernetes/dolphinscheduler/requirements.yaml
  18. 13
      docker/kubernetes/dolphinscheduler/templates/NOTES.txt
  19. 8
      docker/kubernetes/dolphinscheduler/templates/_helpers.tpl
  20. 1
      docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml
  21. 35
      docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml
  22. 98
      docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml
  23. 93
      docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
  24. 17
      docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml
  25. 93
      docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
  26. 163
      docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
  27. 87
      docker/kubernetes/dolphinscheduler/values.yaml
  28. 30
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java
  29. 8
      dolphinscheduler-alert/src/main/resources/alert.properties
  30. 4
      dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java
  31. 2
      dolphinscheduler-ui/package.json
  32. 5
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/startingParam/index.vue
  33. 6
      dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue
  34. 6
      dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue
  35. 2
      script/dolphinscheduler-daemon.sh

4
ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py

@ -76,8 +76,8 @@ else:
dolphin_alert_map = {} dolphin_alert_map = {}
wechat_push_url = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token' wechat_push_url = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token'
wechat_token_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret' wechat_token_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret'
wechat_team_send_msg = '{\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"}' wechat_team_send_msg = '{\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"}'
wechat_user_send_msg = '{\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}}' wechat_user_send_msg = '{\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}}'
dolphin_alert_map['enterprise.wechat.push.ur'] = wechat_push_url dolphin_alert_map['enterprise.wechat.push.ur'] = wechat_push_url
dolphin_alert_map['enterprise.wechat.token.url'] = wechat_token_url dolphin_alert_map['enterprise.wechat.token.url'] = wechat_token_url

4
ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/params.py

@ -77,8 +77,8 @@ else:
dolphin_alert_map = {} dolphin_alert_map = {}
wechat_push_url = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token' wechat_push_url = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token'
wechat_token_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret' wechat_token_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret'
wechat_team_send_msg = '{\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"}' wechat_team_send_msg = '{\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"}'
wechat_user_send_msg = '{\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}}' wechat_user_send_msg = '{\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}}'
dolphin_alert_config_map = config['configurations']['dolphin-alert'] dolphin_alert_config_map = config['configurations']['dolphin-alert']

2
docker/build/Dockerfile

@ -42,7 +42,7 @@ ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz
RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ /opt/dolphinscheduler/ RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ /opt/dolphinscheduler/
ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
#4. install pg #4. install database, if use mysql as your backend database, the `mysql-client` package should be installed
RUN apk add postgresql postgresql-contrib RUN apk add postgresql postgresql-contrib
#5. modify nginx #5. modify nginx
RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \ RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \

66
docker/build/README.md

@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org
#### You can start a dolphinscheduler instance #### You can start a dolphinscheduler instance
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \ -e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \
-p 8888:8888 \ -p 8888:8888 \
dolphinscheduler all dolphinscheduler all
``` ```
@ -25,14 +25,14 @@ The default postgres user `root`, postgres password `root` and database `dolphin
The default zookeeper is created in the `startup.sh`. The default zookeeper is created in the `startup.sh`.
#### Or via Environment Variables **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`POSTGRESQL_DATABASE`** **`ZOOKEEPER_QUORUM`** #### Or via Environment Variables **`DATABASE_HOST`** **`DATABASE_PORT`** **`DATABASE_DATABASE`** **`ZOOKEEPER_QUORUM`**
You can specify **existing postgres service**. Example: You can specify **existing postgres service**. Example:
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-p 8888:8888 \ -p 8888:8888 \
dolphinscheduler all dolphinscheduler all
``` ```
@ -42,7 +42,7 @@ You can specify **existing zookeeper service**. Example:
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181" -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \
-p 8888:8888 \ -p 8888:8888 \
dolphinscheduler all dolphinscheduler all
``` ```
@ -56,8 +56,8 @@ You can start a standalone dolphinscheduler server.
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181" -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler master-server dolphinscheduler master-server
``` ```
@ -66,8 +66,8 @@ dolphinscheduler master-server
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181" -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler worker-server dolphinscheduler worker-server
``` ```
@ -75,8 +75,8 @@ dolphinscheduler worker-server
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-p 12345:12345 \ -p 12345:12345 \
dolphinscheduler api-server dolphinscheduler api-server
``` ```
@ -85,8 +85,8 @@ dolphinscheduler api-server
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler alert-server dolphinscheduler alert-server
``` ```
@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \
dolphinscheduler frontend dolphinscheduler frontend
``` ```
**Note**: You must be specify `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server. **Note**: You must be specify `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server.
## How to build a docker image ## How to build a docker image
@ -124,33 +124,51 @@ Please read `./docker/build/hooks/build` `./docker/build/hooks/build.bat` script
The Dolphin Scheduler image uses several environment variables which are easy to miss. While none of the variables are required, they may significantly aid you in using the image. The Dolphin Scheduler image uses several environment variables which are easy to miss. While none of the variables are required, they may significantly aid you in using the image.
**`POSTGRESQL_HOST`** **`DATABASE_TYPE`**
This environment variable sets the host for PostgreSQL. The default value is `127.0.0.1`. This environment variable sets the type for database. The default value is `postgresql`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. **Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`POSTGRESQL_PORT`** **`DATABASE_DRIVER`**
This environment variable sets the port for PostgreSQL. The default value is `5432`. This environment variable sets the type for database. The default value is `org.postgresql.Driver`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. **Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`POSTGRESQL_USERNAME`** **`DATABASE_HOST`**
This environment variable sets the username for PostgreSQL. The default value is `root`. This environment variable sets the host for database. The default value is `127.0.0.1`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. **Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`POSTGRESQL_PASSWORD`** **`DATABASE_PORT`**
This environment variable sets the password for PostgreSQL. The default value is `root`. This environment variable sets the port for database. The default value is `5432`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. **Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`POSTGRESQL_DATABASE`** **`DATABASE_USERNAME`**
This environment variable sets the database for PostgreSQL. The default value is `dolphinscheduler`. This environment variable sets the username for database. The default value is `root`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`DATABASE_PASSWORD`**
This environment variable sets the password for database. The default value is `root`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`DATABASE_DATABASE`**
This environment variable sets the database for database. The default value is `dolphinscheduler`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`DATABASE_PARAMS`**
This environment variable sets the database for database. The default value is `characterEncoding=utf8`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. **Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.

66
docker/build/README_zh_CN.md

@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org
#### 你可以运行一个dolphinscheduler实例 #### 你可以运行一个dolphinscheduler实例
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \ -e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \
-p 8888:8888 \ -p 8888:8888 \
dolphinscheduler all dolphinscheduler all
``` ```
@ -25,14 +25,14 @@ dolphinscheduler all
同时,默认的`Zookeeper`也会在`startup.sh`脚本中被创建。 同时,默认的`Zookeeper`也会在`startup.sh`脚本中被创建。
#### 或者通过环境变量 **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`ZOOKEEPER_QUORUM`** 使用已存在的服务 #### 或者通过环境变量 **`DATABASE_HOST`** **`DATABASE_PORT`** **`ZOOKEEPER_QUORUM`** 使用已存在的服务
你可以指定一个已经存在的 **`Postgres`** 服务. 如下: 你可以指定一个已经存在的 **`Postgres`** 服务. 如下:
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-p 8888:8888 \ -p 8888:8888 \
dolphinscheduler all dolphinscheduler all
``` ```
@ -42,7 +42,7 @@ dolphinscheduler all
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181" -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \
-p 8888:8888 \ -p 8888:8888 \
dolphinscheduler all dolphinscheduler all
``` ```
@ -56,8 +56,8 @@ dolphinscheduler all
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181" -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler master-server dolphinscheduler master-server
``` ```
@ -66,8 +66,8 @@ dolphinscheduler master-server
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181" -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler worker-server dolphinscheduler worker-server
``` ```
@ -75,8 +75,8 @@ dolphinscheduler worker-server
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-p 12345:12345 \ -p 12345:12345 \
dolphinscheduler api-server dolphinscheduler api-server
``` ```
@ -85,8 +85,8 @@ dolphinscheduler api-server
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler alert-server dolphinscheduler alert-server
``` ```
@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \
dolphinscheduler frontend dolphinscheduler frontend
``` ```
**注意**: 当你运行dolphinscheduler中的部分服务时,你必须指定这些环境变量 `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM` **注意**: 当你运行dolphinscheduler中的部分服务时,你必须指定这些环境变量 `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM`
## 如何构建一个docker镜像 ## 如何构建一个docker镜像
@ -124,33 +124,51 @@ c:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat
Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些变量不是必须的,但是可以帮助你更容易配置镜像并根据你的需求定义相应的服务配置。 Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些变量不是必须的,但是可以帮助你更容易配置镜像并根据你的需求定义相应的服务配置。
**`POSTGRESQL_HOST`** **`DATABASE_TYPE`**
配置`PostgreSQL`的`HOST`, 默认值 `127.0.0.1` 配置`database`的`TYPE`, 默认值 `postgresql`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。 **注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`POSTGRESQL_PORT`** **`DATABASE_DRIVER`**
配置`PostgreSQL`的`PORT`, 默认值 `5432` 配置`database`的`DRIVER`, 默认值 `org.postgresql.Driver`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。 **注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`POSTGRESQL_USERNAME`** **`DATABASE_HOST`**
配置`PostgreSQL`的`USERNAME`, 默认值 `root` 配置`database`的`HOST`, 默认值 `127.0.0.1`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。 **注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`POSTGRESQL_PASSWORD`** **`DATABASE_PORT`**
配置`PostgreSQL`的`PASSWORD`, 默认值 `root` 配置`database`的`PORT`, 默认值 `5432`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。 **注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`POSTGRESQL_DATABASE`** **`DATABASE_USERNAME`**
配置`PostgreSQL`的`DATABASE`, 默认值 `dolphinscheduler` 配置`database`的`USERNAME`, 默认值 `root`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`DATABASE_PASSWORD`**
配置`database`的`PASSWORD`, 默认值 `root`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`DATABASE_DATABASE`**
配置`database`的`DATABASE`, 默认值 `dolphinscheduler`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`DATABASE_PARAMS`**
配置`database`的`PARAMS`, 默认值 `characterEncoding=utf8`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。 **注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。

14
docker/build/conf/dolphinscheduler/alert.properties.tpl

@ -35,16 +35,22 @@ mail.smtp.ssl.trust=${MAIL_SMTP_SSL_TRUST}
#xls file path,need create if not exist #xls file path,need create if not exist
xls.file.path=${XLS_FILE_PATH} xls.file.path=${XLS_FILE_PATH}
# plugins dir
plugin.dir=${ALERT_PLUGIN_DIR}
# Enterprise WeChat configuration # Enterprise WeChat configuration
enterprise.wechat.enable=${ENTERPRISE_WECHAT_ENABLE} enterprise.wechat.enable=${ENTERPRISE_WECHAT_ENABLE}
enterprise.wechat.corp.id=${ENTERPRISE_WECHAT_CORP_ID} enterprise.wechat.corp.id=${ENTERPRISE_WECHAT_CORP_ID}
enterprise.wechat.secret=${ENTERPRISE_WECHAT_SECRET} enterprise.wechat.secret=${ENTERPRISE_WECHAT_SECRET}
enterprise.wechat.agent.id=${ENTERPRISE_WECHAT_AGENT_ID} enterprise.wechat.agent.id=${ENTERPRISE_WECHAT_AGENT_ID}
enterprise.wechat.users=${ENTERPRISE_WECHAT_USERS} enterprise.wechat.users=${ENTERPRISE_WECHAT_USERS}
enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid={corpId}&corpsecret={secret}
enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={token}
enterprise.wechat.team.send.msg={\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"} enterprise.wechat.team.send.msg={\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"}
enterprise.wechat.user.send.msg={\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}} enterprise.wechat.user.send.msg={\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}}

12
docker/build/conf/dolphinscheduler/common.properties.tpl

@ -25,13 +25,13 @@ dolphinscheduler.env.path=${DOLPHINSCHEDULER_ENV_PATH}
data.basedir.path=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH} data.basedir.path=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH}
# resource upload startup type : HDFS,S3,NONE # resource upload startup type : HDFS,S3,NONE
resource.storage.type=NONE resource.storage.type=${RESOURCE_STORAGE_TYPE}
#============================================================================ #============================================================================
# HDFS # HDFS
#============================================================================ #============================================================================
# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended # resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended
#resource.upload.path=/dolphinscheduler resource.upload.path=${RESOURCE_UPLOAD_PATH}
# whether kerberos starts # whether kerberos starts
#hadoop.security.authentication.startup.state=false #hadoop.security.authentication.startup.state=false
@ -58,16 +58,16 @@ kerberos.expire.time=7
# S3 # S3
#============================================================================ #============================================================================
# if resource.storage.type=S3,the value like: s3a://dolphinscheduler ; if resource.storage.type=HDFS, When namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir # if resource.storage.type=S3,the value like: s3a://dolphinscheduler ; if resource.storage.type=HDFS, When namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
fs.defaultFS=hdfs://mycluster:8020 fs.defaultFS=${FS_DEFAULT_FS}
# if resource.storage.type=S3,s3 endpoint # if resource.storage.type=S3,s3 endpoint
#fs.s3a.endpoint=http://192.168.199.91:9010 fs.s3a.endpoint=${FS_S3A_ENDPOINT}
# if resource.storage.type=S3,s3 access key # if resource.storage.type=S3,s3 access key
#fs.s3a.access.key=A3DXS30FO22544RE fs.s3a.access.key=${FS_S3A_ACCESS_KEY}
# if resource.storage.type=S3,s3 secret key # if resource.storage.type=S3,s3 secret key
#fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK fs.s3a.secret.key=${FS_S3A_SECRET_KEY}
# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty TODO # if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty TODO
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx

15
docker/build/conf/dolphinscheduler/datasource.properties.tpl

@ -15,16 +15,11 @@
# limitations under the License. # limitations under the License.
# #
# db
# mysql spring.datasource.driver-class-name=${DATABASE_DRIVER}
#spring.datasource.driver-class-name=com.mysql.jdbc.Driver spring.datasource.url=jdbc:${DATABASE_TYPE}://${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DATABASE}?${DATABASE_PARAMS}
#spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 spring.datasource.username=${DATABASE_USERNAME}
spring.datasource.password=${DATABASE_PASSWORD}
# postgre
spring.datasource.driver-class-name=org.postgresql.Driver
spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8
spring.datasource.username=${POSTGRESQL_USERNAME}
spring.datasource.password=${POSTGRESQL_PASSWORD}
## base spring data source configuration todo need to remove ## base spring data source configuration todo need to remove
#spring.datasource.type=com.alibaba.druid.pool.DruidDataSource #spring.datasource.type=com.alibaba.druid.pool.DruidDataSource

2
docker/build/conf/dolphinscheduler/logback/logback-alert.xml

@ -46,7 +46,7 @@
<root level="INFO"> <root level="INFO">
<appender-ref ref="STDOUT"/> <appender-ref ref="STDOUT"/>
<appender-ref ref="APILOGFILE"/> <appender-ref ref="ALERTLOGFILE"/>
</root> </root>
</configuration> </configuration>

2
docker/build/conf/dolphinscheduler/zookeeper.properties.tpl

@ -19,7 +19,7 @@
zookeeper.quorum=${ZOOKEEPER_QUORUM} zookeeper.quorum=${ZOOKEEPER_QUORUM}
# dolphinscheduler root directory # dolphinscheduler root directory
#zookeeper.dolphinscheduler.root=/dolphinscheduler zookeeper.dolphinscheduler.root=${ZOOKEEPER_ROOT}
# dolphinscheduler failover directory # dolphinscheduler failover directory
#zookeeper.session.timeout=60000 #zookeeper.session.timeout=60000

23
docker/build/startup-init-conf.sh

@ -24,22 +24,33 @@ echo "init env variables"
#============================================================================ #============================================================================
# Database Source # Database Source
#============================================================================ #============================================================================
export POSTGRESQL_HOST=${POSTGRESQL_HOST:-"127.0.0.1"} export DATABASE_HOST=${DATABASE_HOST:-"127.0.0.1"}
export POSTGRESQL_PORT=${POSTGRESQL_PORT:-"5432"} export DATABASE_PORT=${DATABASE_PORT:-"5432"}
export POSTGRESQL_USERNAME=${POSTGRESQL_USERNAME:-"root"} export DATABASE_USERNAME=${DATABASE_USERNAME:-"root"}
export POSTGRESQL_PASSWORD=${POSTGRESQL_PASSWORD:-"root"} export DATABASE_PASSWORD=${DATABASE_PASSWORD:-"root"}
export POSTGRESQL_DATABASE=${POSTGRESQL_DATABASE:-"dolphinscheduler"} export DATABASE_DATABASE=${DATABASE_DATABASE:-"dolphinscheduler"}
export DATABASE_TYPE=${DATABASE_TYPE:-"postgresql"}
export DATABASE_DRIVER=${DATABASE_DRIVER:-"org.postgresql.Driver"}
export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"}
#============================================================================ #============================================================================
# System # System
#============================================================================ #============================================================================
export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"} export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"}
export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"} export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"}
export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""}
export RESOURCE_STORAGE_TYPE=${RESOURCE_STORAGE_TYPE:-"NONE"}
export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/ds"}
export FS_DEFAULT_FS=${FS_DEFAULT_FS:-"s3a://xxxx"}
export FS_S3A_ENDPOINT=${FS_S3A_ENDPOINT:-"s3.xxx.amazonaws.com"}
export FS_S3A_ACCESS_KEY=${FS_S3A_ACCESS_KEY:-"xxxxxxx"}
export FS_S3A_SECRET_KEY=${FS_S3A_SECRET_KEY:-"xxxxxxx"}
#============================================================================ #============================================================================
# Zookeeper # Zookeeper
#============================================================================ #============================================================================
export ZOOKEEPER_QUORUM=${ZOOKEEPER_QUORUM:-"127.0.0.1:2181"} export ZOOKEEPER_QUORUM=${ZOOKEEPER_QUORUM:-"127.0.0.1:2181"}
export ZOOKEEPER_ROOT=${ZOOKEEPER_ROOT:-"/dolphinscheduler"}
#============================================================================ #============================================================================
# Master Server # Master Server
@ -67,6 +78,8 @@ export WORKER_GROUP=${WORKER_GROUP:-"default"}
#============================================================================ #============================================================================
# Alert Server # Alert Server
#============================================================================ #============================================================================
# alert plugin dir
export ALERT_PLUGIN_DIR=${ALERT_PLUGIN_DIR:-"/opt/dolphinscheduler"}
# XLS FILE # XLS FILE
export XLS_FILE_PATH=${XLS_FILE_PATH:-"/tmp/xls"} export XLS_FILE_PATH=${XLS_FILE_PATH:-"/tmp/xls"}
# mail # mail

40
docker/build/startup.sh

@ -22,24 +22,32 @@ DOLPHINSCHEDULER_BIN=${DOLPHINSCHEDULER_HOME}/bin
DOLPHINSCHEDULER_SCRIPT=${DOLPHINSCHEDULER_HOME}/script DOLPHINSCHEDULER_SCRIPT=${DOLPHINSCHEDULER_HOME}/script
DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs
# start postgresql # start database
initPostgreSQL() { initDatabase() {
echo "test postgresql service" echo "test ${DATABASE_TYPE} service"
while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
counter=$((counter+1)) counter=$((counter+1))
if [ $counter == 30 ]; then if [ $counter == 30 ]; then
echo "Error: Couldn't connect to postgresql." echo "Error: Couldn't connect to ${DATABASE_TYPE}."
exit 1 exit 1
fi fi
echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter." echo "Trying to connect to ${DATABASE_TYPE} at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
sleep 5 sleep 5
done done
echo "connect postgresql service" echo "connect ${DATABASE_TYPE} service"
v=$(sudo -u postgres PGPASSWORD=${POSTGRESQL_PASSWORD} psql -h ${POSTGRESQL_HOST} -p ${POSTGRESQL_PORT} -U ${POSTGRESQL_USERNAME} -d dolphinscheduler -tAc "select 1") if [ ${DATABASE_TYPE} = "mysql" ]; then
if [ "$(echo '${v}' | grep 'FATAL' | wc -l)" -eq 1 ]; then v=$(mysql -h${DATABASE_HOST} -P${DATABASE_PORT} -u${DATABASE_USERNAME} --password=${DATABASE_PASSWORD} -D ${DATABASE_DATABASE} -e "select 1" 2>&1)
echo "Error: Can't connect to database...${v}" if [ "$(echo '${v}' | grep 'ERROR' | wc -l)" -eq 1 ]; then
exit 1 echo "Error: Can't connect to database...${v}"
exit 1
fi
else
v=$(sudo -u postgres PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "select 1")
if [ "$(echo '${v}' | grep 'FATAL' | wc -l)" -eq 1 ]; then
echo "Error: Can't connect to database...${v}"
exit 1
fi
fi fi
echo "import sql data" echo "import sql data"
@ -123,7 +131,7 @@ LOGFILE=/var/log/nginx/access.log
case "$1" in case "$1" in
(all) (all)
initZK initZK
initPostgreSQL initDatabase
initMasterServer initMasterServer
initWorkerServer initWorkerServer
initApiServer initApiServer
@ -134,25 +142,25 @@ case "$1" in
;; ;;
(master-server) (master-server)
initZK initZK
initPostgreSQL initDatabase
initMasterServer initMasterServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-master.log LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-master.log
;; ;;
(worker-server) (worker-server)
initZK initZK
initPostgreSQL initDatabase
initWorkerServer initWorkerServer
initLoggerServer initLoggerServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log
;; ;;
(api-server) (api-server)
initZK initZK
initPostgreSQL initDatabase
initApiServer initApiServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log
;; ;;
(alert-server) (alert-server)
initPostgreSQL initDatabase
initAlertServer initAlertServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-alert.log LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-alert.log
;; ;;

40
docker/docker-swarm/docker-compose.yml

@ -56,11 +56,11 @@ services:
- 12345:12345 - 12345:12345
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
POSTGRESQL_HOST: dolphinscheduler-postgresql DATABASE_HOST: dolphinscheduler-postgresql
POSTGRESQL_PORT: 5432 DATABASE_PORT: 5432
POSTGRESQL_USERNAME: root DATABASE_USERNAME: root
POSTGRESQL_PASSWORD: root DATABASE_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck: healthcheck:
test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
@ -119,11 +119,11 @@ services:
ENTERPRISE_WECHAT_SECRET: "" ENTERPRISE_WECHAT_SECRET: ""
ENTERPRISE_WECHAT_AGENT_ID: "" ENTERPRISE_WECHAT_AGENT_ID: ""
ENTERPRISE_WECHAT_USERS: "" ENTERPRISE_WECHAT_USERS: ""
POSTGRESQL_HOST: dolphinscheduler-postgresql DATABASE_HOST: dolphinscheduler-postgresql
POSTGRESQL_PORT: 5432 DATABASE_PORT: 5432
POSTGRESQL_USERNAME: root DATABASE_USERNAME: root
POSTGRESQL_PASSWORD: root DATABASE_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler DATABASE_DATABASE: dolphinscheduler
healthcheck: healthcheck:
test: ["CMD", "/root/checkpoint.sh", "AlertServer"] test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
interval: 30s interval: 30s
@ -152,11 +152,11 @@ services:
MASTER_TASK_COMMIT_INTERVAL: "1000" MASTER_TASK_COMMIT_INTERVAL: "1000"
MASTER_MAX_CPULOAD_AVG: "100" MASTER_MAX_CPULOAD_AVG: "100"
MASTER_RESERVED_MEMORY: "0.1" MASTER_RESERVED_MEMORY: "0.1"
POSTGRESQL_HOST: dolphinscheduler-postgresql DATABASE_HOST: dolphinscheduler-postgresql
POSTGRESQL_PORT: 5432 DATABASE_PORT: 5432
POSTGRESQL_USERNAME: root DATABASE_USERNAME: root
POSTGRESQL_PASSWORD: root DATABASE_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck: healthcheck:
test: ["CMD", "/root/checkpoint.sh", "MasterServer"] test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
@ -188,11 +188,11 @@ services:
WORKER_RESERVED_MEMORY: "0.1" WORKER_RESERVED_MEMORY: "0.1"
WORKER_GROUP: "default" WORKER_GROUP: "default"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
POSTGRESQL_HOST: dolphinscheduler-postgresql DATABASE_HOST: dolphinscheduler-postgresql
POSTGRESQL_PORT: 5432 DATABASE_PORT: 5432
POSTGRESQL_USERNAME: root DATABASE_USERNAME: root
POSTGRESQL_PASSWORD: root DATABASE_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck: healthcheck:
test: ["CMD", "/root/checkpoint.sh", "WorkerServer"] test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]

52
docker/docker-swarm/docker-stack.yml

@ -20,13 +20,13 @@ services:
dolphinscheduler-postgresql: dolphinscheduler-postgresql:
image: bitnami/postgresql:latest image: bitnami/postgresql:latest
ports:
- 5432:5432
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
POSTGRESQL_USERNAME: root POSTGRESQL_USERNAME: root
POSTGRESQL_PASSWORD: root POSTGRESQL_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler POSTGRESQL_DATABASE: dolphinscheduler
ports:
- 5432:5432
volumes: volumes:
- dolphinscheduler-postgresql:/bitnami/postgresql - dolphinscheduler-postgresql:/bitnami/postgresql
networks: networks:
@ -37,12 +37,12 @@ services:
dolphinscheduler-zookeeper: dolphinscheduler-zookeeper:
image: bitnami/zookeeper:latest image: bitnami/zookeeper:latest
ports:
- 2181:2181
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
ALLOW_ANONYMOUS_LOGIN: "yes" ALLOW_ANONYMOUS_LOGIN: "yes"
ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
ports:
- 2181:2181
volumes: volumes:
- dolphinscheduler-zookeeper:/bitnami/zookeeper - dolphinscheduler-zookeeper:/bitnami/zookeeper
networks: networks:
@ -54,16 +54,16 @@ services:
dolphinscheduler-api: dolphinscheduler-api:
image: apache/dolphinscheduler:latest image: apache/dolphinscheduler:latest
command: ["api-server"] command: ["api-server"]
ports:
- 12345:12345
environment: environment:
TZ: Asia/Shanghai TZ: Asia/Shanghai
POSTGRESQL_HOST: dolphinscheduler-postgresql DATABASE_HOST: dolphinscheduler-postgresql
POSTGRESQL_PORT: 5432 DATABASE_PORT: 5432
POSTGRESQL_USERNAME: root DATABASE_USERNAME: root
POSTGRESQL_PASSWORD: root DATABASE_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
ports:
- 12345:12345
healthcheck: healthcheck:
test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
interval: 30 interval: 30
@ -120,11 +120,11 @@ services:
ENTERPRISE_WECHAT_SECRET: "" ENTERPRISE_WECHAT_SECRET: ""
ENTERPRISE_WECHAT_AGENT_ID: "" ENTERPRISE_WECHAT_AGENT_ID: ""
ENTERPRISE_WECHAT_USERS: "" ENTERPRISE_WECHAT_USERS: ""
POSTGRESQL_HOST: dolphinscheduler-postgresql DATABASE_HOST: dolphinscheduler-postgresql
POSTGRESQL_PORT: 5432 DATABASE_PORT: 5432
POSTGRESQL_USERNAME: root DATABASE_USERNAME: root
POSTGRESQL_PASSWORD: root DATABASE_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler DATABASE_DATABASE: dolphinscheduler
healthcheck: healthcheck:
test: ["CMD", "/root/checkpoint.sh", "AlertServer"] test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
interval: 30 interval: 30
@ -153,11 +153,11 @@ services:
MASTER_TASK_COMMIT_INTERVAL: "1000" MASTER_TASK_COMMIT_INTERVAL: "1000"
MASTER_MAX_CPULOAD_AVG: "100" MASTER_MAX_CPULOAD_AVG: "100"
MASTER_RESERVED_MEMORY: "0.1" MASTER_RESERVED_MEMORY: "0.1"
POSTGRESQL_HOST: dolphinscheduler-postgresql DATABASE_HOST: dolphinscheduler-postgresql
POSTGRESQL_PORT: 5432 DATABASE_PORT: 5432
POSTGRESQL_USERNAME: root DATABASE_USERNAME: root
POSTGRESQL_PASSWORD: root DATABASE_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck: healthcheck:
test: ["CMD", "/root/checkpoint.sh", "MasterServer"] test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
@ -188,11 +188,11 @@ services:
WORKER_RESERVED_MEMORY: "0.1" WORKER_RESERVED_MEMORY: "0.1"
WORKER_GROUP: "default" WORKER_GROUP: "default"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
POSTGRESQL_HOST: dolphinscheduler-postgresql DATABASE_HOST: dolphinscheduler-postgresql
POSTGRESQL_PORT: 5432 DATABASE_PORT: 5432
POSTGRESQL_USERNAME: root DATABASE_USERNAME: root
POSTGRESQL_PASSWORD: root DATABASE_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck: healthcheck:
test: ["CMD", "/root/checkpoint.sh", "WorkerServer"] test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]

2
docker/kubernetes/dolphinscheduler/Chart.yaml

@ -49,4 +49,4 @@ dependencies:
- name: zookeeper - name: zookeeper
version: 5.x.x version: 5.x.x
repository: https://charts.bitnami.com/bitnami repository: https://charts.bitnami.com/bitnami
condition: redis.enabled condition: zookeeper.enabled

40
docker/kubernetes/dolphinscheduler/README.md

@ -46,8 +46,8 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `image.repository` | Docker image repository for the Dolphins Scheduler | `dolphinscheduler` | | `image.repository` | Docker image repository for the Dolphins Scheduler | `dolphinscheduler` |
| `image.tag` | Docker image version for the Dolphins Scheduler | `1.2.1` | | `image.tag` | Docker image version for the Dolphins Scheduler | `1.2.1` |
| `image.imagePullPolicy` | Image pull policy. One of Always, Never, IfNotPresent | `IfNotPresent` | | `image.imagePullPolicy` | Image pull policy. One of Always, Never, IfNotPresent | `IfNotPresent` |
| `imagePullSecrets` | ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images | `[]` | | `image.pullSecres` | PullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images | `[]` |
| | | | | | | |
| `postgresql.enabled` | If not exists external PostgreSQL, by default, the Dolphins Scheduler will use a internal PostgreSQL | `true` | | `postgresql.enabled` | If not exists external PostgreSQL, by default, the Dolphins Scheduler will use a internal PostgreSQL | `true` |
| `postgresql.postgresqlUsername` | The username for internal PostgreSQL | `root` | | `postgresql.postgresqlUsername` | The username for internal PostgreSQL | `root` |
| `postgresql.postgresqlPassword` | The password for internal PostgreSQL | `root` | | `postgresql.postgresqlPassword` | The password for internal PostgreSQL | `root` |
@ -55,12 +55,15 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `postgresql.persistence.enabled` | Set `postgresql.persistence.enabled` to `true` to mount a new volume for internal PostgreSQL | `false` | | `postgresql.persistence.enabled` | Set `postgresql.persistence.enabled` to `true` to mount a new volume for internal PostgreSQL | `false` |
| `postgresql.persistence.size` | `PersistentVolumeClaim` Size | `20Gi` | | `postgresql.persistence.size` | `PersistentVolumeClaim` Size | `20Gi` |
| `postgresql.persistence.storageClass` | PostgreSQL data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | | `postgresql.persistence.storageClass` | PostgreSQL data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `externalDatabase.type` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database type will use it. | `postgresql` |
| `externalDatabase.driver` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database driver will use it. | `org.postgresql.Driver` |
| `externalDatabase.host` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database host will use it. | `localhost` | | `externalDatabase.host` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database host will use it. | `localhost` |
| `externalDatabase.port` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database port will use it. | `5432` | | `externalDatabase.port` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database port will use it. | `5432` |
| `externalDatabase.username` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database username will use it. | `root` | | `externalDatabase.username` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database username will use it. | `root` |
| `externalDatabase.password` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database password will use it. | `root` | | `externalDatabase.password` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database password will use it. | `root` |
| `externalDatabase.database` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database database will use it. | `dolphinscheduler` | | `externalDatabase.database` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database database will use it. | `dolphinscheduler` |
| | | | | `externalDatabase.params` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database params will use it. | `characterEncoding=utf8` |
| | | |
| `zookeeper.enabled` | If not exists external Zookeeper, by default, the Dolphin Scheduler will use a internal Zookeeper | `true` | | `zookeeper.enabled` | If not exists external Zookeeper, by default, the Dolphin Scheduler will use a internal Zookeeper | `true` |
| `zookeeper.taskQueue` | Specify task queue for `master` and `worker` | `zookeeper` | | `zookeeper.taskQueue` | Specify task queue for `master` and `worker` | `zookeeper` |
| `zookeeper.persistence.enabled` | Set `zookeeper.persistence.enabled` to `true` to mount a new volume for internal Zookeeper | `false` | | `zookeeper.persistence.enabled` | Set `zookeeper.persistence.enabled` to `true` to mount a new volume for internal Zookeeper | `false` |
@ -68,12 +71,25 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `zookeeper.persistence.storageClass` | Zookeeper data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | | `zookeeper.persistence.storageClass` | Zookeeper data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `externalZookeeper.taskQueue` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify task queue for `master` and `worker` | `zookeeper` | | `externalZookeeper.taskQueue` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify task queue for `master` and `worker` | `zookeeper` |
| `externalZookeeper.zookeeperQuorum` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify Zookeeper quorum | `127.0.0.1:2181` | | `externalZookeeper.zookeeperQuorum` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify Zookeeper quorum | `127.0.0.1:2181` |
| | | | | `externalZookeeper.zookeeperRoot` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify Zookeeper root path for `master` and `worker` | `dolphinscheduler` |
| | | |
| `common.configmap.DOLPHINSCHEDULER_ENV_PATH` | Extra env file path. | `/tmp/dolphinscheduler/env` |
| `common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH` | File uploaded path of DS. | `/tmp/dolphinscheduler/files` |
| `common.configmap.RESOURCE_STORAGE_TYPE` | Resource Storate type, support type are: S3、HDFS、NONE. | `NONE` |
| `common.configmap.RESOURCE_UPLOAD_PATH` | The base path of resource. | `/ds` |
| `common.configmap.FS_DEFAULT_FS` | The default fs of resource, for s3 is the `s3a` prefix and bucket name. | `s3a://xxxx` |
| `common.configmap.FS_S3A_ENDPOINT` | If the resource type is `S3`, you should fill this filed, it's the endpoint of s3. | `s3.xxx.amazonaws.com` |
| `common.configmap.FS_S3A_ACCESS_KEY` | The access key for your s3 bucket. | `xxxxxxx` |
| `common.configmap.FS_S3A_SECRET_KEY` | The secret key for your s3 bucket. | `xxxxxxx` |
| `master.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` | | `master.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` |
| | | |
| `master.replicas` | Replicas is the desired number of replicas of the given Template | `3` | | `master.replicas` | Replicas is the desired number of replicas of the given Template | `3` |
| `master.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | | `master.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
| `master.tolerations` | If specified, the pod's tolerations | `{}` | | `master.tolerations` | If specified, the pod's tolerations | `{}` |
| `master.affinity` | If specified, the pod's scheduling constraints | `{}` | | `master.affinity` | If specified, the pod's scheduling constraints | `{}` |
| `master.jvmOptions` | The JVM options for master server. | `""` |
| `master.resources` | The `resource` limit and request config for master server. | `{}` |
| `master.annotations` | The `annotations` for master server. | `{}` |
| `master.configmap.MASTER_EXEC_THREADS` | Master execute thread num | `100` | | `master.configmap.MASTER_EXEC_THREADS` | Master execute thread num | `100` |
| `master.configmap.MASTER_EXEC_TASK_NUM` | Master execute task number in parallel | `20` | | `master.configmap.MASTER_EXEC_TASK_NUM` | Master execute task number in parallel | `20` |
| `master.configmap.MASTER_HEARTBEAT_INTERVAL` | Master heartbeat interval | `10` | | `master.configmap.MASTER_HEARTBEAT_INTERVAL` | Master heartbeat interval | `10` |
@ -97,12 +113,15 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `master.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | | `master.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` |
| `master.persistentVolumeClaim.storageClassName` | `Master` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | | `master.persistentVolumeClaim.storageClassName` | `Master` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `master.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` | | `master.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` |
| | | | | | | |
| `worker.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` | | `worker.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` |
| `worker.replicas` | Replicas is the desired number of replicas of the given Template | `3` | | `worker.replicas` | Replicas is the desired number of replicas of the given Template | `3` |
| `worker.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | | `worker.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
| `worker.tolerations` | If specified, the pod's tolerations | `{}` | | `worker.tolerations` | If specified, the pod's tolerations | `{}` |
| `worker.affinity` | If specified, the pod's scheduling constraints | `{}` | | `worker.affinity` | If specified, the pod's scheduling constraints | `{}` |
| `worker.jvmOptions` | The JVM options for worker server. | `""` |
| `worker.resources` | The `resource` limit and request config for worker server. | `{}` |
| `worker.annotations` | The `annotations` for worker server. | `{}` |
| `worker.configmap.WORKER_EXEC_THREADS` | Worker execute thread num | `100` | | `worker.configmap.WORKER_EXEC_THREADS` | Worker execute thread num | `100` |
| `worker.configmap.WORKER_HEARTBEAT_INTERVAL` | Worker heartbeat interval | `10` | | `worker.configmap.WORKER_HEARTBEAT_INTERVAL` | Worker heartbeat interval | `10` |
| `worker.configmap.WORKER_FETCH_TASK_NUM` | Submit the number of tasks at a time | `3` | | `worker.configmap.WORKER_FETCH_TASK_NUM` | Submit the number of tasks at a time | `3` |
@ -131,7 +150,7 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `worker.persistentVolumeClaim.logsPersistentVolume.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | | `worker.persistentVolumeClaim.logsPersistentVolume.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` |
| `worker.persistentVolumeClaim.logsPersistentVolume.storageClassName` | `Worker` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | | `worker.persistentVolumeClaim.logsPersistentVolume.storageClassName` | `Worker` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `worker.persistentVolumeClaim.logsPersistentVolume.storage` | `PersistentVolumeClaim` Size | `20Gi` | | `worker.persistentVolumeClaim.logsPersistentVolume.storage` | `PersistentVolumeClaim` Size | `20Gi` |
| | | | | | | |
| `alert.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` | | `alert.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` |
| `alert.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` | | `alert.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` |
| `alert.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` | | `alert.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` |
@ -139,6 +158,10 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `alert.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | | `alert.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
| `alert.tolerations` | If specified, the pod's tolerations | `{}` | | `alert.tolerations` | If specified, the pod's tolerations | `{}` |
| `alert.affinity` | If specified, the pod's scheduling constraints | `{}` | | `alert.affinity` | If specified, the pod's scheduling constraints | `{}` |
| `alert.jvmOptions` | The JVM options for alert server. | `""` |
| `alert.resources` | The `resource` limit and request config for alert server. | `{}` |
| `alert.annotations` | The `annotations` for alert server. | `{}` |
| `alert.configmap.ALERT_PLUGIN_DIR` | Alert plugin path. | `/opt/dolphinscheduler/alert/plugin` |
| `alert.configmap.XLS_FILE_PATH` | XLS file path | `/tmp/xls` | | `alert.configmap.XLS_FILE_PATH` | XLS file path | `/tmp/xls` |
| `alert.configmap.MAIL_SERVER_HOST` | Mail `SERVER HOST ` | `nil` | | `alert.configmap.MAIL_SERVER_HOST` | Mail `SERVER HOST ` | `nil` |
| `alert.configmap.MAIL_SERVER_PORT` | Mail `SERVER PORT` | `nil` | | `alert.configmap.MAIL_SERVER_PORT` | Mail `SERVER PORT` | `nil` |
@ -177,6 +200,9 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `api.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | | `api.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
| `api.tolerations` | If specified, the pod's tolerations | `{}` | | `api.tolerations` | If specified, the pod's tolerations | `{}` |
| `api.affinity` | If specified, the pod's scheduling constraints | `{}` | | `api.affinity` | If specified, the pod's scheduling constraints | `{}` |
| `api.jvmOptions` | The JVM options for api server. | `""` |
| `api.resources` | The `resource` limit and request config for api server. | `{}` |
| `api.annotations` | The `annotations` for api server. | `{}` |
| `api.livenessProbe.enabled` | Turn on and off liveness probe | `true` | | `api.livenessProbe.enabled` | Turn on and off liveness probe | `true` |
| `api.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | | `api.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` |
| `api.livenessProbe.periodSeconds` | How often to perform the probe | `30` | | `api.livenessProbe.periodSeconds` | How often to perform the probe | `30` |
@ -201,6 +227,8 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `frontend.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | | `frontend.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
| `frontend.tolerations` | If specified, the pod's tolerations | `{}` | | `frontend.tolerations` | If specified, the pod's tolerations | `{}` |
| `frontend.affinity` | If specified, the pod's scheduling constraints | `{}` | | `frontend.affinity` | If specified, the pod's scheduling constraints | `{}` |
| `frontend.resources` | The `resource` limit and request config for frontend server. | `{}` |
| `frontend.annotations` | The `annotations` for frontend server. | `{}` |
| `frontend.livenessProbe.enabled` | Turn on and off liveness probe | `true` | | `frontend.livenessProbe.enabled` | Turn on and off liveness probe | `true` |
| `frontend.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | | `frontend.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` |
| `frontend.livenessProbe.periodSeconds` | How often to perform the probe | `30` | | `frontend.livenessProbe.periodSeconds` | How often to perform the probe | `30` |

25
docker/kubernetes/dolphinscheduler/requirements.yaml

@ -1,25 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
dependencies:
- name: postgresql
version: 8.x.x
repository: https://charts.bitnami.com/bitnami
condition: postgresql.enabled
- name: zookeeper
version: 5.x.x
repository: https://charts.bitnami.com/bitnami
condition: redis.enabled

13
docker/kubernetes/dolphinscheduler/templates/NOTES.txt

@ -29,16 +29,3 @@
kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-frontend 8888:8888 kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-frontend 8888:8888
{{- end }} {{- end }}
2. Get the Dolphinscheduler URL by running:
{{- if .Values.ingress.enabled }}
export HOSTNAME=$(kubectl get ingress --namespace {{ .Release.Namespace }} {{ template "dolphinscheduler.fullname" . }} -o jsonpath='{.spec.rules[0].host}')
echo "Dolphinscheduler URL: http://$HOSTNAME/"
{{- else }}
kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-frontend 8888:8888
{{- end }}

8
docker/kubernetes/dolphinscheduler/templates/_helpers.tpl

@ -96,6 +96,14 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
{{- printf "%s/%s:%s" (include "dolphinscheduler.image.registry" .) .Values.image.repository .Values.image.tag -}} {{- printf "%s/%s:%s" (include "dolphinscheduler.image.registry" .) .Values.image.repository .Values.image.tag -}}
{{- end -}} {{- end -}}
{{/*
Create a default image pull secrects.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
*/}}
{{- define "dolphinscheduler.image.pullSecrets" -}}
{{- default nil .Values.image.pullSecrets -}}
{{- end -}}
{{/* {{/*
Create a default fully qualified postgresql name. Create a default fully qualified postgresql name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).

1
docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml

@ -24,6 +24,7 @@ metadata:
app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/managed-by: {{ .Release.Service }}
data: data:
ALERT_PLUGIN_DIR: {{ .Values.alert.configmap.ALERT_PLUGIN_DIR | quote }}
XLS_FILE_PATH: {{ .Values.alert.configmap.XLS_FILE_PATH | quote }} XLS_FILE_PATH: {{ .Values.alert.configmap.XLS_FILE_PATH | quote }}
MAIL_SERVER_HOST: {{ .Values.alert.configmap.MAIL_SERVER_HOST | quote }} MAIL_SERVER_HOST: {{ .Values.alert.configmap.MAIL_SERVER_HOST | quote }}
MAIL_SERVER_PORT: {{ .Values.alert.configmap.MAIL_SERVER_PORT | quote }} MAIL_SERVER_PORT: {{ .Values.alert.configmap.MAIL_SERVER_PORT | quote }}

35
docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml

@ -0,0 +1,35 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if .Values.common.configmap }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "dolphinscheduler.fullname" . }}-common
labels:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-common
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
data:
DOLPHINSCHEDULER_ENV_PATH: {{ .Values.common.configmap.DOLPHINSCHEDULER_ENV_PATH | quote }}
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ .Values.common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH | quote }}
RESOURCE_STORAGE_TYPE: {{ .Values.common.configmap.RESOURCE_STORAGE_TYPE | quote }}
RESOURCE_UPLOAD_PATH: {{ .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }}
FS_DEFAULT_FS: {{ .Values.common.configmap.FS_DEFAULT_FS | quote }}
FS_S3A_ENDPOINT: {{ .Values.common.configmap.FS_S3A_ENDPOINT | quote }}
FS_S3A_ACCESS_KEY: {{ .Values.common.configmap.FS_S3A_ACCESS_KEY | quote }}
FS_S3A_SECRET_KEY: {{ .Values.common.configmap.FS_S3A_SECRET_KEY | quote }}
{{- end }}

98
docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml

@ -43,6 +43,10 @@ spec:
app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: alert app.kubernetes.io/component: alert
{{- if .Values.alert.annotations }}
annotations:
{{- toYaml .Values.alert.annotations | nindent 8 }}
{{- end }}
spec: spec:
{{- if .Values.alert.affinity }} {{- if .Values.alert.affinity }}
affinity: {{- toYaml .Values.alert.affinity | nindent 8 }} affinity: {{- toYaml .Values.alert.affinity | nindent 8 }}
@ -54,34 +58,38 @@ spec:
tolerations: {{- toYaml . | nindent 8 }} tolerations: {{- toYaml . | nindent 8 }}
{{- end }} {{- end }}
initContainers: initContainers:
- name: init-postgresql - name: init-database
image: busybox:1.31.0 image: busybox:1.31.0
command: command:
- /bin/sh - /bin/sh
- -ec - -ec
- | - |
while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
counter=$((counter+1)) counter=$((counter+1))
if [ $counter == 5 ]; then if [ $counter == 5 ]; then
echo "Error: Couldn't connect to postgresql." echo "Error: Couldn't connect to database."
exit 1 exit 1
fi fi
echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter." echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
sleep 60 sleep 60
done done
env: env:
- name: POSTGRESQL_HOST - name: DATABASE_HOST
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }} value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.host | quote }} value: {{ .Values.externalDatabase.host | quote }}
{{- end }} {{- end }}
- name: POSTGRESQL_PORT - name: DATABASE_PORT
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: "5432" value: "5432"
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.port }} value: {{ .Values.externalDatabase.port | quote }}
{{- end }} {{- end }}
{{- if .Values.image.pullSecrets }}
imagePullSecrets:
- name: {{ include "dolphinscheduler.image.pullSecrets" . }}
{{- end }}
containers: containers:
- name: {{ include "dolphinscheduler.fullname" . }}-alert - name: {{ include "dolphinscheduler.fullname" . }}-alert
image: {{ include "dolphinscheduler.image.repository" . | quote }} image: {{ include "dolphinscheduler.image.repository" . | quote }}
@ -89,8 +97,15 @@ spec:
- "alert-server" - "alert-server"
imagePullPolicy: {{ .Values.image.pullPolicy }} imagePullPolicy: {{ .Values.image.pullPolicy }}
env: env:
- name: DOLPHINSCHEDULER_OPTS
value: {{ default "" .Values.alert.jvmOptions }}
- name: TZ - name: TZ
value: {{ .Values.timezone }} value: {{ .Values.timezone }}
- name: ALERT_PLUGIN_DIR
valueFrom:
configMapKeyRef:
key: ALERT_PLUGIN_DIR
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: XLS_FILE_PATH - name: XLS_FILE_PATH
valueFrom: valueFrom:
configMapKeyRef: configMapKeyRef:
@ -161,25 +176,37 @@ spec:
configMapKeyRef: configMapKeyRef:
key: ENTERPRISE_WECHAT_USERS key: ENTERPRISE_WECHAT_USERS
name: {{ include "dolphinscheduler.fullname" . }}-alert name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: POSTGRESQL_HOST - name: DATABASE_TYPE
{{- if .Values.postgresql.enabled }}
value: "postgresql"
{{- else }}
value: {{ .Values.externalDatabase.type | quote }}
{{- end }}
- name: DATABASE_DRIVER
{{- if .Values.postgresql.enabled }}
value: "org.postgresql.Driver"
{{- else }}
value: {{ .Values.externalDatabase.driver | quote }}
{{- end }}
- name: DATABASE_HOST
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }} value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.host | quote }} value: {{ .Values.externalDatabase.host | quote }}
{{- end }} {{- end }}
- name: POSTGRESQL_PORT - name: DATABASE_PORT
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: "5432" value: "5432"
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.port }} value: {{ .Values.externalDatabase.port | quote }}
{{- end }} {{- end }}
- name: POSTGRESQL_USERNAME - name: DATABASE_USERNAME
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlUsername }} value: {{ .Values.postgresql.postgresqlUsername }}
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.username | quote }} value: {{ .Values.externalDatabase.username | quote }}
{{- end }} {{- end }}
- name: POSTGRESQL_PASSWORD - name: DATABASE_PASSWORD
valueFrom: valueFrom:
secretKeyRef: secretKeyRef:
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
@ -189,12 +216,57 @@ spec:
name: {{ printf "%s-%s" .Release.Name "externaldb" }} name: {{ printf "%s-%s" .Release.Name "externaldb" }}
key: db-password key: db-password
{{- end }} {{- end }}
- name: POSTGRESQL_DATABASE - name: DATABASE_DATABASE
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlDatabase }} value: {{ .Values.postgresql.postgresqlDatabase }}
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.database | quote }} value: {{ .Values.externalDatabase.database | quote }}
{{- end }} {{- end }}
- name: DATABASE_PARAMS
{{- if .Values.postgresql.enabled }}
value: "characterEncoding=utf8"
{{- else }}
value: {{ .Values.externalDatabase.params | quote }}
{{- end }}
- name: RESOURCE_STORAGE_TYPE
valueFrom:
configMapKeyRef:
key: RESOURCE_STORAGE_TYPE
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: RESOURCE_UPLOAD_PATH
valueFrom:
configMapKeyRef:
key: RESOURCE_UPLOAD_PATH
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_DEFAULT_FS
valueFrom:
configMapKeyRef:
key: FS_DEFAULT_FS
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_ENDPOINT
valueFrom:
configMapKeyRef:
key: FS_S3A_ENDPOINT
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_ACCESS_KEY
valueFrom:
configMapKeyRef:
key: FS_S3A_ACCESS_KEY
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_SECRET_KEY
valueFrom:
configMapKeyRef:
key: FS_S3A_SECRET_KEY
name: {{ include "dolphinscheduler.fullname" . }}-common
{{- if .Values.alert.resources }}
resources:
limits:
memory: {{ .Values.alert.resources.limits.memory }}
cpu: {{ .Values.alert.resources.limits.cpu }}
requests:
memory: {{ .Values.alert.resources.requests.memory }}
cpu: {{ .Values.alert.resources.requests.cpu }}
{{- end }}
{{- if .Values.alert.livenessProbe.enabled }} {{- if .Values.alert.livenessProbe.enabled }}
livenessProbe: livenessProbe:
exec: exec:

93
docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml

@ -43,6 +43,10 @@ spec:
app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: api app.kubernetes.io/component: api
{{- if .Values.alert.annotations }}
annotations:
{{- toYaml .Values.alert.annotations | nindent 8 }}
{{- end }}
spec: spec:
{{- if .Values.api.affinity }} {{- if .Values.api.affinity }}
affinity: {{- toYaml .Values.api.affinity | nindent 8 }} affinity: {{- toYaml .Values.api.affinity | nindent 8 }}
@ -54,34 +58,38 @@ spec:
tolerations: {{- toYaml . | nindent 8 }} tolerations: {{- toYaml . | nindent 8 }}
{{- end }} {{- end }}
initContainers: initContainers:
- name: init-postgresql - name: init-database
image: busybox:1.31.0 image: busybox:1.31.0
command: command:
- /bin/sh - /bin/sh
- -ec - -ec
- | - |
while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
counter=$((counter+1)) counter=$((counter+1))
if [ $counter == 5 ]; then if [ $counter == 5 ]; then
echo "Error: Couldn't connect to postgresql." echo "Error: Couldn't connect to database."
exit 1 exit 1
fi fi
echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter." echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
sleep 60 sleep 60
done done
env: env:
- name: POSTGRESQL_HOST - name: DATABASE_HOST
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }} value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.host | quote }} value: {{ .Values.externalDatabase.host | quote }}
{{- end }} {{- end }}
- name: POSTGRESQL_PORT - name: DATABASE_PORT
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: "5432" value: "5432"
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.port }} value: {{ .Values.externalDatabase.port | quote }}
{{- end }} {{- end }}
{{- if .Values.image.pullSecrets }}
imagePullSecrets:
- name: {{ include "dolphinscheduler.image.pullSecrets" . }}
{{- end }}
containers: containers:
- name: {{ include "dolphinscheduler.fullname" . }}-api - name: {{ include "dolphinscheduler.fullname" . }}-api
image: {{ include "dolphinscheduler.image.repository" . | quote }} image: {{ include "dolphinscheduler.image.repository" . | quote }}
@ -92,27 +100,41 @@ spec:
name: tcp-port name: tcp-port
imagePullPolicy: {{ .Values.image.pullPolicy }} imagePullPolicy: {{ .Values.image.pullPolicy }}
env: env:
- name: DOLPHINSCHEDULER_OPTS
value: {{ default "" .Values.api.jvmOptions }}
- name: TZ - name: TZ
value: {{ .Values.timezone }} value: {{ .Values.timezone }}
- name: POSTGRESQL_HOST - name: DATABASE_TYPE
{{- if .Values.postgresql.enabled }}
value: "postgresql"
{{- else }}
value: {{ .Values.externalDatabase.type | quote }}
{{- end }}
- name: DATABASE_DRIVER
{{- if .Values.postgresql.enabled }}
value: "org.postgresql.Driver"
{{- else }}
value: {{ .Values.externalDatabase.driver | quote }}
{{- end }}
- name: DATABASE_HOST
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }} value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.host | quote }} value: {{ .Values.externalDatabase.host | quote }}
{{- end }} {{- end }}
- name: POSTGRESQL_PORT - name: DATABASE_PORT
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: "5432" value: "5432"
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.port }} value: {{ .Values.externalDatabase.port | quote }}
{{- end }} {{- end }}
- name: POSTGRESQL_USERNAME - name: DATABASE_USERNAME
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlUsername }} value: {{ .Values.postgresql.postgresqlUsername }}
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.username | quote }} value: {{ .Values.externalDatabase.username | quote }}
{{- end }} {{- end }}
- name: POSTGRESQL_PASSWORD - name: DATABASE_PASSWORD
valueFrom: valueFrom:
secretKeyRef: secretKeyRef:
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
@ -122,18 +144,63 @@ spec:
name: {{ printf "%s-%s" .Release.Name "externaldb" }} name: {{ printf "%s-%s" .Release.Name "externaldb" }}
key: db-password key: db-password
{{- end }} {{- end }}
- name: POSTGRESQL_DATABASE - name: DATABASE_DATABASE
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlDatabase }} value: {{ .Values.postgresql.postgresqlDatabase }}
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.database | quote }} value: {{ .Values.externalDatabase.database | quote }}
{{- end }} {{- end }}
- name: DATABASE_PARAMS
{{- if .Values.postgresql.enabled }}
value: "characterEncoding=utf8"
{{- else }}
value: {{ .Values.externalDatabase.params | quote }}
{{- end }}
- name: ZOOKEEPER_QUORUM - name: ZOOKEEPER_QUORUM
{{- if .Values.zookeeper.enabled }} {{- if .Values.zookeeper.enabled }}
value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}" value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}"
{{- else }} {{- else }}
value: {{ .Values.externalZookeeper.zookeeperQuorum }} value: {{ .Values.externalZookeeper.zookeeperQuorum }}
{{- end }} {{- end }}
- name: RESOURCE_STORAGE_TYPE
valueFrom:
configMapKeyRef:
key: RESOURCE_STORAGE_TYPE
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: RESOURCE_UPLOAD_PATH
valueFrom:
configMapKeyRef:
key: RESOURCE_UPLOAD_PATH
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_DEFAULT_FS
valueFrom:
configMapKeyRef:
key: FS_DEFAULT_FS
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_ENDPOINT
valueFrom:
configMapKeyRef:
key: FS_S3A_ENDPOINT
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_ACCESS_KEY
valueFrom:
configMapKeyRef:
key: FS_S3A_ACCESS_KEY
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_SECRET_KEY
valueFrom:
configMapKeyRef:
key: FS_S3A_SECRET_KEY
name: {{ include "dolphinscheduler.fullname" . }}-common
{{- if .Values.api.resources }}
resources:
limits:
memory: {{ .Values.api.resources.limits.memory | quote }}
cpu: {{ .Values.api.resources.limits.cpu | quote }}
requests:
memory: {{ .Values.api.resources.requests.memory | quote }}
cpu: {{ .Values.api.resources.requests.cpu | quote }}
{{- end }}
{{- if .Values.api.livenessProbe.enabled }} {{- if .Values.api.livenessProbe.enabled }}
livenessProbe: livenessProbe:
tcpSocket: tcpSocket:

17
docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml

@ -43,6 +43,10 @@ spec:
app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: frontend app.kubernetes.io/component: frontend
{{- if .Values.alert.annotations }}
annotations:
{{- toYaml .Values.alert.annotations | nindent 8 }}
{{- end }}
spec: spec:
{{- if .Values.frontend.affinity }} {{- if .Values.frontend.affinity }}
affinity: {{- toYaml .Values.frontend.affinity | nindent 8 }} affinity: {{- toYaml .Values.frontend.affinity | nindent 8 }}
@ -53,6 +57,10 @@ spec:
{{- if .Values.frontend.tolerations }} {{- if .Values.frontend.tolerations }}
tolerations: {{- toYaml . | nindent 8 }} tolerations: {{- toYaml . | nindent 8 }}
{{- end }} {{- end }}
{{- if .Values.image.pullSecrets }}
imagePullSecrets:
- name: {{ include "dolphinscheduler.image.pullSecrets" . }}
{{- end }}
containers: containers:
- name: {{ include "dolphinscheduler.fullname" . }}-frontend - name: {{ include "dolphinscheduler.fullname" . }}-frontend
image: {{ include "dolphinscheduler.image.repository" . | quote }} image: {{ include "dolphinscheduler.image.repository" . | quote }}
@ -69,6 +77,15 @@ spec:
value: '{{ include "dolphinscheduler.fullname" . }}-api' value: '{{ include "dolphinscheduler.fullname" . }}-api'
- name: FRONTEND_API_SERVER_PORT - name: FRONTEND_API_SERVER_PORT
value: "12345" value: "12345"
{{- if .Values.frontend.resources }}
resources:
limits:
memory: {{ .Values.frontend.resources.limits.memory | quote }}
cpu: {{ .Values.frontend.resources.limits.cpu | quote }}
requests:
memory: {{ .Values.frontend.resources.requests.memory | quote }}
cpu: {{ .Values.frontend.resources.requests.cpu | quote }}
{{- end }}
{{- if .Values.frontend.livenessProbe.enabled }} {{- if .Values.frontend.livenessProbe.enabled }}
livenessProbe: livenessProbe:
tcpSocket: tcpSocket:

93
docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml

@ -40,6 +40,10 @@ spec:
app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: master app.kubernetes.io/component: master
{{- if .Values.alert.annotations }}
annotations:
{{- toYaml .Values.alert.annotations | nindent 8 }}
{{- end }}
spec: spec:
{{- if .Values.master.affinity }} {{- if .Values.master.affinity }}
affinity: {{- toYaml .Values.master.affinity | nindent 8 }} affinity: {{- toYaml .Values.master.affinity | nindent 8 }}
@ -75,34 +79,38 @@ spec:
{{- else }} {{- else }}
value: {{ .Values.externalZookeeper.zookeeperQuorum }} value: {{ .Values.externalZookeeper.zookeeperQuorum }}
{{- end }} {{- end }}
- name: init-postgresql - name: init-database
image: busybox:1.31.0 image: busybox:1.31.0
command: command:
- /bin/sh - /bin/sh
- -ec - -ec
- | - |
while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
counter=$((counter+1)) counter=$((counter+1))
if [ $counter == 5 ]; then if [ $counter == 5 ]; then
echo "Error: Couldn't connect to postgresql." echo "Error: Couldn't connect to database."
exit 1 exit 1
fi fi
echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter." echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
sleep 60 sleep 60
done done
env: env:
- name: POSTGRESQL_HOST - name: DATABASE_HOST
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }} value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.host | quote }} value: {{ .Values.externalDatabase.host | quote }}
{{- end }} {{- end }}
- name: POSTGRESQL_PORT - name: DATABASE_PORT
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: "5432" value: "5432"
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.port }} value: {{ .Values.externalDatabase.port | quote }}
{{- end }} {{- end }}
{{- if .Values.image.pullSecrets }}
imagePullSecrets:
- name: {{ include "dolphinscheduler.image.pullSecrets" . }}
{{- end }}
containers: containers:
- name: {{ include "dolphinscheduler.fullname" . }}-master - name: {{ include "dolphinscheduler.fullname" . }}-master
image: {{ include "dolphinscheduler.image.repository" . | quote }} image: {{ include "dolphinscheduler.image.repository" . | quote }}
@ -113,6 +121,8 @@ spec:
name: "master-port" name: "master-port"
imagePullPolicy: {{ .Values.image.pullPolicy }} imagePullPolicy: {{ .Values.image.pullPolicy }}
env: env:
- name: DOLPHINSCHEDULER_OPTS
value: {{ default "" .Values.master.jvmOptions }}
- name: TZ - name: TZ
value: {{ .Values.timezone }} value: {{ .Values.timezone }}
- name: MASTER_EXEC_THREADS - name: MASTER_EXEC_THREADS
@ -160,25 +170,37 @@ spec:
configMapKeyRef: configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-master name: {{ include "dolphinscheduler.fullname" . }}-master
key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
- name: POSTGRESQL_HOST - name: DATABASE_TYPE
{{- if .Values.postgresql.enabled }}
value: "postgresql"
{{- else }}
value: {{ .Values.externalDatabase.type | quote }}
{{- end }}
- name: DATABASE_DRIVER
{{- if .Values.postgresql.enabled }}
value: "org.postgresql.Driver"
{{- else }}
value: {{ .Values.externalDatabase.driver | quote }}
{{- end }}
- name: DATABASE_HOST
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }} value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.host | quote }} value: {{ .Values.externalDatabase.host | quote }}
{{- end }} {{- end }}
- name: POSTGRESQL_PORT - name: DATABASE_PORT
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: "5432" value: "5432"
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.port }} value: {{ .Values.externalDatabase.port | quote }}
{{- end }} {{- end }}
- name: POSTGRESQL_USERNAME - name: DATABASE_USERNAME
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlUsername }} value: {{ .Values.postgresql.postgresqlUsername }}
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.username | quote }} value: {{ .Values.externalDatabase.username | quote }}
{{- end }} {{- end }}
- name: POSTGRESQL_PASSWORD - name: DATABASE_PASSWORD
valueFrom: valueFrom:
secretKeyRef: secretKeyRef:
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
@ -188,18 +210,63 @@ spec:
name: {{ printf "%s-%s" .Release.Name "externaldb" }} name: {{ printf "%s-%s" .Release.Name "externaldb" }}
key: db-password key: db-password
{{- end }} {{- end }}
- name: POSTGRESQL_DATABASE - name: DATABASE_DATABASE
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlDatabase }} value: {{ .Values.postgresql.postgresqlDatabase }}
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.database | quote }} value: {{ .Values.externalDatabase.database | quote }}
{{- end }} {{- end }}
- name: DATABASE_PARAMS
{{- if .Values.postgresql.enabled }}
value: "characterEncoding=utf8"
{{- else }}
value: {{ .Values.externalDatabase.params | quote }}
{{- end }}
- name: ZOOKEEPER_QUORUM - name: ZOOKEEPER_QUORUM
{{- if .Values.zookeeper.enabled }} {{- if .Values.zookeeper.enabled }}
value: {{ template "dolphinscheduler.zookeeper.quorum" . }} value: {{ template "dolphinscheduler.zookeeper.quorum" . }}
{{- else }} {{- else }}
value: {{ .Values.externalZookeeper.zookeeperQuorum }} value: {{ .Values.externalZookeeper.zookeeperQuorum }}
{{- end }} {{- end }}
- name: RESOURCE_STORAGE_TYPE
valueFrom:
configMapKeyRef:
key: RESOURCE_STORAGE_TYPE
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: RESOURCE_UPLOAD_PATH
valueFrom:
configMapKeyRef:
key: RESOURCE_UPLOAD_PATH
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_DEFAULT_FS
valueFrom:
configMapKeyRef:
key: FS_DEFAULT_FS
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_ENDPOINT
valueFrom:
configMapKeyRef:
key: FS_S3A_ENDPOINT
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_ACCESS_KEY
valueFrom:
configMapKeyRef:
key: FS_S3A_ACCESS_KEY
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_SECRET_KEY
valueFrom:
configMapKeyRef:
key: FS_S3A_SECRET_KEY
name: {{ include "dolphinscheduler.fullname" . }}-common
{{- if .Values.master.resources }}
resources:
limits:
memory: {{ .Values.master.resources.limits.memory | quote }}
cpu: {{ .Values.master.resources.limits.cpu | quote }}
requests:
memory: {{ .Values.master.resources.requests.memory | quote }}
cpu: {{ .Values.master.resources.requests.cpu | quote }}
{{- end }}
{{- if .Values.master.livenessProbe.enabled }} {{- if .Values.master.livenessProbe.enabled }}
livenessProbe: livenessProbe:
exec: exec:

163
docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml

@ -40,6 +40,10 @@ spec:
app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: worker app.kubernetes.io/component: worker
{{- if .Values.alert.annotations }}
annotations:
{{- toYaml .Values.alert.annotations | nindent 8 }}
{{- end }}
spec: spec:
{{- if .Values.worker.affinity }} {{- if .Values.worker.affinity }}
affinity: {{- toYaml .Values.worker.affinity | nindent 8 }} affinity: {{- toYaml .Values.worker.affinity | nindent 8 }}
@ -75,34 +79,38 @@ spec:
{{- else }} {{- else }}
value: {{ .Values.externalZookeeper.zookeeperQuorum }} value: {{ .Values.externalZookeeper.zookeeperQuorum }}
{{- end }} {{- end }}
- name: init-postgresql - name: init-database
image: busybox:1.31.0 image: busybox:1.31.0
command: command:
- /bin/sh - /bin/sh
- -ec - -ec
- | - |
while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
counter=$((counter+1)) counter=$((counter+1))
if [ $counter == 5 ]; then if [ $counter == 5 ]; then
echo "Error: Couldn't connect to postgresql." echo "Error: Couldn't connect to database."
exit 1 exit 1
fi fi
echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter." echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
sleep 60 sleep 60
done done
env: env:
- name: POSTGRESQL_HOST - name: DATABASE_HOST
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }} value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.host | quote }} value: {{ .Values.externalDatabase.host | quote }}
{{- end }} {{- end }}
- name: POSTGRESQL_PORT - name: DATABASE_PORT
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: "5432" value: "5432"
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.port }} value: {{ .Values.externalDatabase.port | quote }}
{{- end }} {{- end }}
{{- if .Values.image.pullSecrets }}
imagePullSecrets:
- name: {{ include "dolphinscheduler.image.pullSecrets" . }}
{{- end }}
containers: containers:
- name: {{ include "dolphinscheduler.fullname" . }}-worker - name: {{ include "dolphinscheduler.fullname" . }}-worker
image: {{ include "dolphinscheduler.image.repository" . | quote }} image: {{ include "dolphinscheduler.image.repository" . | quote }}
@ -115,6 +123,8 @@ spec:
name: "logs-port" name: "logs-port"
imagePullPolicy: {{ .Values.image.pullPolicy }} imagePullPolicy: {{ .Values.image.pullPolicy }}
env: env:
- name: DOLPHINSCHEDULER_OPTS
value: {{ default "" .Values.worker.jvmOptions }}
- name: TZ - name: TZ
value: {{ .Values.timezone }} value: {{ .Values.timezone }}
- name: WORKER_EXEC_THREADS - name: WORKER_EXEC_THREADS
@ -157,25 +167,37 @@ spec:
configMapKeyRef: configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-master name: {{ include "dolphinscheduler.fullname" . }}-master
key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
- name: POSTGRESQL_HOST - name: DATABASE_TYPE
{{- if .Values.postgresql.enabled }}
value: "postgresql"
{{- else }}
value: {{ .Values.externalDatabase.type | quote }}
{{- end }}
- name: DATABASE_DRIVER
{{- if .Values.postgresql.enabled }}
value: "org.postgresql.Driver"
{{- else }}
value: {{ .Values.externalDatabase.driver | quote }}
{{- end }}
- name: DATABASE_HOST
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }} value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.host | quote }} value: {{ .Values.externalDatabase.host | quote }}
{{- end }} {{- end }}
- name: POSTGRESQL_PORT - name: DATABASE_PORT
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: "5432" value: "5432"
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.port }} value: {{ .Values.externalDatabase.port | quote }}
{{- end }} {{- end }}
- name: POSTGRESQL_USERNAME - name: DATABASE_USERNAME
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlUsername }} value: {{ .Values.postgresql.postgresqlUsername }}
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.username | quote }} value: {{ .Values.externalDatabase.username | quote }}
{{- end }} {{- end }}
- name: POSTGRESQL_PASSWORD - name: DATABASE_PASSWORD
valueFrom: valueFrom:
secretKeyRef: secretKeyRef:
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
@ -185,18 +207,133 @@ spec:
name: {{ printf "%s-%s" .Release.Name "externaldb" }} name: {{ printf "%s-%s" .Release.Name "externaldb" }}
key: db-password key: db-password
{{- end }} {{- end }}
- name: POSTGRESQL_DATABASE - name: DATABASE_DATABASE
{{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlDatabase }} value: {{ .Values.postgresql.postgresqlDatabase }}
{{- else }} {{- else }}
value: {{ .Values.externalDatabase.database | quote }} value: {{ .Values.externalDatabase.database | quote }}
{{- end }} {{- end }}
- name: DATABASE_PARAMS
{{- if .Values.postgresql.enabled }}
value: "characterEncoding=utf8"
{{- else }}
value: {{ .Values.externalDatabase.params | quote }}
{{- end }}
- name: ZOOKEEPER_QUORUM - name: ZOOKEEPER_QUORUM
{{- if .Values.zookeeper.enabled }} {{- if .Values.zookeeper.enabled }}
value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}" value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}"
{{- else }} {{- else }}
value: {{ .Values.externalZookeeper.zookeeperQuorum }} value: {{ .Values.externalZookeeper.zookeeperQuorum }}
{{- end }} {{- end }}
- name: RESOURCE_STORAGE_TYPE
valueFrom:
configMapKeyRef:
key: RESOURCE_STORAGE_TYPE
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: RESOURCE_UPLOAD_PATH
valueFrom:
configMapKeyRef:
key: RESOURCE_UPLOAD_PATH
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_DEFAULT_FS
valueFrom:
configMapKeyRef:
key: FS_DEFAULT_FS
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_ENDPOINT
valueFrom:
configMapKeyRef:
key: FS_S3A_ENDPOINT
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_ACCESS_KEY
valueFrom:
configMapKeyRef:
key: FS_S3A_ACCESS_KEY
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: FS_S3A_SECRET_KEY
valueFrom:
configMapKeyRef:
key: FS_S3A_SECRET_KEY
name: {{ include "dolphinscheduler.fullname" . }}-common
- name: XLS_FILE_PATH
valueFrom:
configMapKeyRef:
key: XLS_FILE_PATH
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SERVER_HOST
valueFrom:
configMapKeyRef:
key: MAIL_SERVER_HOST
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SERVER_PORT
valueFrom:
configMapKeyRef:
key: MAIL_SERVER_PORT
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SENDER
valueFrom:
configMapKeyRef:
key: MAIL_SENDER
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_USER
valueFrom:
configMapKeyRef:
key: MAIL_USER
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_PASSWD
valueFrom:
configMapKeyRef:
key: MAIL_PASSWD
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SMTP_STARTTLS_ENABLE
valueFrom:
configMapKeyRef:
key: MAIL_SMTP_STARTTLS_ENABLE
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SMTP_SSL_ENABLE
valueFrom:
configMapKeyRef:
key: MAIL_SMTP_SSL_ENABLE
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: MAIL_SMTP_SSL_TRUST
valueFrom:
configMapKeyRef:
key: MAIL_SMTP_SSL_TRUST
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: ENTERPRISE_WECHAT_ENABLE
valueFrom:
configMapKeyRef:
key: ENTERPRISE_WECHAT_ENABLE
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: ENTERPRISE_WECHAT_CORP_ID
valueFrom:
configMapKeyRef:
key: ENTERPRISE_WECHAT_CORP_ID
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: ENTERPRISE_WECHAT_SECRET
valueFrom:
configMapKeyRef:
key: ENTERPRISE_WECHAT_SECRET
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: ENTERPRISE_WECHAT_AGENT_ID
valueFrom:
configMapKeyRef:
key: ENTERPRISE_WECHAT_AGENT_ID
name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: ENTERPRISE_WECHAT_USERS
valueFrom:
configMapKeyRef:
key: ENTERPRISE_WECHAT_USERS
name: {{ include "dolphinscheduler.fullname" . }}-alert
{{- if .Values.worker.resources }}
resources:
limits:
memory: {{ .Values.worker.resources.limits.memory | quote }}
cpu: {{ .Values.worker.resources.limits.cpu | quote }}
requests:
memory: {{ .Values.worker.resources.requests.memory | quote }}
cpu: {{ .Values.worker.resources.requests.cpu | quote }}
{{- end }}
{{- if .Values.worker.livenessProbe.enabled }} {{- if .Values.worker.livenessProbe.enabled }}
livenessProbe: livenessProbe:
exec: exec:

87
docker/kubernetes/dolphinscheduler/values.yaml

@ -29,10 +29,9 @@ image:
repository: "dolphinscheduler" repository: "dolphinscheduler"
tag: "latest" tag: "latest"
pullPolicy: "IfNotPresent" pullPolicy: "IfNotPresent"
pullSecrets: []
imagePullSecrets: [] # If not exists external database, by default, Dolphinscheduler's database will use it.
# If not exists external postgresql, by default, Dolphinscheduler's database will use it.
postgresql: postgresql:
enabled: true enabled: true
postgresqlUsername: "root" postgresqlUsername: "root"
@ -43,21 +42,24 @@ postgresql:
size: "20Gi" size: "20Gi"
storageClass: "-" storageClass: "-"
# If exists external postgresql, and set postgresql.enable value to false. # If exists external database, and set postgresql.enable value to false.
# If postgresql.enable is false, Dolphinscheduler's database will use it. # external database will be used, otherwise Dolphinscheduler's database will be used.
externalDatabase: externalDatabase:
type: "postgresql"
driver: "org.postgresql.Driver"
host: "localhost" host: "localhost"
port: "5432" port: "5432"
username: "root" username: "root"
password: "root" password: "root"
database: "dolphinscheduler" database: "dolphinscheduler"
# multi params should join with & char
params: "characterEncoding=utf8"
# If not exists external zookeeper, by default, Dolphinscheduler's zookeeper will use it. # If not exists external zookeeper, by default, Dolphinscheduler's zookeeper will use it.
zookeeper: zookeeper:
enabled: true enabled: true
taskQueue: "zookeeper" taskQueue: "zookeeper"
config: config: null
ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
service: service:
port: "2181" port: "2181"
persistence: persistence:
@ -70,6 +72,18 @@ zookeeper:
externalZookeeper: externalZookeeper:
taskQueue: "zookeeper" taskQueue: "zookeeper"
zookeeperQuorum: "127.0.0.1:2181" zookeeperQuorum: "127.0.0.1:2181"
zookeeperRoot: "/dolphinscheduler"
common:
configmap:
DOLPHINSCHEDULER_ENV_PATH: "/tmp/dolphinscheduler/env"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler/files"
RESOURCE_STORAGE_TYPE: "NONE"
RESOURCE_UPLOAD_PATH: "/ds"
FS_DEFAULT_FS: "s3a://xxxx"
FS_S3A_ENDPOINT: "s3.xxx.amazonaws.com"
FS_S3A_ACCESS_KEY: "xxxxxxx"
FS_S3A_SECRET_KEY: "xxxxxxx"
master: master:
podManagementPolicy: "Parallel" podManagementPolicy: "Parallel"
@ -85,6 +99,18 @@ master:
# If specified, the pod's scheduling constraints. # If specified, the pod's scheduling constraints.
# More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
affinity: {} affinity: {}
# The jvm options for java instance startup
jvmOptions: ""
resources: {}
# limits:
# memory: "18Gi"
# cpu: "4"
# requests:
# memory: "2Gi"
# cpu: "500m"
# You can use annotations to attach arbitrary non-identifying metadata to objects.
# Clients such as tools and libraries can retrieve this metadata.
annotations: {}
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
configmap: configmap:
@ -137,6 +163,18 @@ worker:
# If specified, the pod's scheduling constraints. # If specified, the pod's scheduling constraints.
# More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
affinity: {} affinity: {}
# The jvm options for java instance startup
jvmOptions: ""
resources: {}
# limits:
# memory: "18Gi"
# cpu: "4"
# requests:
# memory: "2Gi"
# cpu: "500m"
# You can use annotations to attach arbitrary non-identifying metadata to objects.
# Clients such as tools and libraries can retrieve this metadata.
annotations: {}
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
livenessProbe: livenessProbe:
@ -213,9 +251,22 @@ alert:
# If specified, the pod's scheduling constraints. # If specified, the pod's scheduling constraints.
# More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
affinity: {} affinity: {}
# The jvm options for java instance startup
jvmOptions: ""
resources: {}
# limits:
# memory: "4Gi"
# cpu: "1"
# requests:
# memory: "2Gi"
# cpu: "500m"
# You can use annotations to attach arbitrary non-identifying metadata to objects.
# Clients such as tools and libraries can retrieve this metadata.
annotations: {}
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
configmap: configmap:
ALERT_PLUGIN_DIR: "/opt/dolphinscheduler/alert/plugin"
XLS_FILE_PATH: "/tmp/xls" XLS_FILE_PATH: "/tmp/xls"
MAIL_SERVER_HOST: "" MAIL_SERVER_HOST: ""
MAIL_SERVER_PORT: "" MAIL_SERVER_PORT: ""
@ -275,6 +326,18 @@ api:
# If specified, the pod's scheduling constraints. # If specified, the pod's scheduling constraints.
# More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
affinity: {} affinity: {}
# The jvm options for java instance startup
jvmOptions: ""
resources: {}
# limits:
# memory: "4Gi"
# cpu: "2"
# requests:
# memory: "2Gi"
# cpu: "500m"
# You can use annotations to attach arbitrary non-identifying metadata to objects.
# Clients such as tools and libraries can retrieve this metadata.
annotations: {}
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
livenessProbe: livenessProbe:
@ -322,6 +385,16 @@ frontend:
# If specified, the pod's scheduling constraints. # If specified, the pod's scheduling constraints.
# More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
affinity: {} affinity: {}
resources: {}
# limits:
# memory: "256Mi"
# cpu: "1"
# requests:
# memory: "256Mi"
# cpu: "500m"
# You can use annotations to attach arbitrary non-identifying metadata to objects.
# Clients such as tools and libraries can retrieve this metadata.
annotations: {}
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
livenessProbe: livenessProbe:

30
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java

@ -48,8 +48,8 @@ public class EnterpriseWeChatUtils {
private static final String ENTERPRISE_WE_CHAT_TOKEN_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TOKEN_URL); private static final String ENTERPRISE_WE_CHAT_TOKEN_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TOKEN_URL);
private static final String ENTERPRISE_WE_CHAT_TOKEN_URL_REPLACE = ENTERPRISE_WE_CHAT_TOKEN_URL == null ? null : ENTERPRISE_WE_CHAT_TOKEN_URL private static final String ENTERPRISE_WE_CHAT_TOKEN_URL_REPLACE = ENTERPRISE_WE_CHAT_TOKEN_URL == null ? null : ENTERPRISE_WE_CHAT_TOKEN_URL
.replaceAll("\\$corpId", ENTERPRISE_WE_CHAT_CORP_ID) .replaceAll("\\{corpId\\}", ENTERPRISE_WE_CHAT_CORP_ID)
.replaceAll("\\$secret", ENTERPRISE_WE_CHAT_SECRET); .replaceAll("\\{secret\\}", ENTERPRISE_WE_CHAT_SECRET);
private static final String ENTERPRISE_WE_CHAT_PUSH_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_PUSH_URL); private static final String ENTERPRISE_WE_CHAT_PUSH_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_PUSH_URL);
@ -120,9 +120,9 @@ public class EnterpriseWeChatUtils {
* @return Enterprise WeChat send message * @return Enterprise WeChat send message
*/ */
public static String makeTeamSendMsg(String toParty, String agentId, String msg) { public static String makeTeamSendMsg(String toParty, String agentId, String msg) {
return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\$toParty", toParty) return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty\\}", toParty)
.replaceAll("\\$agentId", agentId) .replaceAll("\\{agentId\\}", agentId)
.replaceAll("\\$msg", msg); .replaceAll("\\{msg\\}", msg);
} }
/** /**
@ -135,9 +135,9 @@ public class EnterpriseWeChatUtils {
*/ */
public static String makeTeamSendMsg(Collection<String> toParty, String agentId, String msg) { public static String makeTeamSendMsg(Collection<String> toParty, String agentId, String msg) {
String listParty = FuncUtils.mkString(toParty, "|"); String listParty = FuncUtils.mkString(toParty, "|");
return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\$toParty", listParty) return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty\\}", listParty)
.replaceAll("\\$agentId", agentId) .replaceAll("\\{agentId\\}", agentId)
.replaceAll("\\$msg", msg); .replaceAll("\\{msg\\}", msg);
} }
/** /**
@ -149,9 +149,9 @@ public class EnterpriseWeChatUtils {
* @return Enterprise WeChat send message * @return Enterprise WeChat send message
*/ */
public static String makeUserSendMsg(String toUser, String agentId, String msg) { public static String makeUserSendMsg(String toUser, String agentId, String msg) {
return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\$toUser", toUser) return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\{toUser\\}", toUser)
.replaceAll("\\$agentId", agentId) .replaceAll("\\{agentId\\}", agentId)
.replaceAll("\\$msg", msg); .replaceAll("\\{msg\\}", msg);
} }
/** /**
@ -164,9 +164,9 @@ public class EnterpriseWeChatUtils {
*/ */
public static String makeUserSendMsg(Collection<String> toUser, String agentId, String msg) { public static String makeUserSendMsg(Collection<String> toUser, String agentId, String msg) {
String listUser = FuncUtils.mkString(toUser, "|"); String listUser = FuncUtils.mkString(toUser, "|");
return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\$toUser", listUser) return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\{toUser\\}", listUser)
.replaceAll("\\$agentId", agentId) .replaceAll("\\{agentId\\}", agentId)
.replaceAll("\\$msg", msg); .replaceAll("\\{msg\\}", msg);
} }
/** /**
@ -179,7 +179,7 @@ public class EnterpriseWeChatUtils {
* @throws IOException the IOException * @throws IOException the IOException
*/ */
public static String sendEnterpriseWeChat(String charset, String data, String token) throws IOException { public static String sendEnterpriseWeChat(String charset, String data, String token) throws IOException {
String enterpriseWeChatPushUrlReplace = ENTERPRISE_WE_CHAT_PUSH_URL.replaceAll("\\$token", token); String enterpriseWeChatPushUrlReplace = ENTERPRISE_WE_CHAT_PUSH_URL.replaceAll("\\{token\\}", token);
CloseableHttpClient httpClient = HttpClients.createDefault(); CloseableHttpClient httpClient = HttpClients.createDefault();
try { try {

8
dolphinscheduler-alert/src/main/resources/alert.properties

@ -41,10 +41,10 @@ enterprise.wechat.enable=false
#enterprise.wechat.secret=xxxxxxx #enterprise.wechat.secret=xxxxxxx
#enterprise.wechat.agent.id=xxxxxxx #enterprise.wechat.agent.id=xxxxxxx
#enterprise.wechat.users=xxxxxxx #enterprise.wechat.users=xxxxxxx
#enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret #enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid={corpId}&corpsecret={secret}
#enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token #enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={token}
#enterprise.wechat.team.send.msg={\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"} #enterprise.wechat.team.send.msg={\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"}
#enterprise.wechat.user.send.msg={\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}} #enterprise.wechat.user.send.msg={\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}}
plugin.dir=/Users/xx/your/path/to/plugin/dir plugin.dir=/Users/xx/your/path/to/plugin/dir

4
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java

@ -55,8 +55,8 @@ public class EnterpriseWeChatUtilsTest {
private static final String enterpriseWechatUsers="LiGang,journey"; private static final String enterpriseWechatUsers="LiGang,journey";
private static final String msg = "hello world"; private static final String msg = "hello world";
private static final String enterpriseWechatTeamSendMsg = "{\\\"toparty\\\":\\\"$toParty\\\",\\\"agentid\\\":\\\"$agentId\\\",\\\"msgtype\\\":\\\"text\\\",\\\"text\\\":{\\\"content\\\":\\\"$msg\\\"},\\\"safe\\\":\\\"0\\\"}"; private static final String enterpriseWechatTeamSendMsg = "{\\\"toparty\\\":\\\"{toParty}\\\",\\\"agentid\\\":\\\"{agentId}\\\",\\\"msgtype\\\":\\\"text\\\",\\\"text\\\":{\\\"content\\\":\\\"{msg}\\\"},\\\"safe\\\":\\\"0\\\"}";
private static final String enterpriseWechatUserSendMsg = "{\\\"touser\\\":\\\"$toUser\\\",\\\"agentid\\\":\\\"$agentId\\\",\\\"msgtype\\\":\\\"markdown\\\",\\\"markdown\\\":{\\\"content\\\":\\\"$msg\\\"}}"; private static final String enterpriseWechatUserSendMsg = "{\\\"touser\\\":\\\"{toUser}\\\",\\\"agentid\\\":\\\"{agentId}\\\",\\\"msgtype\\\":\\\"markdown\\\",\\\"markdown\\\":{\\\"content\\\":\\\"{msg}\\\"}}";
@Before @Before
public void init(){ public void init(){

2
dolphinscheduler-ui/package.json

@ -63,7 +63,7 @@
"html-loader": "^0.5.5", "html-loader": "^0.5.5",
"html-webpack-plugin": "^3.2.0", "html-webpack-plugin": "^3.2.0",
"mini-css-extract-plugin": "^0.8.2", "mini-css-extract-plugin": "^0.8.2",
"node-sass": "^4.14.0", "node-sass": "^4.14.1",
"postcss-loader": "^3.0.0", "postcss-loader": "^3.0.0",
"progress-bar-webpack-plugin": "^1.12.1", "progress-bar-webpack-plugin": "^1.12.1",
"rimraf": "^2.6.2", "rimraf": "^2.6.2",

5
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/startingParam/index.vue

@ -76,14 +76,9 @@
return '-' return '-'
}, },
_getNotifyGroupList () { _getNotifyGroupList () {
let notifyGroupListS = _.cloneDeep(this.store.state.dag.notifyGroupListS) || []
if (!notifyGroupListS.length) {
this.store.dispatch('dag/getNotifyGroupList').then(res => { this.store.dispatch('dag/getNotifyGroupList').then(res => {
this.notifyGroupList = res this.notifyGroupList = res
}) })
} else {
this.notifyGroupList = notifyGroupListS
}
}, },
_getWorkerGroupList () { _getWorkerGroupList () {
let stateWorkerGroupsList = this.store.state.security.workerGroupsListAll || [] let stateWorkerGroupsList = this.store.state.security.workerGroupsListAll || []

6
dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue

@ -243,16 +243,10 @@
}, },
_getNotifyGroupList () { _getNotifyGroupList () {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let notifyGroupListS = _.cloneDeep(this.store.state.dag.notifyGroupListS) || []
if (!notifyGroupListS.length) {
this.store.dispatch('dag/getNotifyGroupList').then(res => { this.store.dispatch('dag/getNotifyGroupList').then(res => {
this.notifyGroupList = res this.notifyGroupList = res
resolve() resolve()
}) })
} else {
this.notifyGroupList = notifyGroupListS
resolve()
}
}) })
}, },
_getReceiver () { _getReceiver () {

6
dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue

@ -280,8 +280,6 @@
_getNotifyGroupList () { _getNotifyGroupList () {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let notifyGroupListS = _.cloneDeep(this.store.state.dag.notifyGroupListS) || []
if (!notifyGroupListS.length) {
this.store.dispatch('dag/getNotifyGroupList').then(res => { this.store.dispatch('dag/getNotifyGroupList').then(res => {
this.notifyGroupList = res this.notifyGroupList = res
if (this.notifyGroupList.length) { if (this.notifyGroupList.length) {
@ -290,10 +288,6 @@
reject(new Error(0)) reject(new Error(0))
} }
}) })
} else {
this.notifyGroupList = notifyGroupListS
resolve()
}
}) })
}, },
ok () { ok () {

2
script/dolphinscheduler-daemon.sh

@ -46,7 +46,7 @@ export DOLPHINSCHEDULER_LOG_DIR=$DOLPHINSCHEDULER_HOME/logs
export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf
export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/* export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/*
export DOLPHINSCHEDULER_OPTS="-server -Xmx16g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-"-server -Xmx16g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"}
export STOP_TIMEOUT=5 export STOP_TIMEOUT=5
if [ ! -d "$DOLPHINSCHEDULER_LOG_DIR" ]; then if [ ! -d "$DOLPHINSCHEDULER_LOG_DIR" ]; then

Loading…
Cancel
Save