diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py b/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py
index 93b3249614..049b2cf3ae 100644
--- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py
+++ b/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py
@@ -76,8 +76,8 @@ else:
dolphin_alert_map = {}
wechat_push_url = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token'
wechat_token_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret'
-wechat_team_send_msg = '{\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"}'
-wechat_user_send_msg = '{\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}}'
+wechat_team_send_msg = '{\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"}'
+wechat_user_send_msg = '{\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}}'
dolphin_alert_map['enterprise.wechat.push.ur'] = wechat_push_url
dolphin_alert_map['enterprise.wechat.token.url'] = wechat_token_url
diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/params.py b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/params.py
index 3ca46f2182..5a9994f559 100644
--- a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/params.py
+++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/params.py
@@ -77,8 +77,8 @@ else:
dolphin_alert_map = {}
wechat_push_url = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token'
wechat_token_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret'
-wechat_team_send_msg = '{\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"}'
-wechat_user_send_msg = '{\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}}'
+wechat_team_send_msg = '{\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"}'
+wechat_user_send_msg = '{\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}}'
dolphin_alert_config_map = config['configurations']['dolphin-alert']
diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile
index bb69347797..d0f16d5d0d 100644
--- a/docker/build/Dockerfile
+++ b/docker/build/Dockerfile
@@ -42,7 +42,7 @@ ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz
RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ /opt/dolphinscheduler/
ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
-#4. install pg
+#4. install database, if use mysql as your backend database, the `mysql-client` package should be installed
RUN apk add postgresql postgresql-contrib
#5. modify nginx
RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \
diff --git a/docker/build/README.md b/docker/build/README.md
index 5725ec56ce..bc516bc214 100644
--- a/docker/build/README.md
+++ b/docker/build/README.md
@@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org
#### You can start a dolphinscheduler instance
```
$ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \
+-e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \
-p 8888:8888 \
dolphinscheduler all
```
@@ -25,14 +25,14 @@ The default postgres user `root`, postgres password `root` and database `dolphin
The default zookeeper is created in the `startup.sh`.
-#### Or via Environment Variables **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`POSTGRESQL_DATABASE`** **`ZOOKEEPER_QUORUM`**
+#### Or via Environment Variables **`DATABASE_HOST`** **`DATABASE_PORT`** **`DATABASE_DATABASE`** **`ZOOKEEPER_QUORUM`**
You can specify **existing postgres service**. Example:
```
$ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-p 8888:8888 \
dolphinscheduler all
```
@@ -42,7 +42,7 @@ You can specify **existing zookeeper service**. Example:
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \
-p 8888:8888 \
dolphinscheduler all
```
@@ -56,8 +56,8 @@ You can start a standalone dolphinscheduler server.
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler master-server
```
@@ -66,8 +66,8 @@ dolphinscheduler master-server
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler worker-server
```
@@ -75,8 +75,8 @@ dolphinscheduler worker-server
```
$ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-p 12345:12345 \
dolphinscheduler api-server
```
@@ -85,8 +85,8 @@ dolphinscheduler api-server
```
$ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler alert-server
```
@@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \
dolphinscheduler frontend
```
-**Note**: You must be specify `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server.
+**Note**: You must be specify `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server.
## How to build a docker image
@@ -124,33 +124,51 @@ Please read `./docker/build/hooks/build` `./docker/build/hooks/build.bat` script
The Dolphin Scheduler image uses several environment variables which are easy to miss. While none of the variables are required, they may significantly aid you in using the image.
-**`POSTGRESQL_HOST`**
+**`DATABASE_TYPE`**
-This environment variable sets the host for PostgreSQL. The default value is `127.0.0.1`.
+This environment variable sets the type for database. The default value is `postgresql`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
-**`POSTGRESQL_PORT`**
+**`DATABASE_DRIVER`**
-This environment variable sets the port for PostgreSQL. The default value is `5432`.
+This environment variable sets the type for database. The default value is `org.postgresql.Driver`.
+
+**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
+
+**`DATABASE_HOST`**
+
+This environment variable sets the host for database. The default value is `127.0.0.1`.
+
+**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
+
+**`DATABASE_PORT`**
+
+This environment variable sets the port for database. The default value is `5432`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
-**`POSTGRESQL_USERNAME`**
+**`DATABASE_USERNAME`**
+
+This environment variable sets the username for database. The default value is `root`.
+
+**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
+
+**`DATABASE_PASSWORD`**
-This environment variable sets the username for PostgreSQL. The default value is `root`.
+This environment variable sets the password for database. The default value is `root`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
-**`POSTGRESQL_PASSWORD`**
+**`DATABASE_DATABASE`**
-This environment variable sets the password for PostgreSQL. The default value is `root`.
+This environment variable sets the database for database. The default value is `dolphinscheduler`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
-**`POSTGRESQL_DATABASE`**
+**`DATABASE_PARAMS`**
-This environment variable sets the database for PostgreSQL. The default value is `dolphinscheduler`.
+This environment variable sets the database for database. The default value is `characterEncoding=utf8`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md
index 0605bef547..c2affc0691 100644
--- a/docker/build/README_zh_CN.md
+++ b/docker/build/README_zh_CN.md
@@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org
#### 你可以运行一个dolphinscheduler实例
```
$ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \
+-e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \
-p 8888:8888 \
dolphinscheduler all
```
@@ -25,14 +25,14 @@ dolphinscheduler all
同时，默认的`Zookeeper`也会在`startup.sh`脚本中被创建。
-#### 或者通过环境变量 **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`ZOOKEEPER_QUORUM`** 使用已存在的服务
+#### 或者通过环境变量 **`DATABASE_HOST`** **`DATABASE_PORT`** **`ZOOKEEPER_QUORUM`** 使用已存在的服务
你可以指定一个已经存在的 **`Postgres`** 服务. 如下:
```
$ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-p 8888:8888 \
dolphinscheduler all
```
@@ -42,7 +42,7 @@ dolphinscheduler all
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \
-p 8888:8888 \
dolphinscheduler all
```
@@ -56,8 +56,8 @@ dolphinscheduler all
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler master-server
```
@@ -66,8 +66,8 @@ dolphinscheduler master-server
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler worker-server
```
@@ -75,8 +75,8 @@ dolphinscheduler worker-server
```
$ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-p 12345:12345 \
dolphinscheduler api-server
```
@@ -85,8 +85,8 @@ dolphinscheduler api-server
```
$ docker run -dit --name dolphinscheduler \
--e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
--e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
+-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
+-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
dolphinscheduler alert-server
```
@@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \
dolphinscheduler frontend
```
-**注意**: 当你运行dolphinscheduler中的部分服务时，你必须指定这些环境变量 `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM`。
+**注意**: 当你运行dolphinscheduler中的部分服务时，你必须指定这些环境变量 `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM`。
## 如何构建一个docker镜像
@@ -124,33 +124,51 @@ c:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat
Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些变量不是必须的，但是可以帮助你更容易配置镜像并根据你的需求定义相应的服务配置。
-**`POSTGRESQL_HOST`**
+**`DATABASE_TYPE`**
-配置`PostgreSQL`的`HOST`， 默认值 `127.0.0.1`。
+配置`database`的`TYPE`， 默认值 `postgresql`。
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
-**`POSTGRESQL_PORT`**
+**`DATABASE_DRIVER`**
-配置`PostgreSQL`的`PORT`， 默认值 `5432`。
+配置`database`的`DRIVER`， 默认值 `org.postgresql.Driver`。
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
-**`POSTGRESQL_USERNAME`**
+**`DATABASE_HOST`**
-配置`PostgreSQL`的`USERNAME`， 默认值 `root`。
+配置`database`的`HOST`， 默认值 `127.0.0.1`。
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
-**`POSTGRESQL_PASSWORD`**
+**`DATABASE_PORT`**
-配置`PostgreSQL`的`PASSWORD`， 默认值 `root`。
+配置`database`的`PORT`， 默认值 `5432`。
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
-**`POSTGRESQL_DATABASE`**
+**`DATABASE_USERNAME`**
-配置`PostgreSQL`的`DATABASE`， 默认值 `dolphinscheduler`。
+配置`database`的`USERNAME`， 默认值 `root`。
+
+**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
+
+**`DATABASE_PASSWORD`**
+
+配置`database`的`PASSWORD`， 默认值 `root`。
+
+**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
+
+**`DATABASE_DATABASE`**
+
+配置`database`的`DATABASE`， 默认值 `dolphinscheduler`。
+
+**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
+
+**`DATABASE_PARAMS`**
+
+配置`database`的`PARAMS`， 默认值 `characterEncoding=utf8`。
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。
diff --git a/docker/build/conf/dolphinscheduler/alert.properties.tpl b/docker/build/conf/dolphinscheduler/alert.properties.tpl
index b940ecd203..6d7450e13d 100644
--- a/docker/build/conf/dolphinscheduler/alert.properties.tpl
+++ b/docker/build/conf/dolphinscheduler/alert.properties.tpl
@@ -35,16 +35,22 @@ mail.smtp.ssl.trust=${MAIL_SMTP_SSL_TRUST}
#xls file path,need create if not exist
xls.file.path=${XLS_FILE_PATH}
+# plugins dir
+plugin.dir=${ALERT_PLUGIN_DIR}
+
# Enterprise WeChat configuration
enterprise.wechat.enable=${ENTERPRISE_WECHAT_ENABLE}
enterprise.wechat.corp.id=${ENTERPRISE_WECHAT_CORP_ID}
enterprise.wechat.secret=${ENTERPRISE_WECHAT_SECRET}
enterprise.wechat.agent.id=${ENTERPRISE_WECHAT_AGENT_ID}
enterprise.wechat.users=${ENTERPRISE_WECHAT_USERS}
-enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret
-enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token
-enterprise.wechat.team.send.msg={\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"}
-enterprise.wechat.user.send.msg={\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}}
+enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid={corpId}&corpsecret={secret}
+enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={token}
+enterprise.wechat.team.send.msg={\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"}
+enterprise.wechat.user.send.msg={\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}}
+
+
+
diff --git a/docker/build/conf/dolphinscheduler/common.properties.tpl b/docker/build/conf/dolphinscheduler/common.properties.tpl
index f318ff8414..ff74598fd4 100644
--- a/docker/build/conf/dolphinscheduler/common.properties.tpl
+++ b/docker/build/conf/dolphinscheduler/common.properties.tpl
@@ -25,13 +25,13 @@ dolphinscheduler.env.path=${DOLPHINSCHEDULER_ENV_PATH}
data.basedir.path=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH}
# resource upload startup type : HDFS,S3,NONE
-resource.storage.type=NONE
+resource.storage.type=${RESOURCE_STORAGE_TYPE}
#============================================================================
# HDFS
#============================================================================
# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended
-#resource.upload.path=/dolphinscheduler
+resource.upload.path=${RESOURCE_UPLOAD_PATH}
# whether kerberos starts
#hadoop.security.authentication.startup.state=false
@@ -58,16 +58,16 @@ kerberos.expire.time=7
# S3
#============================================================================
# if resource.storage.type=S3，the value like: s3a://dolphinscheduler ; if resource.storage.type=HDFS, When namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
-fs.defaultFS=hdfs://mycluster:8020
+fs.defaultFS=${FS_DEFAULT_FS}
# if resource.storage.type=S3，s3 endpoint
-#fs.s3a.endpoint=http://192.168.199.91:9010
+fs.s3a.endpoint=${FS_S3A_ENDPOINT}
# if resource.storage.type=S3，s3 access key
-#fs.s3a.access.key=A3DXS30FO22544RE
+fs.s3a.access.key=${FS_S3A_ACCESS_KEY}
# if resource.storage.type=S3，s3 secret key
-#fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK
+fs.s3a.secret.key=${FS_S3A_SECRET_KEY}
# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty TODO
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
diff --git a/docker/build/conf/dolphinscheduler/datasource.properties.tpl b/docker/build/conf/dolphinscheduler/datasource.properties.tpl
index aefb9e3b0b..f7c5ee6881 100644
--- a/docker/build/conf/dolphinscheduler/datasource.properties.tpl
+++ b/docker/build/conf/dolphinscheduler/datasource.properties.tpl
@@ -15,16 +15,11 @@
# limitations under the License.
#
-
-# mysql
-#spring.datasource.driver-class-name=com.mysql.jdbc.Driver
-#spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
-
-# postgre
-spring.datasource.driver-class-name=org.postgresql.Driver
-spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8
-spring.datasource.username=${POSTGRESQL_USERNAME}
-spring.datasource.password=${POSTGRESQL_PASSWORD}
+# db
+spring.datasource.driver-class-name=${DATABASE_DRIVER}
+spring.datasource.url=jdbc:${DATABASE_TYPE}://${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DATABASE}?${DATABASE_PARAMS}
+spring.datasource.username=${DATABASE_USERNAME}
+spring.datasource.password=${DATABASE_PASSWORD}
## base spring data source configuration todo need to remove
#spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
diff --git a/docker/build/conf/dolphinscheduler/logback/logback-alert.xml b/docker/build/conf/dolphinscheduler/logback/logback-alert.xml
index 5d1c07858d..1718947dd1 100644
--- a/docker/build/conf/dolphinscheduler/logback/logback-alert.xml
+++ b/docker/build/conf/dolphinscheduler/logback/logback-alert.xml
@@ -46,7 +46,7 @@
-
+
\ No newline at end of file
diff --git a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl b/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
index 362a4e85ad..51540aa345 100644
--- a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
+++ b/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl
@@ -19,7 +19,7 @@
zookeeper.quorum=${ZOOKEEPER_QUORUM}
# dolphinscheduler root directory
-#zookeeper.dolphinscheduler.root=/dolphinscheduler
+zookeeper.dolphinscheduler.root=${ZOOKEEPER_ROOT}
# dolphinscheduler failover directory
#zookeeper.session.timeout=60000
diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh
index da6eb21b7d..73fdad6798 100644
--- a/docker/build/startup-init-conf.sh
+++ b/docker/build/startup-init-conf.sh
@@ -24,22 +24,33 @@ echo "init env variables"
#============================================================================
# Database Source
#============================================================================
-export POSTGRESQL_HOST=${POSTGRESQL_HOST:-"127.0.0.1"}
-export POSTGRESQL_PORT=${POSTGRESQL_PORT:-"5432"}
-export POSTGRESQL_USERNAME=${POSTGRESQL_USERNAME:-"root"}
-export POSTGRESQL_PASSWORD=${POSTGRESQL_PASSWORD:-"root"}
-export POSTGRESQL_DATABASE=${POSTGRESQL_DATABASE:-"dolphinscheduler"}
+export DATABASE_HOST=${DATABASE_HOST:-"127.0.0.1"}
+export DATABASE_PORT=${DATABASE_PORT:-"5432"}
+export DATABASE_USERNAME=${DATABASE_USERNAME:-"root"}
+export DATABASE_PASSWORD=${DATABASE_PASSWORD:-"root"}
+export DATABASE_DATABASE=${DATABASE_DATABASE:-"dolphinscheduler"}
+export DATABASE_TYPE=${DATABASE_TYPE:-"postgresql"}
+export DATABASE_DRIVER=${DATABASE_DRIVER:-"org.postgresql.Driver"}
+export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"}
#============================================================================
# System
#============================================================================
export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"}
export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"}
+export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""}
+export RESOURCE_STORAGE_TYPE=${RESOURCE_STORAGE_TYPE:-"NONE"}
+export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/ds"}
+export FS_DEFAULT_FS=${FS_DEFAULT_FS:-"s3a://xxxx"}
+export FS_S3A_ENDPOINT=${FS_S3A_ENDPOINT:-"s3.xxx.amazonaws.com"}
+export FS_S3A_ACCESS_KEY=${FS_S3A_ACCESS_KEY:-"xxxxxxx"}
+export FS_S3A_SECRET_KEY=${FS_S3A_SECRET_KEY:-"xxxxxxx"}
#============================================================================
# Zookeeper
#============================================================================
export ZOOKEEPER_QUORUM=${ZOOKEEPER_QUORUM:-"127.0.0.1:2181"}
+export ZOOKEEPER_ROOT=${ZOOKEEPER_ROOT:-"/dolphinscheduler"}
#============================================================================
# Master Server
@@ -67,6 +78,8 @@ export WORKER_GROUP=${WORKER_GROUP:-"default"}
#============================================================================
# Alert Server
#============================================================================
+# alert plugin dir
+export ALERT_PLUGIN_DIR=${ALERT_PLUGIN_DIR:-"/opt/dolphinscheduler"}
# XLS FILE
export XLS_FILE_PATH=${XLS_FILE_PATH:-"/tmp/xls"}
# mail
diff --git a/docker/build/startup.sh b/docker/build/startup.sh
index 00bb9cdc52..ea4fbc0cd1 100644
--- a/docker/build/startup.sh
+++ b/docker/build/startup.sh
@@ -22,24 +22,32 @@ DOLPHINSCHEDULER_BIN=${DOLPHINSCHEDULER_HOME}/bin
DOLPHINSCHEDULER_SCRIPT=${DOLPHINSCHEDULER_HOME}/script
DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs
-# start postgresql
-initPostgreSQL() {
- echo "test postgresql service"
- while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do
+# start database
+initDatabase() {
+ echo "test ${DATABASE_TYPE} service"
+ while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
counter=$((counter+1))
if [ $counter == 30 ]; then
- echo "Error: Couldn't connect to postgresql."
+ echo "Error: Couldn't connect to ${DATABASE_TYPE}."
exit 1
fi
- echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter."
+ echo "Trying to connect to ${DATABASE_TYPE} at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
sleep 5
done
- echo "connect postgresql service"
- v=$(sudo -u postgres PGPASSWORD=${POSTGRESQL_PASSWORD} psql -h ${POSTGRESQL_HOST} -p ${POSTGRESQL_PORT} -U ${POSTGRESQL_USERNAME} -d dolphinscheduler -tAc "select 1")
- if [ "$(echo '${v}' | grep 'FATAL' | wc -l)" -eq 1 ]; then
- echo "Error: Can't connect to database...${v}"
- exit 1
+ echo "connect ${DATABASE_TYPE} service"
+ if [ ${DATABASE_TYPE} = "mysql" ]; then
+ v=$(mysql -h${DATABASE_HOST} -P${DATABASE_PORT} -u${DATABASE_USERNAME} --password=${DATABASE_PASSWORD} -D ${DATABASE_DATABASE} -e "select 1" 2>&1)
+ if [ "$(echo '${v}' | grep 'ERROR' | wc -l)" -eq 1 ]; then
+ echo "Error: Can't connect to database...${v}"
+ exit 1
+ fi
+ else
+ v=$(sudo -u postgres PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "select 1")
+ if [ "$(echo '${v}' | grep 'FATAL' | wc -l)" -eq 1 ]; then
+ echo "Error: Can't connect to database...${v}"
+ exit 1
+ fi
fi
echo "import sql data"
@@ -123,7 +131,7 @@ LOGFILE=/var/log/nginx/access.log
case "$1" in
(all)
initZK
- initPostgreSQL
+ initDatabase
initMasterServer
initWorkerServer
initApiServer
@@ -134,25 +142,25 @@ case "$1" in
;;
(master-server)
initZK
- initPostgreSQL
+ initDatabase
initMasterServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-master.log
;;
(worker-server)
initZK
- initPostgreSQL
+ initDatabase
initWorkerServer
initLoggerServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log
;;
(api-server)
initZK
- initPostgreSQL
+ initDatabase
initApiServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log
;;
(alert-server)
- initPostgreSQL
+ initDatabase
initAlertServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-alert.log
;;
diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml
index 5cb6717f15..51eb0aeaa5 100644
--- a/docker/docker-swarm/docker-compose.yml
+++ b/docker/docker-swarm/docker-compose.yml
@@ -56,11 +56,11 @@ services:
- 12345:12345
environment:
TZ: Asia/Shanghai
- POSTGRESQL_HOST: dolphinscheduler-postgresql
- POSTGRESQL_PORT: 5432
- POSTGRESQL_USERNAME: root
- POSTGRESQL_PASSWORD: root
- POSTGRESQL_DATABASE: dolphinscheduler
+ DATABASE_HOST: dolphinscheduler-postgresql
+ DATABASE_PORT: 5432
+ DATABASE_USERNAME: root
+ DATABASE_PASSWORD: root
+ DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
@@ -119,11 +119,11 @@ services:
ENTERPRISE_WECHAT_SECRET: ""
ENTERPRISE_WECHAT_AGENT_ID: ""
ENTERPRISE_WECHAT_USERS: ""
- POSTGRESQL_HOST: dolphinscheduler-postgresql
- POSTGRESQL_PORT: 5432
- POSTGRESQL_USERNAME: root
- POSTGRESQL_PASSWORD: root
- POSTGRESQL_DATABASE: dolphinscheduler
+ DATABASE_HOST: dolphinscheduler-postgresql
+ DATABASE_PORT: 5432
+ DATABASE_USERNAME: root
+ DATABASE_PASSWORD: root
+ DATABASE_DATABASE: dolphinscheduler
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
interval: 30s
@@ -152,11 +152,11 @@ services:
MASTER_TASK_COMMIT_INTERVAL: "1000"
MASTER_MAX_CPULOAD_AVG: "100"
MASTER_RESERVED_MEMORY: "0.1"
- POSTGRESQL_HOST: dolphinscheduler-postgresql
- POSTGRESQL_PORT: 5432
- POSTGRESQL_USERNAME: root
- POSTGRESQL_PASSWORD: root
- POSTGRESQL_DATABASE: dolphinscheduler
+ DATABASE_HOST: dolphinscheduler-postgresql
+ DATABASE_PORT: 5432
+ DATABASE_USERNAME: root
+ DATABASE_PASSWORD: root
+ DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
@@ -188,11 +188,11 @@ services:
WORKER_RESERVED_MEMORY: "0.1"
WORKER_GROUP: "default"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
- POSTGRESQL_HOST: dolphinscheduler-postgresql
- POSTGRESQL_PORT: 5432
- POSTGRESQL_USERNAME: root
- POSTGRESQL_PASSWORD: root
- POSTGRESQL_DATABASE: dolphinscheduler
+ DATABASE_HOST: dolphinscheduler-postgresql
+ DATABASE_PORT: 5432
+ DATABASE_USERNAME: root
+ DATABASE_PASSWORD: root
+ DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]
diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml
index 5c3873fb07..ca9f7c88c7 100644
--- a/docker/docker-swarm/docker-stack.yml
+++ b/docker/docker-swarm/docker-stack.yml
@@ -20,13 +20,13 @@ services:
dolphinscheduler-postgresql:
image: bitnami/postgresql:latest
- ports:
- - 5432:5432
environment:
TZ: Asia/Shanghai
POSTGRESQL_USERNAME: root
POSTGRESQL_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler
+ ports:
+ - 5432:5432
volumes:
- dolphinscheduler-postgresql:/bitnami/postgresql
networks:
@@ -37,12 +37,12 @@ services:
dolphinscheduler-zookeeper:
image: bitnami/zookeeper:latest
- ports:
- - 2181:2181
environment:
TZ: Asia/Shanghai
ALLOW_ANONYMOUS_LOGIN: "yes"
ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
+ ports:
+ - 2181:2181
volumes:
- dolphinscheduler-zookeeper:/bitnami/zookeeper
networks:
@@ -54,16 +54,16 @@ services:
dolphinscheduler-api:
image: apache/dolphinscheduler:latest
command: ["api-server"]
- ports:
- - 12345:12345
environment:
TZ: Asia/Shanghai
- POSTGRESQL_HOST: dolphinscheduler-postgresql
- POSTGRESQL_PORT: 5432
- POSTGRESQL_USERNAME: root
- POSTGRESQL_PASSWORD: root
- POSTGRESQL_DATABASE: dolphinscheduler
+ DATABASE_HOST: dolphinscheduler-postgresql
+ DATABASE_PORT: 5432
+ DATABASE_USERNAME: root
+ DATABASE_PASSWORD: root
+ DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
+ ports:
+ - 12345:12345
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
interval: 30
@@ -120,11 +120,11 @@ services:
ENTERPRISE_WECHAT_SECRET: ""
ENTERPRISE_WECHAT_AGENT_ID: ""
ENTERPRISE_WECHAT_USERS: ""
- POSTGRESQL_HOST: dolphinscheduler-postgresql
- POSTGRESQL_PORT: 5432
- POSTGRESQL_USERNAME: root
- POSTGRESQL_PASSWORD: root
- POSTGRESQL_DATABASE: dolphinscheduler
+ DATABASE_HOST: dolphinscheduler-postgresql
+ DATABASE_PORT: 5432
+ DATABASE_USERNAME: root
+ DATABASE_PASSWORD: root
+ DATABASE_DATABASE: dolphinscheduler
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
interval: 30
@@ -153,11 +153,11 @@ services:
MASTER_TASK_COMMIT_INTERVAL: "1000"
MASTER_MAX_CPULOAD_AVG: "100"
MASTER_RESERVED_MEMORY: "0.1"
- POSTGRESQL_HOST: dolphinscheduler-postgresql
- POSTGRESQL_PORT: 5432
- POSTGRESQL_USERNAME: root
- POSTGRESQL_PASSWORD: root
- POSTGRESQL_DATABASE: dolphinscheduler
+ DATABASE_HOST: dolphinscheduler-postgresql
+ DATABASE_PORT: 5432
+ DATABASE_USERNAME: root
+ DATABASE_PASSWORD: root
+ DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
@@ -188,11 +188,11 @@ services:
WORKER_RESERVED_MEMORY: "0.1"
WORKER_GROUP: "default"
DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
- POSTGRESQL_HOST: dolphinscheduler-postgresql
- POSTGRESQL_PORT: 5432
- POSTGRESQL_USERNAME: root
- POSTGRESQL_PASSWORD: root
- POSTGRESQL_DATABASE: dolphinscheduler
+ DATABASE_HOST: dolphinscheduler-postgresql
+ DATABASE_PORT: 5432
+ DATABASE_USERNAME: root
+ DATABASE_PASSWORD: root
+ DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]
diff --git a/docker/kubernetes/dolphinscheduler/Chart.yaml b/docker/kubernetes/dolphinscheduler/Chart.yaml
index ac989d571f..9d640869dd 100644
--- a/docker/kubernetes/dolphinscheduler/Chart.yaml
+++ b/docker/kubernetes/dolphinscheduler/Chart.yaml
@@ -49,4 +49,4 @@ dependencies:
- name: zookeeper
version: 5.x.x
repository: https://charts.bitnami.com/bitnami
- condition: redis.enabled
+ condition: zookeeper.enabled
diff --git a/docker/kubernetes/dolphinscheduler/README.md b/docker/kubernetes/dolphinscheduler/README.md
index 9e6d1c6448..0691b1ed2e 100644
--- a/docker/kubernetes/dolphinscheduler/README.md
+++ b/docker/kubernetes/dolphinscheduler/README.md
@@ -46,8 +46,8 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `image.repository` | Docker image repository for the Dolphins Scheduler | `dolphinscheduler` |
| `image.tag` | Docker image version for the Dolphins Scheduler | `1.2.1` |
| `image.imagePullPolicy` | Image pull policy. One of Always, Never, IfNotPresent | `IfNotPresent` |
-| `imagePullSecrets` | ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images | `[]` |
-| | | |
+| `image.pullSecres` | PullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images | `[]` |
+| | | |
| `postgresql.enabled` | If not exists external PostgreSQL, by default, the Dolphins Scheduler will use a internal PostgreSQL | `true` |
| `postgresql.postgresqlUsername` | The username for internal PostgreSQL | `root` |
| `postgresql.postgresqlPassword` | The password for internal PostgreSQL | `root` |
@@ -55,12 +55,15 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `postgresql.persistence.enabled` | Set `postgresql.persistence.enabled` to `true` to mount a new volume for internal PostgreSQL | `false` |
| `postgresql.persistence.size` | `PersistentVolumeClaim` Size | `20Gi` |
| `postgresql.persistence.storageClass` | PostgreSQL data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
-| `externalDatabase.host` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database host will use it. | `localhost` |
+| `externalDatabase.type` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database type will use it. | `postgresql` |
+| `externalDatabase.driver` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database driver will use it. | `org.postgresql.Driver` |
+| `externalDatabase.host` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database host will use it. | `localhost` |
| `externalDatabase.port` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database port will use it. | `5432` |
| `externalDatabase.username` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database username will use it. | `root` |
| `externalDatabase.password` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database password will use it. | `root` |
| `externalDatabase.database` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database database will use it. | `dolphinscheduler` |
-| | | |
+| `externalDatabase.params` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database params will use it. | `characterEncoding=utf8` |
+| | | |
| `zookeeper.enabled` | If not exists external Zookeeper, by default, the Dolphin Scheduler will use a internal Zookeeper | `true` |
| `zookeeper.taskQueue` | Specify task queue for `master` and `worker` | `zookeeper` |
| `zookeeper.persistence.enabled` | Set `zookeeper.persistence.enabled` to `true` to mount a new volume for internal Zookeeper | `false` |
@@ -68,12 +71,25 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `zookeeper.persistence.storageClass` | Zookeeper data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `externalZookeeper.taskQueue` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify task queue for `master` and `worker` | `zookeeper` |
| `externalZookeeper.zookeeperQuorum` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify Zookeeper quorum | `127.0.0.1:2181` |
-| | | |
+| `externalZookeeper.zookeeperRoot` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify Zookeeper root path for `master` and `worker` | `dolphinscheduler` |
+| | | |
+| `common.configmap.DOLPHINSCHEDULER_ENV_PATH` | Extra env file path. | `/tmp/dolphinscheduler/env` |
+| `common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH` | File uploaded path of DS. | `/tmp/dolphinscheduler/files` |
+| `common.configmap.RESOURCE_STORAGE_TYPE` | Resource Storate type, support type are: S3、HDFS、NONE. | `NONE` |
+| `common.configmap.RESOURCE_UPLOAD_PATH` | The base path of resource. | `/ds` |
+| `common.configmap.FS_DEFAULT_FS` | The default fs of resource, for s3 is the `s3a` prefix and bucket name. | `s3a://xxxx` |
+| `common.configmap.FS_S3A_ENDPOINT` | If the resource type is `S3`, you should fill this filed, it's the endpoint of s3. | `s3.xxx.amazonaws.com` |
+| `common.configmap.FS_S3A_ACCESS_KEY` | The access key for your s3 bucket. | `xxxxxxx` |
+| `common.configmap.FS_S3A_SECRET_KEY` | The secret key for your s3 bucket. | `xxxxxxx` |
| `master.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` |
+| | | |
| `master.replicas` | Replicas is the desired number of replicas of the given Template | `3` |
| `master.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
| `master.tolerations` | If specified, the pod's tolerations | `{}` |
| `master.affinity` | If specified, the pod's scheduling constraints | `{}` |
+| `master.jvmOptions` | The JVM options for master server. | `""` |
+| `master.resources` | The `resource` limit and request config for master server. | `{}` |
+| `master.annotations` | The `annotations` for master server. | `{}` |
| `master.configmap.MASTER_EXEC_THREADS` | Master execute thread num | `100` |
| `master.configmap.MASTER_EXEC_TASK_NUM` | Master execute task number in parallel | `20` |
| `master.configmap.MASTER_HEARTBEAT_INTERVAL` | Master heartbeat interval | `10` |
@@ -97,12 +113,15 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `master.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` |
| `master.persistentVolumeClaim.storageClassName` | `Master` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `master.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` |
-| | | |
+| | | |
| `worker.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` |
| `worker.replicas` | Replicas is the desired number of replicas of the given Template | `3` |
| `worker.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
| `worker.tolerations` | If specified, the pod's tolerations | `{}` |
| `worker.affinity` | If specified, the pod's scheduling constraints | `{}` |
+| `worker.jvmOptions` | The JVM options for worker server. | `""` |
+| `worker.resources` | The `resource` limit and request config for worker server. | `{}` |
+| `worker.annotations` | The `annotations` for worker server. | `{}` |
| `worker.configmap.WORKER_EXEC_THREADS` | Worker execute thread num | `100` |
| `worker.configmap.WORKER_HEARTBEAT_INTERVAL` | Worker heartbeat interval | `10` |
| `worker.configmap.WORKER_FETCH_TASK_NUM` | Submit the number of tasks at a time | `3` |
@@ -131,7 +150,7 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `worker.persistentVolumeClaim.logsPersistentVolume.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` |
| `worker.persistentVolumeClaim.logsPersistentVolume.storageClassName` | `Worker` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
| `worker.persistentVolumeClaim.logsPersistentVolume.storage` | `PersistentVolumeClaim` Size | `20Gi` |
-| | | |
+| | | |
| `alert.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` |
| `alert.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` |
| `alert.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` |
@@ -139,6 +158,10 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `alert.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
| `alert.tolerations` | If specified, the pod's tolerations | `{}` |
| `alert.affinity` | If specified, the pod's scheduling constraints | `{}` |
+| `alert.jvmOptions` | The JVM options for alert server. | `""` |
+| `alert.resources` | The `resource` limit and request config for alert server. | `{}` |
+| `alert.annotations` | The `annotations` for alert server. | `{}` |
+| `alert.configmap.ALERT_PLUGIN_DIR` | Alert plugin path. | `/opt/dolphinscheduler/alert/plugin` |
| `alert.configmap.XLS_FILE_PATH` | XLS file path | `/tmp/xls` |
| `alert.configmap.MAIL_SERVER_HOST` | Mail `SERVER HOST ` | `nil` |
| `alert.configmap.MAIL_SERVER_PORT` | Mail `SERVER PORT` | `nil` |
@@ -177,6 +200,9 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `api.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
| `api.tolerations` | If specified, the pod's tolerations | `{}` |
| `api.affinity` | If specified, the pod's scheduling constraints | `{}` |
+| `api.jvmOptions` | The JVM options for api server. | `""` |
+| `api.resources` | The `resource` limit and request config for api server. | `{}` |
+| `api.annotations` | The `annotations` for api server. | `{}` |
| `api.livenessProbe.enabled` | Turn on and off liveness probe | `true` |
| `api.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` |
| `api.livenessProbe.periodSeconds` | How often to perform the probe | `30` |
@@ -201,6 +227,8 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `frontend.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` |
| `frontend.tolerations` | If specified, the pod's tolerations | `{}` |
| `frontend.affinity` | If specified, the pod's scheduling constraints | `{}` |
+| `frontend.resources` | The `resource` limit and request config for frontend server. | `{}` |
+| `frontend.annotations` | The `annotations` for frontend server. | `{}` |
| `frontend.livenessProbe.enabled` | Turn on and off liveness probe | `true` |
| `frontend.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` |
| `frontend.livenessProbe.periodSeconds` | How often to perform the probe | `30` |
diff --git a/docker/kubernetes/dolphinscheduler/requirements.yaml b/docker/kubernetes/dolphinscheduler/requirements.yaml
deleted file mode 100644
index e219975995..0000000000
--- a/docker/kubernetes/dolphinscheduler/requirements.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-dependencies:
-- name: postgresql
- version: 8.x.x
- repository: https://charts.bitnami.com/bitnami
- condition: postgresql.enabled
-- name: zookeeper
- version: 5.x.x
- repository: https://charts.bitnami.com/bitnami
- condition: redis.enabled
\ No newline at end of file
diff --git a/docker/kubernetes/dolphinscheduler/templates/NOTES.txt b/docker/kubernetes/dolphinscheduler/templates/NOTES.txt
index eb3a9cfc52..256c53ca57 100644
--- a/docker/kubernetes/dolphinscheduler/templates/NOTES.txt
+++ b/docker/kubernetes/dolphinscheduler/templates/NOTES.txt
@@ -29,16 +29,3 @@
kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-frontend 8888:8888
{{- end }}
-
-2. Get the Dolphinscheduler URL by running:
-
-{{- if .Values.ingress.enabled }}
-
- export HOSTNAME=$(kubectl get ingress --namespace {{ .Release.Namespace }} {{ template "dolphinscheduler.fullname" . }} -o jsonpath='{.spec.rules[0].host}')
- echo "Dolphinscheduler URL: http://$HOSTNAME/"
-
-{{- else }}
-
- kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-frontend 8888:8888
-
-{{- end }}
\ No newline at end of file
diff --git a/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl
index 9ba290b771..1f121dfea8 100644
--- a/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl
+++ b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl
@@ -96,6 +96,14 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
{{- printf "%s/%s:%s" (include "dolphinscheduler.image.registry" .) .Values.image.repository .Values.image.tag -}}
{{- end -}}
+{{/*
+Create a default image pull secrects.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "dolphinscheduler.image.pullSecrets" -}}
+{{- default nil .Values.image.pullSecrets -}}
+{{- end -}}
+
{{/*
Create a default fully qualified postgresql name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml
index 76daad8568..2c5b76c3f0 100644
--- a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml
+++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml
@@ -24,6 +24,7 @@ metadata:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
data:
+ ALERT_PLUGIN_DIR: {{ .Values.alert.configmap.ALERT_PLUGIN_DIR | quote }}
XLS_FILE_PATH: {{ .Values.alert.configmap.XLS_FILE_PATH | quote }}
MAIL_SERVER_HOST: {{ .Values.alert.configmap.MAIL_SERVER_HOST | quote }}
MAIL_SERVER_PORT: {{ .Values.alert.configmap.MAIL_SERVER_PORT | quote }}
diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml
new file mode 100644
index 0000000000..931f6d48e4
--- /dev/null
+++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+{{- if .Values.common.configmap }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ labels:
+ app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-common
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/managed-by: {{ .Release.Service }}
+data:
+ DOLPHINSCHEDULER_ENV_PATH: {{ .Values.common.configmap.DOLPHINSCHEDULER_ENV_PATH | quote }}
+ DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ .Values.common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH | quote }}
+ RESOURCE_STORAGE_TYPE: {{ .Values.common.configmap.RESOURCE_STORAGE_TYPE | quote }}
+ RESOURCE_UPLOAD_PATH: {{ .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }}
+ FS_DEFAULT_FS: {{ .Values.common.configmap.FS_DEFAULT_FS | quote }}
+ FS_S3A_ENDPOINT: {{ .Values.common.configmap.FS_S3A_ENDPOINT | quote }}
+ FS_S3A_ACCESS_KEY: {{ .Values.common.configmap.FS_S3A_ACCESS_KEY | quote }}
+ FS_S3A_SECRET_KEY: {{ .Values.common.configmap.FS_S3A_SECRET_KEY | quote }}
+{{- end }}
\ No newline at end of file
diff --git a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml
index 69662e95d9..fb9a23786b 100644
--- a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml
+++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml
@@ -43,6 +43,10 @@ spec:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: alert
+ {{- if .Values.alert.annotations }}
+ annotations:
+ {{- toYaml .Values.alert.annotations | nindent 8 }}
+ {{- end }}
spec:
{{- if .Values.alert.affinity }}
affinity: {{- toYaml .Values.alert.affinity | nindent 8 }}
@@ -54,34 +58,38 @@ spec:
tolerations: {{- toYaml . | nindent 8 }}
{{- end }}
initContainers:
- - name: init-postgresql
+ - name: init-database
image: busybox:1.31.0
command:
- /bin/sh
- -ec
- |
- while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do
+ while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
counter=$((counter+1))
if [ $counter == 5 ]; then
- echo "Error: Couldn't connect to postgresql."
+ echo "Error: Couldn't connect to database."
exit 1
fi
- echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter."
+ echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
sleep 60
done
env:
- - name: POSTGRESQL_HOST
+ - name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- - name: POSTGRESQL_PORT
+ - name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
- value: {{ .Values.externalDatabase.port }}
+ value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
+ {{- if .Values.image.pullSecrets }}
+ imagePullSecrets:
+ - name: {{ include "dolphinscheduler.image.pullSecrets" . }}
+ {{- end }}
containers:
- name: {{ include "dolphinscheduler.fullname" . }}-alert
image: {{ include "dolphinscheduler.image.repository" . | quote }}
@@ -89,8 +97,15 @@ spec:
- "alert-server"
imagePullPolicy: {{ .Values.image.pullPolicy }}
env:
+ - name: DOLPHINSCHEDULER_OPTS
+ value: {{ default "" .Values.alert.jvmOptions }}
- name: TZ
value: {{ .Values.timezone }}
+ - name: ALERT_PLUGIN_DIR
+ valueFrom:
+ configMapKeyRef:
+ key: ALERT_PLUGIN_DIR
+ name: {{ include "dolphinscheduler.fullname" . }}-alert
- name: XLS_FILE_PATH
valueFrom:
configMapKeyRef:
@@ -161,25 +176,37 @@ spec:
configMapKeyRef:
key: ENTERPRISE_WECHAT_USERS
name: {{ include "dolphinscheduler.fullname" . }}-alert
- - name: POSTGRESQL_HOST
+ - name: DATABASE_TYPE
+ {{- if .Values.postgresql.enabled }}
+ value: "postgresql"
+ {{- else }}
+ value: {{ .Values.externalDatabase.type | quote }}
+ {{- end }}
+ - name: DATABASE_DRIVER
+ {{- if .Values.postgresql.enabled }}
+ value: "org.postgresql.Driver"
+ {{- else }}
+ value: {{ .Values.externalDatabase.driver | quote }}
+ {{- end }}
+ - name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- - name: POSTGRESQL_PORT
+ - name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
- value: {{ .Values.externalDatabase.port }}
+ value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
- - name: POSTGRESQL_USERNAME
+ - name: DATABASE_USERNAME
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlUsername }}
{{- else }}
value: {{ .Values.externalDatabase.username | quote }}
{{- end }}
- - name: POSTGRESQL_PASSWORD
+ - name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
{{- if .Values.postgresql.enabled }}
@@ -189,12 +216,57 @@ spec:
name: {{ printf "%s-%s" .Release.Name "externaldb" }}
key: db-password
{{- end }}
- - name: POSTGRESQL_DATABASE
+ - name: DATABASE_DATABASE
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlDatabase }}
{{- else }}
value: {{ .Values.externalDatabase.database | quote }}
{{- end }}
+ - name: DATABASE_PARAMS
+ {{- if .Values.postgresql.enabled }}
+ value: "characterEncoding=utf8"
+ {{- else }}
+ value: {{ .Values.externalDatabase.params | quote }}
+ {{- end }}
+ - name: RESOURCE_STORAGE_TYPE
+ valueFrom:
+ configMapKeyRef:
+ key: RESOURCE_STORAGE_TYPE
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ - name: RESOURCE_UPLOAD_PATH
+ valueFrom:
+ configMapKeyRef:
+ key: RESOURCE_UPLOAD_PATH
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ - name: FS_DEFAULT_FS
+ valueFrom:
+ configMapKeyRef:
+ key: FS_DEFAULT_FS
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ - name: FS_S3A_ENDPOINT
+ valueFrom:
+ configMapKeyRef:
+ key: FS_S3A_ENDPOINT
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ - name: FS_S3A_ACCESS_KEY
+ valueFrom:
+ configMapKeyRef:
+ key: FS_S3A_ACCESS_KEY
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ - name: FS_S3A_SECRET_KEY
+ valueFrom:
+ configMapKeyRef:
+ key: FS_S3A_SECRET_KEY
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ {{- if .Values.alert.resources }}
+ resources:
+ limits:
+ memory: {{ .Values.alert.resources.limits.memory }}
+ cpu: {{ .Values.alert.resources.limits.cpu }}
+ requests:
+ memory: {{ .Values.alert.resources.requests.memory }}
+ cpu: {{ .Values.alert.resources.requests.cpu }}
+ {{- end }}
{{- if .Values.alert.livenessProbe.enabled }}
livenessProbe:
exec:
diff --git a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
index 487ede0b8f..51a83bcfa7 100644
--- a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
+++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
@@ -43,6 +43,10 @@ spec:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: api
+ {{- if .Values.alert.annotations }}
+ annotations:
+ {{- toYaml .Values.alert.annotations | nindent 8 }}
+ {{- end }}
spec:
{{- if .Values.api.affinity }}
affinity: {{- toYaml .Values.api.affinity | nindent 8 }}
@@ -54,34 +58,38 @@ spec:
tolerations: {{- toYaml . | nindent 8 }}
{{- end }}
initContainers:
- - name: init-postgresql
+ - name: init-database
image: busybox:1.31.0
command:
- /bin/sh
- -ec
- |
- while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do
+ while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
counter=$((counter+1))
if [ $counter == 5 ]; then
- echo "Error: Couldn't connect to postgresql."
+ echo "Error: Couldn't connect to database."
exit 1
fi
- echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter."
+ echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
sleep 60
done
env:
- - name: POSTGRESQL_HOST
+ - name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- - name: POSTGRESQL_PORT
+ - name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
- value: {{ .Values.externalDatabase.port }}
+ value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
+ {{- if .Values.image.pullSecrets }}
+ imagePullSecrets:
+ - name: {{ include "dolphinscheduler.image.pullSecrets" . }}
+ {{- end }}
containers:
- name: {{ include "dolphinscheduler.fullname" . }}-api
image: {{ include "dolphinscheduler.image.repository" . | quote }}
@@ -92,27 +100,41 @@ spec:
name: tcp-port
imagePullPolicy: {{ .Values.image.pullPolicy }}
env:
+ - name: DOLPHINSCHEDULER_OPTS
+ value: {{ default "" .Values.api.jvmOptions }}
- name: TZ
value: {{ .Values.timezone }}
- - name: POSTGRESQL_HOST
+ - name: DATABASE_TYPE
+ {{- if .Values.postgresql.enabled }}
+ value: "postgresql"
+ {{- else }}
+ value: {{ .Values.externalDatabase.type | quote }}
+ {{- end }}
+ - name: DATABASE_DRIVER
+ {{- if .Values.postgresql.enabled }}
+ value: "org.postgresql.Driver"
+ {{- else }}
+ value: {{ .Values.externalDatabase.driver | quote }}
+ {{- end }}
+ - name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- - name: POSTGRESQL_PORT
+ - name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
- value: {{ .Values.externalDatabase.port }}
+ value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
- - name: POSTGRESQL_USERNAME
+ - name: DATABASE_USERNAME
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlUsername }}
{{- else }}
value: {{ .Values.externalDatabase.username | quote }}
{{- end }}
- - name: POSTGRESQL_PASSWORD
+ - name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
{{- if .Values.postgresql.enabled }}
@@ -122,18 +144,63 @@ spec:
name: {{ printf "%s-%s" .Release.Name "externaldb" }}
key: db-password
{{- end }}
- - name: POSTGRESQL_DATABASE
+ - name: DATABASE_DATABASE
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlDatabase }}
{{- else }}
value: {{ .Values.externalDatabase.database | quote }}
{{- end }}
+ - name: DATABASE_PARAMS
+ {{- if .Values.postgresql.enabled }}
+ value: "characterEncoding=utf8"
+ {{- else }}
+ value: {{ .Values.externalDatabase.params | quote }}
+ {{- end }}
- name: ZOOKEEPER_QUORUM
{{- if .Values.zookeeper.enabled }}
value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}"
{{- else }}
value: {{ .Values.externalZookeeper.zookeeperQuorum }}
{{- end }}
+ - name: RESOURCE_STORAGE_TYPE
+ valueFrom:
+ configMapKeyRef:
+ key: RESOURCE_STORAGE_TYPE
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ - name: RESOURCE_UPLOAD_PATH
+ valueFrom:
+ configMapKeyRef:
+ key: RESOURCE_UPLOAD_PATH
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ - name: FS_DEFAULT_FS
+ valueFrom:
+ configMapKeyRef:
+ key: FS_DEFAULT_FS
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ - name: FS_S3A_ENDPOINT
+ valueFrom:
+ configMapKeyRef:
+ key: FS_S3A_ENDPOINT
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ - name: FS_S3A_ACCESS_KEY
+ valueFrom:
+ configMapKeyRef:
+ key: FS_S3A_ACCESS_KEY
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ - name: FS_S3A_SECRET_KEY
+ valueFrom:
+ configMapKeyRef:
+ key: FS_S3A_SECRET_KEY
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ {{- if .Values.api.resources }}
+ resources:
+ limits:
+ memory: {{ .Values.api.resources.limits.memory | quote }}
+ cpu: {{ .Values.api.resources.limits.cpu | quote }}
+ requests:
+ memory: {{ .Values.api.resources.requests.memory | quote }}
+ cpu: {{ .Values.api.resources.requests.cpu | quote }}
+ {{- end }}
{{- if .Values.api.livenessProbe.enabled }}
livenessProbe:
tcpSocket:
diff --git a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml
index aea09f107f..04b9408d9f 100644
--- a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml
+++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml
@@ -43,6 +43,10 @@ spec:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: frontend
+ {{- if .Values.alert.annotations }}
+ annotations:
+ {{- toYaml .Values.alert.annotations | nindent 8 }}
+ {{- end }}
spec:
{{- if .Values.frontend.affinity }}
affinity: {{- toYaml .Values.frontend.affinity | nindent 8 }}
@@ -53,6 +57,10 @@ spec:
{{- if .Values.frontend.tolerations }}
tolerations: {{- toYaml . | nindent 8 }}
{{- end }}
+ {{- if .Values.image.pullSecrets }}
+ imagePullSecrets:
+ - name: {{ include "dolphinscheduler.image.pullSecrets" . }}
+ {{- end }}
containers:
- name: {{ include "dolphinscheduler.fullname" . }}-frontend
image: {{ include "dolphinscheduler.image.repository" . | quote }}
@@ -69,6 +77,15 @@ spec:
value: '{{ include "dolphinscheduler.fullname" . }}-api'
- name: FRONTEND_API_SERVER_PORT
value: "12345"
+ {{- if .Values.frontend.resources }}
+ resources:
+ limits:
+ memory: {{ .Values.frontend.resources.limits.memory | quote }}
+ cpu: {{ .Values.frontend.resources.limits.cpu | quote }}
+ requests:
+ memory: {{ .Values.frontend.resources.requests.memory | quote }}
+ cpu: {{ .Values.frontend.resources.requests.cpu | quote }}
+ {{- end }}
{{- if .Values.frontend.livenessProbe.enabled }}
livenessProbe:
tcpSocket:
diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
index b142fac4fe..0949127dda 100644
--- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
+++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
@@ -40,6 +40,10 @@ spec:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: master
+ {{- if .Values.alert.annotations }}
+ annotations:
+ {{- toYaml .Values.alert.annotations | nindent 8 }}
+ {{- end }}
spec:
{{- if .Values.master.affinity }}
affinity: {{- toYaml .Values.master.affinity | nindent 8 }}
@@ -75,34 +79,38 @@ spec:
{{- else }}
value: {{ .Values.externalZookeeper.zookeeperQuorum }}
{{- end }}
- - name: init-postgresql
+ - name: init-database
image: busybox:1.31.0
command:
- /bin/sh
- -ec
- |
- while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do
+ while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
counter=$((counter+1))
if [ $counter == 5 ]; then
- echo "Error: Couldn't connect to postgresql."
+ echo "Error: Couldn't connect to database."
exit 1
fi
- echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter."
+ echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
sleep 60
done
env:
- - name: POSTGRESQL_HOST
+ - name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- - name: POSTGRESQL_PORT
+ - name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
- value: {{ .Values.externalDatabase.port }}
+ value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
+ {{- if .Values.image.pullSecrets }}
+ imagePullSecrets:
+ - name: {{ include "dolphinscheduler.image.pullSecrets" . }}
+ {{- end }}
containers:
- name: {{ include "dolphinscheduler.fullname" . }}-master
image: {{ include "dolphinscheduler.image.repository" . | quote }}
@@ -113,6 +121,8 @@ spec:
name: "master-port"
imagePullPolicy: {{ .Values.image.pullPolicy }}
env:
+ - name: DOLPHINSCHEDULER_OPTS
+ value: {{ default "" .Values.master.jvmOptions }}
- name: TZ
value: {{ .Values.timezone }}
- name: MASTER_EXEC_THREADS
@@ -160,25 +170,37 @@ spec:
configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-master
key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
- - name: POSTGRESQL_HOST
+ - name: DATABASE_TYPE
+ {{- if .Values.postgresql.enabled }}
+ value: "postgresql"
+ {{- else }}
+ value: {{ .Values.externalDatabase.type | quote }}
+ {{- end }}
+ - name: DATABASE_DRIVER
+ {{- if .Values.postgresql.enabled }}
+ value: "org.postgresql.Driver"
+ {{- else }}
+ value: {{ .Values.externalDatabase.driver | quote }}
+ {{- end }}
+ - name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- - name: POSTGRESQL_PORT
+ - name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
- value: {{ .Values.externalDatabase.port }}
+ value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
- - name: POSTGRESQL_USERNAME
+ - name: DATABASE_USERNAME
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlUsername }}
{{- else }}
value: {{ .Values.externalDatabase.username | quote }}
{{- end }}
- - name: POSTGRESQL_PASSWORD
+ - name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
{{- if .Values.postgresql.enabled }}
@@ -188,18 +210,63 @@ spec:
name: {{ printf "%s-%s" .Release.Name "externaldb" }}
key: db-password
{{- end }}
- - name: POSTGRESQL_DATABASE
+ - name: DATABASE_DATABASE
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlDatabase }}
{{- else }}
value: {{ .Values.externalDatabase.database | quote }}
{{- end }}
+ - name: DATABASE_PARAMS
+ {{- if .Values.postgresql.enabled }}
+ value: "characterEncoding=utf8"
+ {{- else }}
+ value: {{ .Values.externalDatabase.params | quote }}
+ {{- end }}
- name: ZOOKEEPER_QUORUM
{{- if .Values.zookeeper.enabled }}
value: {{ template "dolphinscheduler.zookeeper.quorum" . }}
{{- else }}
value: {{ .Values.externalZookeeper.zookeeperQuorum }}
{{- end }}
+ - name: RESOURCE_STORAGE_TYPE
+ valueFrom:
+ configMapKeyRef:
+ key: RESOURCE_STORAGE_TYPE
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ - name: RESOURCE_UPLOAD_PATH
+ valueFrom:
+ configMapKeyRef:
+ key: RESOURCE_UPLOAD_PATH
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ - name: FS_DEFAULT_FS
+ valueFrom:
+ configMapKeyRef:
+ key: FS_DEFAULT_FS
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ - name: FS_S3A_ENDPOINT
+ valueFrom:
+ configMapKeyRef:
+ key: FS_S3A_ENDPOINT
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ - name: FS_S3A_ACCESS_KEY
+ valueFrom:
+ configMapKeyRef:
+ key: FS_S3A_ACCESS_KEY
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ - name: FS_S3A_SECRET_KEY
+ valueFrom:
+ configMapKeyRef:
+ key: FS_S3A_SECRET_KEY
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ {{- if .Values.master.resources }}
+ resources:
+ limits:
+ memory: {{ .Values.master.resources.limits.memory | quote }}
+ cpu: {{ .Values.master.resources.limits.cpu | quote }}
+ requests:
+ memory: {{ .Values.master.resources.requests.memory | quote }}
+ cpu: {{ .Values.master.resources.requests.cpu | quote }}
+ {{- end }}
{{- if .Values.master.livenessProbe.enabled }}
livenessProbe:
exec:
diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
index 198cef43b6..097f8d8580 100644
--- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
+++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
@@ -40,6 +40,10 @@ spec:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/component: worker
+ {{- if .Values.alert.annotations }}
+ annotations:
+ {{- toYaml .Values.alert.annotations | nindent 8 }}
+ {{- end }}
spec:
{{- if .Values.worker.affinity }}
affinity: {{- toYaml .Values.worker.affinity | nindent 8 }}
@@ -75,34 +79,38 @@ spec:
{{- else }}
value: {{ .Values.externalZookeeper.zookeeperQuorum }}
{{- end }}
- - name: init-postgresql
+ - name: init-database
image: busybox:1.31.0
command:
- /bin/sh
- -ec
- |
- while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do
+ while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
counter=$((counter+1))
if [ $counter == 5 ]; then
- echo "Error: Couldn't connect to postgresql."
+ echo "Error: Couldn't connect to database."
exit 1
fi
- echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter."
+ echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter."
sleep 60
done
env:
- - name: POSTGRESQL_HOST
+ - name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- - name: POSTGRESQL_PORT
+ - name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
- value: {{ .Values.externalDatabase.port }}
+ value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
+ {{- if .Values.image.pullSecrets }}
+ imagePullSecrets:
+ - name: {{ include "dolphinscheduler.image.pullSecrets" . }}
+ {{- end }}
containers:
- name: {{ include "dolphinscheduler.fullname" . }}-worker
image: {{ include "dolphinscheduler.image.repository" . | quote }}
@@ -115,6 +123,8 @@ spec:
name: "logs-port"
imagePullPolicy: {{ .Values.image.pullPolicy }}
env:
+ - name: DOLPHINSCHEDULER_OPTS
+ value: {{ default "" .Values.worker.jvmOptions }}
- name: TZ
value: {{ .Values.timezone }}
- name: WORKER_EXEC_THREADS
@@ -157,25 +167,37 @@ spec:
configMapKeyRef:
name: {{ include "dolphinscheduler.fullname" . }}-master
key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH
- - name: POSTGRESQL_HOST
+ - name: DATABASE_TYPE
+ {{- if .Values.postgresql.enabled }}
+ value: "postgresql"
+ {{- else }}
+ value: {{ .Values.externalDatabase.type | quote }}
+ {{- end }}
+ - name: DATABASE_DRIVER
+ {{- if .Values.postgresql.enabled }}
+ value: "org.postgresql.Driver"
+ {{- else }}
+ value: {{ .Values.externalDatabase.driver | quote }}
+ {{- end }}
+ - name: DATABASE_HOST
{{- if .Values.postgresql.enabled }}
value: {{ template "dolphinscheduler.postgresql.fullname" . }}
{{- else }}
value: {{ .Values.externalDatabase.host | quote }}
{{- end }}
- - name: POSTGRESQL_PORT
+ - name: DATABASE_PORT
{{- if .Values.postgresql.enabled }}
value: "5432"
{{- else }}
- value: {{ .Values.externalDatabase.port }}
+ value: {{ .Values.externalDatabase.port | quote }}
{{- end }}
- - name: POSTGRESQL_USERNAME
+ - name: DATABASE_USERNAME
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlUsername }}
{{- else }}
value: {{ .Values.externalDatabase.username | quote }}
{{- end }}
- - name: POSTGRESQL_PASSWORD
+ - name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
{{- if .Values.postgresql.enabled }}
@@ -185,18 +207,133 @@ spec:
name: {{ printf "%s-%s" .Release.Name "externaldb" }}
key: db-password
{{- end }}
- - name: POSTGRESQL_DATABASE
+ - name: DATABASE_DATABASE
{{- if .Values.postgresql.enabled }}
value: {{ .Values.postgresql.postgresqlDatabase }}
{{- else }}
value: {{ .Values.externalDatabase.database | quote }}
{{- end }}
+ - name: DATABASE_PARAMS
+ {{- if .Values.postgresql.enabled }}
+ value: "characterEncoding=utf8"
+ {{- else }}
+ value: {{ .Values.externalDatabase.params | quote }}
+ {{- end }}
- name: ZOOKEEPER_QUORUM
{{- if .Values.zookeeper.enabled }}
value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}"
{{- else }}
value: {{ .Values.externalZookeeper.zookeeperQuorum }}
{{- end }}
+ - name: RESOURCE_STORAGE_TYPE
+ valueFrom:
+ configMapKeyRef:
+ key: RESOURCE_STORAGE_TYPE
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ - name: RESOURCE_UPLOAD_PATH
+ valueFrom:
+ configMapKeyRef:
+ key: RESOURCE_UPLOAD_PATH
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ - name: FS_DEFAULT_FS
+ valueFrom:
+ configMapKeyRef:
+ key: FS_DEFAULT_FS
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ - name: FS_S3A_ENDPOINT
+ valueFrom:
+ configMapKeyRef:
+ key: FS_S3A_ENDPOINT
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ - name: FS_S3A_ACCESS_KEY
+ valueFrom:
+ configMapKeyRef:
+ key: FS_S3A_ACCESS_KEY
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ - name: FS_S3A_SECRET_KEY
+ valueFrom:
+ configMapKeyRef:
+ key: FS_S3A_SECRET_KEY
+ name: {{ include "dolphinscheduler.fullname" . }}-common
+ - name: XLS_FILE_PATH
+ valueFrom:
+ configMapKeyRef:
+ key: XLS_FILE_PATH
+ name: {{ include "dolphinscheduler.fullname" . }}-alert
+ - name: MAIL_SERVER_HOST
+ valueFrom:
+ configMapKeyRef:
+ key: MAIL_SERVER_HOST
+ name: {{ include "dolphinscheduler.fullname" . }}-alert
+ - name: MAIL_SERVER_PORT
+ valueFrom:
+ configMapKeyRef:
+ key: MAIL_SERVER_PORT
+ name: {{ include "dolphinscheduler.fullname" . }}-alert
+ - name: MAIL_SENDER
+ valueFrom:
+ configMapKeyRef:
+ key: MAIL_SENDER
+ name: {{ include "dolphinscheduler.fullname" . }}-alert
+ - name: MAIL_USER
+ valueFrom:
+ configMapKeyRef:
+ key: MAIL_USER
+ name: {{ include "dolphinscheduler.fullname" . }}-alert
+ - name: MAIL_PASSWD
+ valueFrom:
+ configMapKeyRef:
+ key: MAIL_PASSWD
+ name: {{ include "dolphinscheduler.fullname" . }}-alert
+ - name: MAIL_SMTP_STARTTLS_ENABLE
+ valueFrom:
+ configMapKeyRef:
+ key: MAIL_SMTP_STARTTLS_ENABLE
+ name: {{ include "dolphinscheduler.fullname" . }}-alert
+ - name: MAIL_SMTP_SSL_ENABLE
+ valueFrom:
+ configMapKeyRef:
+ key: MAIL_SMTP_SSL_ENABLE
+ name: {{ include "dolphinscheduler.fullname" . }}-alert
+ - name: MAIL_SMTP_SSL_TRUST
+ valueFrom:
+ configMapKeyRef:
+ key: MAIL_SMTP_SSL_TRUST
+ name: {{ include "dolphinscheduler.fullname" . }}-alert
+ - name: ENTERPRISE_WECHAT_ENABLE
+ valueFrom:
+ configMapKeyRef:
+ key: ENTERPRISE_WECHAT_ENABLE
+ name: {{ include "dolphinscheduler.fullname" . }}-alert
+ - name: ENTERPRISE_WECHAT_CORP_ID
+ valueFrom:
+ configMapKeyRef:
+ key: ENTERPRISE_WECHAT_CORP_ID
+ name: {{ include "dolphinscheduler.fullname" . }}-alert
+ - name: ENTERPRISE_WECHAT_SECRET
+ valueFrom:
+ configMapKeyRef:
+ key: ENTERPRISE_WECHAT_SECRET
+ name: {{ include "dolphinscheduler.fullname" . }}-alert
+ - name: ENTERPRISE_WECHAT_AGENT_ID
+ valueFrom:
+ configMapKeyRef:
+ key: ENTERPRISE_WECHAT_AGENT_ID
+ name: {{ include "dolphinscheduler.fullname" . }}-alert
+ - name: ENTERPRISE_WECHAT_USERS
+ valueFrom:
+ configMapKeyRef:
+ key: ENTERPRISE_WECHAT_USERS
+ name: {{ include "dolphinscheduler.fullname" . }}-alert
+ {{- if .Values.worker.resources }}
+ resources:
+ limits:
+ memory: {{ .Values.worker.resources.limits.memory | quote }}
+ cpu: {{ .Values.worker.resources.limits.cpu | quote }}
+ requests:
+ memory: {{ .Values.worker.resources.requests.memory | quote }}
+ cpu: {{ .Values.worker.resources.requests.cpu | quote }}
+ {{- end }}
{{- if .Values.worker.livenessProbe.enabled }}
livenessProbe:
exec:
diff --git a/docker/kubernetes/dolphinscheduler/values.yaml b/docker/kubernetes/dolphinscheduler/values.yaml
index 3cb35c19e0..8acb1d326a 100644
--- a/docker/kubernetes/dolphinscheduler/values.yaml
+++ b/docker/kubernetes/dolphinscheduler/values.yaml
@@ -29,10 +29,9 @@ image:
repository: "dolphinscheduler"
tag: "latest"
pullPolicy: "IfNotPresent"
+ pullSecrets: []
-imagePullSecrets: []
-
-# If not exists external postgresql, by default, Dolphinscheduler's database will use it.
+# If not exists external database, by default, Dolphinscheduler's database will use it.
postgresql:
enabled: true
postgresqlUsername: "root"
@@ -43,21 +42,24 @@ postgresql:
size: "20Gi"
storageClass: "-"
-# If exists external postgresql, and set postgresql.enable value to false.
-# If postgresql.enable is false, Dolphinscheduler's database will use it.
+# If exists external database, and set postgresql.enable value to false.
+# external database will be used, otherwise Dolphinscheduler's database will be used.
externalDatabase:
+ type: "postgresql"
+ driver: "org.postgresql.Driver"
host: "localhost"
port: "5432"
username: "root"
password: "root"
database: "dolphinscheduler"
+ # multi params should join with & char
+ params: "characterEncoding=utf8"
# If not exists external zookeeper, by default, Dolphinscheduler's zookeeper will use it.
zookeeper:
enabled: true
taskQueue: "zookeeper"
- config:
- ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
+ config: null
service:
port: "2181"
persistence:
@@ -70,6 +72,18 @@ zookeeper:
externalZookeeper:
taskQueue: "zookeeper"
zookeeperQuorum: "127.0.0.1:2181"
+ zookeeperRoot: "/dolphinscheduler"
+
+common:
+ configmap:
+ DOLPHINSCHEDULER_ENV_PATH: "/tmp/dolphinscheduler/env"
+ DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler/files"
+ RESOURCE_STORAGE_TYPE: "NONE"
+ RESOURCE_UPLOAD_PATH: "/ds"
+ FS_DEFAULT_FS: "s3a://xxxx"
+ FS_S3A_ENDPOINT: "s3.xxx.amazonaws.com"
+ FS_S3A_ACCESS_KEY: "xxxxxxx"
+ FS_S3A_SECRET_KEY: "xxxxxxx"
master:
podManagementPolicy: "Parallel"
@@ -85,6 +99,18 @@ master:
# If specified, the pod's scheduling constraints.
# More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
affinity: {}
+ # The jvm options for java instance startup
+ jvmOptions: ""
+ resources: {}
+ # limits:
+ # memory: "18Gi"
+ # cpu: "4"
+ # requests:
+ # memory: "2Gi"
+ # cpu: "500m"
+ # You can use annotations to attach arbitrary non-identifying metadata to objects.
+ # Clients such as tools and libraries can retrieve this metadata.
+ annotations: {}
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
configmap:
@@ -137,6 +163,18 @@ worker:
# If specified, the pod's scheduling constraints.
# More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
affinity: {}
+ # The jvm options for java instance startup
+ jvmOptions: ""
+ resources: {}
+ # limits:
+ # memory: "18Gi"
+ # cpu: "4"
+ # requests:
+ # memory: "2Gi"
+ # cpu: "500m"
+ # You can use annotations to attach arbitrary non-identifying metadata to objects.
+ # Clients such as tools and libraries can retrieve this metadata.
+ annotations: {}
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
livenessProbe:
@@ -213,9 +251,22 @@ alert:
# If specified, the pod's scheduling constraints.
# More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
affinity: {}
+ # The jvm options for java instance startup
+ jvmOptions: ""
+ resources: {}
+ # limits:
+ # memory: "4Gi"
+ # cpu: "1"
+ # requests:
+ # memory: "2Gi"
+ # cpu: "500m"
+ # You can use annotations to attach arbitrary non-identifying metadata to objects.
+ # Clients such as tools and libraries can retrieve this metadata.
+ annotations: {}
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
configmap:
+ ALERT_PLUGIN_DIR: "/opt/dolphinscheduler/alert/plugin"
XLS_FILE_PATH: "/tmp/xls"
MAIL_SERVER_HOST: ""
MAIL_SERVER_PORT: ""
@@ -275,6 +326,18 @@ api:
# If specified, the pod's scheduling constraints.
# More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
affinity: {}
+ # The jvm options for java instance startup
+ jvmOptions: ""
+ resources: {}
+ # limits:
+ # memory: "4Gi"
+ # cpu: "2"
+ # requests:
+ # memory: "2Gi"
+ # cpu: "500m"
+ # You can use annotations to attach arbitrary non-identifying metadata to objects.
+ # Clients such as tools and libraries can retrieve this metadata.
+ annotations: {}
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
livenessProbe:
@@ -322,6 +385,16 @@ frontend:
# If specified, the pod's scheduling constraints.
# More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core
affinity: {}
+ resources: {}
+ # limits:
+ # memory: "256Mi"
+ # cpu: "1"
+ # requests:
+ # memory: "256Mi"
+ # cpu: "500m"
+ # You can use annotations to attach arbitrary non-identifying metadata to objects.
+ # Clients such as tools and libraries can retrieve this metadata.
+ annotations: {}
## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated.
## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
livenessProbe:
diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java
index 38dc38b4a8..36f903c25d 100644
--- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java
+++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java
@@ -48,8 +48,8 @@ public class EnterpriseWeChatUtils {
private static final String ENTERPRISE_WE_CHAT_TOKEN_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TOKEN_URL);
private static final String ENTERPRISE_WE_CHAT_TOKEN_URL_REPLACE = ENTERPRISE_WE_CHAT_TOKEN_URL == null ? null : ENTERPRISE_WE_CHAT_TOKEN_URL
- .replaceAll("\\$corpId", ENTERPRISE_WE_CHAT_CORP_ID)
- .replaceAll("\\$secret", ENTERPRISE_WE_CHAT_SECRET);
+ .replaceAll("\\{corpId\\}", ENTERPRISE_WE_CHAT_CORP_ID)
+ .replaceAll("\\{secret\\}", ENTERPRISE_WE_CHAT_SECRET);
private static final String ENTERPRISE_WE_CHAT_PUSH_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_PUSH_URL);
@@ -120,9 +120,9 @@ public class EnterpriseWeChatUtils {
* @return Enterprise WeChat send message
*/
public static String makeTeamSendMsg(String toParty, String agentId, String msg) {
- return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\$toParty", toParty)
- .replaceAll("\\$agentId", agentId)
- .replaceAll("\\$msg", msg);
+ return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty\\}", toParty)
+ .replaceAll("\\{agentId\\}", agentId)
+ .replaceAll("\\{msg\\}", msg);
}
/**
@@ -135,9 +135,9 @@ public class EnterpriseWeChatUtils {
*/
public static String makeTeamSendMsg(Collection toParty, String agentId, String msg) {
String listParty = FuncUtils.mkString(toParty, "|");
- return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\$toParty", listParty)
- .replaceAll("\\$agentId", agentId)
- .replaceAll("\\$msg", msg);
+ return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty\\}", listParty)
+ .replaceAll("\\{agentId\\}", agentId)
+ .replaceAll("\\{msg\\}", msg);
}
/**
@@ -149,9 +149,9 @@ public class EnterpriseWeChatUtils {
* @return Enterprise WeChat send message
*/
public static String makeUserSendMsg(String toUser, String agentId, String msg) {
- return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\$toUser", toUser)
- .replaceAll("\\$agentId", agentId)
- .replaceAll("\\$msg", msg);
+ return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\{toUser\\}", toUser)
+ .replaceAll("\\{agentId\\}", agentId)
+ .replaceAll("\\{msg\\}", msg);
}
/**
@@ -164,9 +164,9 @@ public class EnterpriseWeChatUtils {
*/
public static String makeUserSendMsg(Collection toUser, String agentId, String msg) {
String listUser = FuncUtils.mkString(toUser, "|");
- return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\$toUser", listUser)
- .replaceAll("\\$agentId", agentId)
- .replaceAll("\\$msg", msg);
+ return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\{toUser\\}", listUser)
+ .replaceAll("\\{agentId\\}", agentId)
+ .replaceAll("\\{msg\\}", msg);
}
/**
@@ -179,7 +179,7 @@ public class EnterpriseWeChatUtils {
* @throws IOException the IOException
*/
public static String sendEnterpriseWeChat(String charset, String data, String token) throws IOException {
- String enterpriseWeChatPushUrlReplace = ENTERPRISE_WE_CHAT_PUSH_URL.replaceAll("\\$token", token);
+ String enterpriseWeChatPushUrlReplace = ENTERPRISE_WE_CHAT_PUSH_URL.replaceAll("\\{token\\}", token);
CloseableHttpClient httpClient = HttpClients.createDefault();
try {
diff --git a/dolphinscheduler-alert/src/main/resources/alert.properties b/dolphinscheduler-alert/src/main/resources/alert.properties
index 4e48e7d0e1..6eb701841c 100644
--- a/dolphinscheduler-alert/src/main/resources/alert.properties
+++ b/dolphinscheduler-alert/src/main/resources/alert.properties
@@ -41,10 +41,10 @@ enterprise.wechat.enable=false
#enterprise.wechat.secret=xxxxxxx
#enterprise.wechat.agent.id=xxxxxxx
#enterprise.wechat.users=xxxxxxx
-#enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret
-#enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token
-#enterprise.wechat.team.send.msg={\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"}
-#enterprise.wechat.user.send.msg={\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}}
+#enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid={corpId}&corpsecret={secret}
+#enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={token}
+#enterprise.wechat.team.send.msg={\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"}
+#enterprise.wechat.user.send.msg={\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}}
plugin.dir=/Users/xx/your/path/to/plugin/dir
diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java
index 01b81101fd..7b6cdd013b 100644
--- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java
+++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java
@@ -55,8 +55,8 @@ public class EnterpriseWeChatUtilsTest {
private static final String enterpriseWechatUsers="LiGang,journey";
private static final String msg = "hello world";
- private static final String enterpriseWechatTeamSendMsg = "{\\\"toparty\\\":\\\"$toParty\\\",\\\"agentid\\\":\\\"$agentId\\\",\\\"msgtype\\\":\\\"text\\\",\\\"text\\\":{\\\"content\\\":\\\"$msg\\\"},\\\"safe\\\":\\\"0\\\"}";
- private static final String enterpriseWechatUserSendMsg = "{\\\"touser\\\":\\\"$toUser\\\",\\\"agentid\\\":\\\"$agentId\\\",\\\"msgtype\\\":\\\"markdown\\\",\\\"markdown\\\":{\\\"content\\\":\\\"$msg\\\"}}";
+ private static final String enterpriseWechatTeamSendMsg = "{\\\"toparty\\\":\\\"{toParty}\\\",\\\"agentid\\\":\\\"{agentId}\\\",\\\"msgtype\\\":\\\"text\\\",\\\"text\\\":{\\\"content\\\":\\\"{msg}\\\"},\\\"safe\\\":\\\"0\\\"}";
+ private static final String enterpriseWechatUserSendMsg = "{\\\"touser\\\":\\\"{toUser}\\\",\\\"agentid\\\":\\\"{agentId}\\\",\\\"msgtype\\\":\\\"markdown\\\",\\\"markdown\\\":{\\\"content\\\":\\\"{msg}\\\"}}";
@Before
public void init(){
diff --git a/dolphinscheduler-ui/package.json b/dolphinscheduler-ui/package.json
index 1f5170637e..a5642c8f97 100644
--- a/dolphinscheduler-ui/package.json
+++ b/dolphinscheduler-ui/package.json
@@ -63,7 +63,7 @@
"html-loader": "^0.5.5",
"html-webpack-plugin": "^3.2.0",
"mini-css-extract-plugin": "^0.8.2",
- "node-sass": "^4.14.0",
+ "node-sass": "^4.14.1",
"postcss-loader": "^3.0.0",
"progress-bar-webpack-plugin": "^1.12.1",
"rimraf": "^2.6.2",
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/startingParam/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/startingParam/index.vue
index bea5d92227..516f03bc2f 100644
--- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/startingParam/index.vue
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/startingParam/index.vue
@@ -76,14 +76,9 @@
return '-'
},
_getNotifyGroupList () {
- let notifyGroupListS = _.cloneDeep(this.store.state.dag.notifyGroupListS) || []
- if (!notifyGroupListS.length) {
this.store.dispatch('dag/getNotifyGroupList').then(res => {
this.notifyGroupList = res
})
- } else {
- this.notifyGroupList = notifyGroupListS
- }
},
_getWorkerGroupList () {
let stateWorkerGroupsList = this.store.state.security.workerGroupsListAll || []
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue
index e47d8c757f..87c0f62cdb 100644
--- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue
@@ -243,16 +243,10 @@
},
_getNotifyGroupList () {
return new Promise((resolve, reject) => {
- let notifyGroupListS = _.cloneDeep(this.store.state.dag.notifyGroupListS) || []
- if (!notifyGroupListS.length) {
this.store.dispatch('dag/getNotifyGroupList').then(res => {
this.notifyGroupList = res
resolve()
})
- } else {
- this.notifyGroupList = notifyGroupListS
- resolve()
- }
})
},
_getReceiver () {
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue
index 82a82dec45..65b9176c18 100644
--- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue
@@ -280,8 +280,6 @@
_getNotifyGroupList () {
return new Promise((resolve, reject) => {
- let notifyGroupListS = _.cloneDeep(this.store.state.dag.notifyGroupListS) || []
- if (!notifyGroupListS.length) {
this.store.dispatch('dag/getNotifyGroupList').then(res => {
this.notifyGroupList = res
if (this.notifyGroupList.length) {
@@ -290,10 +288,6 @@
reject(new Error(0))
}
})
- } else {
- this.notifyGroupList = notifyGroupListS
- resolve()
- }
})
},
ok () {
diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh
index b0437a30c2..4ffd7fe4c2 100644
--- a/script/dolphinscheduler-daemon.sh
+++ b/script/dolphinscheduler-daemon.sh
@@ -46,7 +46,7 @@ export DOLPHINSCHEDULER_LOG_DIR=$DOLPHINSCHEDULER_HOME/logs
export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf
export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/*
-export DOLPHINSCHEDULER_OPTS="-server -Xmx16g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
+export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-"-server -Xmx16g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"}
export STOP_TIMEOUT=5
if [ ! -d "$DOLPHINSCHEDULER_LOG_DIR" ]; then