From b6a0d55949d8379ff5649993361aed858a98ca13 Mon Sep 17 00:00:00 2001 From: rockxsj Date: Thu, 16 Jul 2020 16:36:29 +0800 Subject: [PATCH] fix and update k8s deploy (#3215) * Modify the AWS S3 request encryption method to V4. * * update Dockerfile to add mysql-client for mysql database service test * update wechat alert config replace placeholder from to {xx} because of the will be treat as env variable * fix unit test of EnterpriseWeChatUtilsTest * update values.yaml default config for helm deploy DS cluster * update values.yaml default config for custom install * update templates of helm for custom install * update some name of env varibale, for example change from POSTGRESQL_HOST to DATABASE_HOST * delele the redundancy requirements.yaml for heml * update README* in the docker dir and subdir * update startup scripts for support mysql as backend database * * fix the ENV name wrong in docker-compose.yml and docker-stack.yml * remove the didn't need port mapping in postgresql & zookeeper & dolphinscheduler-api * * keep expose the port of postgresql & zk & api because of unit test needed * * remove mysql-client package from dockerfile Co-authored-by: rockxsj --- .../DOLPHIN/1.2.1/package/scripts/params.py | 4 +- .../DOLPHIN/1.3.0/package/scripts/params.py | 4 +- docker/build/Dockerfile | 2 +- docker/build/README.md | 66 ++++--- docker/build/README_zh_CN.md | 66 ++++--- .../dolphinscheduler/alert.properties.tpl | 14 +- .../dolphinscheduler/common.properties.tpl | 12 +- .../datasource.properties.tpl | 15 +- .../logback/logback-alert.xml | 2 +- .../dolphinscheduler/zookeeper.properties.tpl | 2 +- docker/build/startup-init-conf.sh | 23 ++- docker/build/startup.sh | 40 +++-- docker/docker-swarm/docker-compose.yml | 40 ++--- docker/docker-swarm/docker-stack.yml | 52 +++--- docker/kubernetes/dolphinscheduler/Chart.yaml | 2 +- docker/kubernetes/dolphinscheduler/README.md | 42 ++++- .../dolphinscheduler/requirements.yaml | 25 --- .../dolphinscheduler/templates/NOTES.txt | 13 -- .../dolphinscheduler/templates/_helpers.tpl | 8 + .../configmap-dolphinscheduler-alert.yaml | 1 + .../configmap-dolphinscheduler-common.yaml | 35 ++++ .../deployment-dolphinscheduler-alert.yaml | 98 +++++++++-- .../deployment-dolphinscheduler-api.yaml | 93 ++++++++-- .../deployment-dolphinscheduler-frontend.yaml | 17 ++ .../statefulset-dolphinscheduler-master.yaml | 93 ++++++++-- .../statefulset-dolphinscheduler-worker.yaml | 163 ++++++++++++++++-- .../kubernetes/dolphinscheduler/values.yaml | 87 +++++++++- .../alert/utils/EnterpriseWeChatUtils.java | 30 ++-- .../src/main/resources/alert.properties | 8 +- .../utils/EnterpriseWeChatUtilsTest.java | 4 +- dolphinscheduler-ui/package.json | 2 +- script/dolphinscheduler-daemon.sh | 2 +- 32 files changed, 795 insertions(+), 270 deletions(-) delete mode 100644 docker/kubernetes/dolphinscheduler/requirements.yaml create mode 100644 docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py b/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py index 93b3249614..049b2cf3ae 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py +++ b/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py @@ -76,8 +76,8 @@ else: dolphin_alert_map = {} wechat_push_url = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token' wechat_token_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret' -wechat_team_send_msg = '{\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"}' -wechat_user_send_msg = '{\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}}' +wechat_team_send_msg = '{\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"}' +wechat_user_send_msg = '{\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}}' dolphin_alert_map['enterprise.wechat.push.ur'] = wechat_push_url dolphin_alert_map['enterprise.wechat.token.url'] = wechat_token_url diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/params.py b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/params.py index 3ca46f2182..5a9994f559 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/params.py +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/params.py @@ -77,8 +77,8 @@ else: dolphin_alert_map = {} wechat_push_url = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token' wechat_token_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret' -wechat_team_send_msg = '{\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"}' -wechat_user_send_msg = '{\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}}' +wechat_team_send_msg = '{\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"}' +wechat_user_send_msg = '{\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}}' dolphin_alert_config_map = config['configurations']['dolphin-alert'] diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile index bb69347797..d0f16d5d0d 100644 --- a/docker/build/Dockerfile +++ b/docker/build/Dockerfile @@ -42,7 +42,7 @@ ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ /opt/dolphinscheduler/ ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler -#4. install pg +#4. install database, if use mysql as your backend database, the `mysql-client` package should be installed RUN apk add postgresql postgresql-contrib #5. modify nginx RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \ diff --git a/docker/build/README.md b/docker/build/README.md index 5725ec56ce..bc516bc214 100644 --- a/docker/build/README.md +++ b/docker/build/README.md @@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org #### You can start a dolphinscheduler instance ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \ +-e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \ -p 8888:8888 \ dolphinscheduler all ``` @@ -25,14 +25,14 @@ The default postgres user `root`, postgres password `root` and database `dolphin The default zookeeper is created in the `startup.sh`. -#### Or via Environment Variables **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`POSTGRESQL_DATABASE`** **`ZOOKEEPER_QUORUM`** +#### Or via Environment Variables **`DATABASE_HOST`** **`DATABASE_PORT`** **`DATABASE_DATABASE`** **`ZOOKEEPER_QUORUM`** You can specify **existing postgres service**. Example: ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ -p 8888:8888 \ dolphinscheduler all ``` @@ -42,7 +42,7 @@ You can specify **existing zookeeper service**. Example: ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \ -p 8888:8888 \ dolphinscheduler all ``` @@ -56,8 +56,8 @@ You can start a standalone dolphinscheduler server. ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ dolphinscheduler master-server ``` @@ -66,8 +66,8 @@ dolphinscheduler master-server ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ dolphinscheduler worker-server ``` @@ -75,8 +75,8 @@ dolphinscheduler worker-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ -p 12345:12345 \ dolphinscheduler api-server ``` @@ -85,8 +85,8 @@ dolphinscheduler api-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ dolphinscheduler alert-server ``` @@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \ dolphinscheduler frontend ``` -**Note**: You must be specify `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server. +**Note**: You must be specify `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server. ## How to build a docker image @@ -124,33 +124,51 @@ Please read `./docker/build/hooks/build` `./docker/build/hooks/build.bat` script The Dolphin Scheduler image uses several environment variables which are easy to miss. While none of the variables are required, they may significantly aid you in using the image. -**`POSTGRESQL_HOST`** +**`DATABASE_TYPE`** -This environment variable sets the host for PostgreSQL. The default value is `127.0.0.1`. +This environment variable sets the type for database. The default value is `postgresql`. **Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. -**`POSTGRESQL_PORT`** +**`DATABASE_DRIVER`** -This environment variable sets the port for PostgreSQL. The default value is `5432`. +This environment variable sets the type for database. The default value is `org.postgresql.Driver`. + +**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. + +**`DATABASE_HOST`** + +This environment variable sets the host for database. The default value is `127.0.0.1`. + +**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. + +**`DATABASE_PORT`** + +This environment variable sets the port for database. The default value is `5432`. **Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. -**`POSTGRESQL_USERNAME`** +**`DATABASE_USERNAME`** + +This environment variable sets the username for database. The default value is `root`. + +**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. + +**`DATABASE_PASSWORD`** -This environment variable sets the username for PostgreSQL. The default value is `root`. +This environment variable sets the password for database. The default value is `root`. **Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. -**`POSTGRESQL_PASSWORD`** +**`DATABASE_DATABASE`** -This environment variable sets the password for PostgreSQL. The default value is `root`. +This environment variable sets the database for database. The default value is `dolphinscheduler`. **Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. -**`POSTGRESQL_DATABASE`** +**`DATABASE_PARAMS`** -This environment variable sets the database for PostgreSQL. The default value is `dolphinscheduler`. +This environment variable sets the database for database. The default value is `characterEncoding=utf8`. **Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md index 0605bef547..c2affc0691 100644 --- a/docker/build/README_zh_CN.md +++ b/docker/build/README_zh_CN.md @@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org #### 你可以运行一个dolphinscheduler实例 ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \ +-e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \ -p 8888:8888 \ dolphinscheduler all ``` @@ -25,14 +25,14 @@ dolphinscheduler all 同时，默认的`Zookeeper`也会在`startup.sh`脚本中被创建。 -#### 或者通过环境变量 **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`ZOOKEEPER_QUORUM`** 使用已存在的服务 +#### 或者通过环境变量 **`DATABASE_HOST`** **`DATABASE_PORT`** **`ZOOKEEPER_QUORUM`** 使用已存在的服务 你可以指定一个已经存在的 **`Postgres`** 服务. 如下: ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ -p 8888:8888 \ dolphinscheduler all ``` @@ -42,7 +42,7 @@ dolphinscheduler all ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \ -p 8888:8888 \ dolphinscheduler all ``` @@ -56,8 +56,8 @@ dolphinscheduler all ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ dolphinscheduler master-server ``` @@ -66,8 +66,8 @@ dolphinscheduler master-server ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ dolphinscheduler worker-server ``` @@ -75,8 +75,8 @@ dolphinscheduler worker-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ -p 12345:12345 \ dolphinscheduler api-server ``` @@ -85,8 +85,8 @@ dolphinscheduler api-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ dolphinscheduler alert-server ``` @@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \ dolphinscheduler frontend ``` -**注意**: 当你运行dolphinscheduler中的部分服务时，你必须指定这些环境变量 `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM`。 +**注意**: 当你运行dolphinscheduler中的部分服务时，你必须指定这些环境变量 `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM`。 ## 如何构建一个docker镜像 @@ -124,33 +124,51 @@ c:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些变量不是必须的，但是可以帮助你更容易配置镜像并根据你的需求定义相应的服务配置。 -**`POSTGRESQL_HOST`** +**`DATABASE_TYPE`** -配置`PostgreSQL`的`HOST`， 默认值 `127.0.0.1`。 +配置`database`的`TYPE`， 默认值 `postgresql`。 **注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 -**`POSTGRESQL_PORT`** +**`DATABASE_DRIVER`** -配置`PostgreSQL`的`PORT`， 默认值 `5432`。 +配置`database`的`DRIVER`， 默认值 `org.postgresql.Driver`。 **注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 -**`POSTGRESQL_USERNAME`** +**`DATABASE_HOST`** -配置`PostgreSQL`的`USERNAME`， 默认值 `root`。 +配置`database`的`HOST`， 默认值 `127.0.0.1`。 **注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 -**`POSTGRESQL_PASSWORD`** +**`DATABASE_PORT`** -配置`PostgreSQL`的`PASSWORD`， 默认值 `root`。 +配置`database`的`PORT`， 默认值 `5432`。 **注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 -**`POSTGRESQL_DATABASE`** +**`DATABASE_USERNAME`** -配置`PostgreSQL`的`DATABASE`， 默认值 `dolphinscheduler`。 +配置`database`的`USERNAME`， 默认值 `root`。 + +**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 + +**`DATABASE_PASSWORD`** + +配置`database`的`PASSWORD`， 默认值 `root`。 + +**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 + +**`DATABASE_DATABASE`** + +配置`database`的`DATABASE`， 默认值 `dolphinscheduler`。 + +**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 + +**`DATABASE_PARAMS`** + +配置`database`的`PARAMS`， 默认值 `characterEncoding=utf8`。 **注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 diff --git a/docker/build/conf/dolphinscheduler/alert.properties.tpl b/docker/build/conf/dolphinscheduler/alert.properties.tpl index b940ecd203..6d7450e13d 100644 --- a/docker/build/conf/dolphinscheduler/alert.properties.tpl +++ b/docker/build/conf/dolphinscheduler/alert.properties.tpl @@ -35,16 +35,22 @@ mail.smtp.ssl.trust=${MAIL_SMTP_SSL_TRUST} #xls file path,need create if not exist xls.file.path=${XLS_FILE_PATH} +# plugins dir +plugin.dir=${ALERT_PLUGIN_DIR} + # Enterprise WeChat configuration enterprise.wechat.enable=${ENTERPRISE_WECHAT_ENABLE} enterprise.wechat.corp.id=${ENTERPRISE_WECHAT_CORP_ID} enterprise.wechat.secret=${ENTERPRISE_WECHAT_SECRET} enterprise.wechat.agent.id=${ENTERPRISE_WECHAT_AGENT_ID} enterprise.wechat.users=${ENTERPRISE_WECHAT_USERS} -enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret -enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token -enterprise.wechat.team.send.msg={\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"} -enterprise.wechat.user.send.msg={\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}} +enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid={corpId}&corpsecret={secret} +enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={token} +enterprise.wechat.team.send.msg={\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"} +enterprise.wechat.user.send.msg={\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}} + + + diff --git a/docker/build/conf/dolphinscheduler/common.properties.tpl b/docker/build/conf/dolphinscheduler/common.properties.tpl index f318ff8414..ff74598fd4 100644 --- a/docker/build/conf/dolphinscheduler/common.properties.tpl +++ b/docker/build/conf/dolphinscheduler/common.properties.tpl @@ -25,13 +25,13 @@ dolphinscheduler.env.path=${DOLPHINSCHEDULER_ENV_PATH} data.basedir.path=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH} # resource upload startup type : HDFS,S3,NONE -resource.storage.type=NONE +resource.storage.type=${RESOURCE_STORAGE_TYPE} #============================================================================ # HDFS #============================================================================ # resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended -#resource.upload.path=/dolphinscheduler +resource.upload.path=${RESOURCE_UPLOAD_PATH} # whether kerberos starts #hadoop.security.authentication.startup.state=false @@ -58,16 +58,16 @@ kerberos.expire.time=7 # S3 #============================================================================ # if resource.storage.type=S3，the value like: s3a://dolphinscheduler ; if resource.storage.type=HDFS, When namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir -fs.defaultFS=hdfs://mycluster:8020 +fs.defaultFS=${FS_DEFAULT_FS} # if resource.storage.type=S3，s3 endpoint -#fs.s3a.endpoint=http://192.168.199.91:9010 +fs.s3a.endpoint=${FS_S3A_ENDPOINT} # if resource.storage.type=S3，s3 access key -#fs.s3a.access.key=A3DXS30FO22544RE +fs.s3a.access.key=${FS_S3A_ACCESS_KEY} # if resource.storage.type=S3，s3 secret key -#fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK +fs.s3a.secret.key=${FS_S3A_SECRET_KEY} # if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty TODO yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx diff --git a/docker/build/conf/dolphinscheduler/datasource.properties.tpl b/docker/build/conf/dolphinscheduler/datasource.properties.tpl index aefb9e3b0b..f7c5ee6881 100644 --- a/docker/build/conf/dolphinscheduler/datasource.properties.tpl +++ b/docker/build/conf/dolphinscheduler/datasource.properties.tpl @@ -15,16 +15,11 @@ # limitations under the License. # - -# mysql -#spring.datasource.driver-class-name=com.mysql.jdbc.Driver -#spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 - -# postgre -spring.datasource.driver-class-name=org.postgresql.Driver -spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8 -spring.datasource.username=${POSTGRESQL_USERNAME} -spring.datasource.password=${POSTGRESQL_PASSWORD} +# db +spring.datasource.driver-class-name=${DATABASE_DRIVER} +spring.datasource.url=jdbc:${DATABASE_TYPE}://${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DATABASE}?${DATABASE_PARAMS} +spring.datasource.username=${DATABASE_USERNAME} +spring.datasource.password=${DATABASE_PASSWORD} ## base spring data source configuration todo need to remove #spring.datasource.type=com.alibaba.druid.pool.DruidDataSource diff --git a/docker/build/conf/dolphinscheduler/logback/logback-alert.xml b/docker/build/conf/dolphinscheduler/logback/logback-alert.xml index 5d1c07858d..1718947dd1 100644 --- a/docker/build/conf/dolphinscheduler/logback/logback-alert.xml +++ b/docker/build/conf/dolphinscheduler/logback/logback-alert.xml @@ -46,7 +46,7 @@ - + \ No newline at end of file diff --git a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl b/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl index 362a4e85ad..51540aa345 100644 --- a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl +++ b/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl @@ -19,7 +19,7 @@ zookeeper.quorum=${ZOOKEEPER_QUORUM} # dolphinscheduler root directory -#zookeeper.dolphinscheduler.root=/dolphinscheduler +zookeeper.dolphinscheduler.root=${ZOOKEEPER_ROOT} # dolphinscheduler failover directory #zookeeper.session.timeout=60000 diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh index da6eb21b7d..73fdad6798 100644 --- a/docker/build/startup-init-conf.sh +++ b/docker/build/startup-init-conf.sh @@ -24,22 +24,33 @@ echo "init env variables" #============================================================================ # Database Source #============================================================================ -export POSTGRESQL_HOST=${POSTGRESQL_HOST:-"127.0.0.1"} -export POSTGRESQL_PORT=${POSTGRESQL_PORT:-"5432"} -export POSTGRESQL_USERNAME=${POSTGRESQL_USERNAME:-"root"} -export POSTGRESQL_PASSWORD=${POSTGRESQL_PASSWORD:-"root"} -export POSTGRESQL_DATABASE=${POSTGRESQL_DATABASE:-"dolphinscheduler"} +export DATABASE_HOST=${DATABASE_HOST:-"127.0.0.1"} +export DATABASE_PORT=${DATABASE_PORT:-"5432"} +export DATABASE_USERNAME=${DATABASE_USERNAME:-"root"} +export DATABASE_PASSWORD=${DATABASE_PASSWORD:-"root"} +export DATABASE_DATABASE=${DATABASE_DATABASE:-"dolphinscheduler"} +export DATABASE_TYPE=${DATABASE_TYPE:-"postgresql"} +export DATABASE_DRIVER=${DATABASE_DRIVER:-"org.postgresql.Driver"} +export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"} #============================================================================ # System #============================================================================ export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"} export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"} +export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""} +export RESOURCE_STORAGE_TYPE=${RESOURCE_STORAGE_TYPE:-"NONE"} +export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/ds"} +export FS_DEFAULT_FS=${FS_DEFAULT_FS:-"s3a://xxxx"} +export FS_S3A_ENDPOINT=${FS_S3A_ENDPOINT:-"s3.xxx.amazonaws.com"} +export FS_S3A_ACCESS_KEY=${FS_S3A_ACCESS_KEY:-"xxxxxxx"} +export FS_S3A_SECRET_KEY=${FS_S3A_SECRET_KEY:-"xxxxxxx"} #============================================================================ # Zookeeper #============================================================================ export ZOOKEEPER_QUORUM=${ZOOKEEPER_QUORUM:-"127.0.0.1:2181"} +export ZOOKEEPER_ROOT=${ZOOKEEPER_ROOT:-"/dolphinscheduler"} #============================================================================ # Master Server @@ -67,6 +78,8 @@ export WORKER_GROUP=${WORKER_GROUP:-"default"} #============================================================================ # Alert Server #============================================================================ +# alert plugin dir +export ALERT_PLUGIN_DIR=${ALERT_PLUGIN_DIR:-"/opt/dolphinscheduler"} # XLS FILE export XLS_FILE_PATH=${XLS_FILE_PATH:-"/tmp/xls"} # mail diff --git a/docker/build/startup.sh b/docker/build/startup.sh index 00bb9cdc52..ea4fbc0cd1 100644 --- a/docker/build/startup.sh +++ b/docker/build/startup.sh @@ -22,24 +22,32 @@ DOLPHINSCHEDULER_BIN=${DOLPHINSCHEDULER_HOME}/bin DOLPHINSCHEDULER_SCRIPT=${DOLPHINSCHEDULER_HOME}/script DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs -# start postgresql -initPostgreSQL() { - echo "test postgresql service" - while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do +# start database +initDatabase() { + echo "test ${DATABASE_TYPE} service" + while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do counter=$((counter+1)) if [ $counter == 30 ]; then - echo "Error: Couldn't connect to postgresql." + echo "Error: Couldn't connect to ${DATABASE_TYPE}." exit 1 fi - echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter." + echo "Trying to connect to ${DATABASE_TYPE} at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter." sleep 5 done - echo "connect postgresql service" - v=$(sudo -u postgres PGPASSWORD=${POSTGRESQL_PASSWORD} psql -h ${POSTGRESQL_HOST} -p ${POSTGRESQL_PORT} -U ${POSTGRESQL_USERNAME} -d dolphinscheduler -tAc "select 1") - if [ "$(echo '${v}' | grep 'FATAL' | wc -l)" -eq 1 ]; then - echo "Error: Can't connect to database...${v}" - exit 1 + echo "connect ${DATABASE_TYPE} service" + if [ ${DATABASE_TYPE} = "mysql" ]; then + v=$(mysql -h${DATABASE_HOST} -P${DATABASE_PORT} -u${DATABASE_USERNAME} --password=${DATABASE_PASSWORD} -D ${DATABASE_DATABASE} -e "select 1" 2>&1) + if [ "$(echo '${v}' | grep 'ERROR' | wc -l)" -eq 1 ]; then + echo "Error: Can't connect to database...${v}" + exit 1 + fi + else + v=$(sudo -u postgres PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "select 1") + if [ "$(echo '${v}' | grep 'FATAL' | wc -l)" -eq 1 ]; then + echo "Error: Can't connect to database...${v}" + exit 1 + fi fi echo "import sql data" @@ -123,7 +131,7 @@ LOGFILE=/var/log/nginx/access.log case "$1" in (all) initZK - initPostgreSQL + initDatabase initMasterServer initWorkerServer initApiServer @@ -134,25 +142,25 @@ case "$1" in ;; (master-server) initZK - initPostgreSQL + initDatabase initMasterServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-master.log ;; (worker-server) initZK - initPostgreSQL + initDatabase initWorkerServer initLoggerServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log ;; (api-server) initZK - initPostgreSQL + initDatabase initApiServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log ;; (alert-server) - initPostgreSQL + initDatabase initAlertServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-alert.log ;; diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index 5cb6717f15..51eb0aeaa5 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -56,11 +56,11 @@ services: - 12345:12345 environment: TZ: Asia/Shanghai - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 healthcheck: test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] @@ -119,11 +119,11 @@ services: ENTERPRISE_WECHAT_SECRET: "" ENTERPRISE_WECHAT_AGENT_ID: "" ENTERPRISE_WECHAT_USERS: "" - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler healthcheck: test: ["CMD", "/root/checkpoint.sh", "AlertServer"] interval: 30s @@ -152,11 +152,11 @@ services: MASTER_TASK_COMMIT_INTERVAL: "1000" MASTER_MAX_CPULOAD_AVG: "100" MASTER_RESERVED_MEMORY: "0.1" - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 healthcheck: test: ["CMD", "/root/checkpoint.sh", "MasterServer"] @@ -188,11 +188,11 @@ services: WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUP: "default" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 healthcheck: test: ["CMD", "/root/checkpoint.sh", "WorkerServer"] diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml index 5c3873fb07..ca9f7c88c7 100644 --- a/docker/docker-swarm/docker-stack.yml +++ b/docker/docker-swarm/docker-stack.yml @@ -20,13 +20,13 @@ services: dolphinscheduler-postgresql: image: bitnami/postgresql:latest - ports: - - 5432:5432 environment: TZ: Asia/Shanghai POSTGRESQL_USERNAME: root POSTGRESQL_PASSWORD: root POSTGRESQL_DATABASE: dolphinscheduler + ports: + - 5432:5432 volumes: - dolphinscheduler-postgresql:/bitnami/postgresql networks: @@ -37,12 +37,12 @@ services: dolphinscheduler-zookeeper: image: bitnami/zookeeper:latest - ports: - - 2181:2181 environment: TZ: Asia/Shanghai ALLOW_ANONYMOUS_LOGIN: "yes" ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons + ports: + - 2181:2181 volumes: - dolphinscheduler-zookeeper:/bitnami/zookeeper networks: @@ -54,16 +54,16 @@ services: dolphinscheduler-api: image: apache/dolphinscheduler:latest command: ["api-server"] - ports: - - 12345:12345 environment: TZ: Asia/Shanghai - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 + ports: + - 12345:12345 healthcheck: test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] interval: 30 @@ -120,11 +120,11 @@ services: ENTERPRISE_WECHAT_SECRET: "" ENTERPRISE_WECHAT_AGENT_ID: "" ENTERPRISE_WECHAT_USERS: "" - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler healthcheck: test: ["CMD", "/root/checkpoint.sh", "AlertServer"] interval: 30 @@ -153,11 +153,11 @@ services: MASTER_TASK_COMMIT_INTERVAL: "1000" MASTER_MAX_CPULOAD_AVG: "100" MASTER_RESERVED_MEMORY: "0.1" - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 healthcheck: test: ["CMD", "/root/checkpoint.sh", "MasterServer"] @@ -188,11 +188,11 @@ services: WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUP: "default" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 healthcheck: test: ["CMD", "/root/checkpoint.sh", "WorkerServer"] diff --git a/docker/kubernetes/dolphinscheduler/Chart.yaml b/docker/kubernetes/dolphinscheduler/Chart.yaml index ac989d571f..9d640869dd 100644 --- a/docker/kubernetes/dolphinscheduler/Chart.yaml +++ b/docker/kubernetes/dolphinscheduler/Chart.yaml @@ -49,4 +49,4 @@ dependencies: - name: zookeeper version: 5.x.x repository: https://charts.bitnami.com/bitnami - condition: redis.enabled + condition: zookeeper.enabled diff --git a/docker/kubernetes/dolphinscheduler/README.md b/docker/kubernetes/dolphinscheduler/README.md index 9e6d1c6448..0691b1ed2e 100644 --- a/docker/kubernetes/dolphinscheduler/README.md +++ b/docker/kubernetes/dolphinscheduler/README.md @@ -46,8 +46,8 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `image.repository` | Docker image repository for the Dolphins Scheduler | `dolphinscheduler` | | `image.tag` | Docker image version for the Dolphins Scheduler | `1.2.1` | | `image.imagePullPolicy` | Image pull policy. One of Always, Never, IfNotPresent | `IfNotPresent` | -| `imagePullSecrets` | ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images | `[]` | -| | | | +| `image.pullSecres` | PullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images | `[]` | +| | | | | `postgresql.enabled` | If not exists external PostgreSQL, by default, the Dolphins Scheduler will use a internal PostgreSQL | `true` | | `postgresql.postgresqlUsername` | The username for internal PostgreSQL | `root` | | `postgresql.postgresqlPassword` | The password for internal PostgreSQL | `root` | @@ -55,12 +55,15 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `postgresql.persistence.enabled` | Set `postgresql.persistence.enabled` to `true` to mount a new volume for internal PostgreSQL | `false` | | `postgresql.persistence.size` | `PersistentVolumeClaim` Size | `20Gi` | | `postgresql.persistence.storageClass` | PostgreSQL data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | -| `externalDatabase.host` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database host will use it. | `localhost` | +| `externalDatabase.type` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database type will use it. | `postgresql` | +| `externalDatabase.driver` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database driver will use it. | `org.postgresql.Driver` | +| `externalDatabase.host` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database host will use it. | `localhost` | | `externalDatabase.port` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database port will use it. | `5432` | | `externalDatabase.username` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database username will use it. | `root` | | `externalDatabase.password` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database password will use it. | `root` | | `externalDatabase.database` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database database will use it. | `dolphinscheduler` | -| | | | +| `externalDatabase.params` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database params will use it. | `characterEncoding=utf8` | +| | | | | `zookeeper.enabled` | If not exists external Zookeeper, by default, the Dolphin Scheduler will use a internal Zookeeper | `true` | | `zookeeper.taskQueue` | Specify task queue for `master` and `worker` | `zookeeper` | | `zookeeper.persistence.enabled` | Set `zookeeper.persistence.enabled` to `true` to mount a new volume for internal Zookeeper | `false` | @@ -68,12 +71,25 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `zookeeper.persistence.storageClass` | Zookeeper data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | | `externalZookeeper.taskQueue` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify task queue for `master` and `worker` | `zookeeper` | | `externalZookeeper.zookeeperQuorum` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify Zookeeper quorum | `127.0.0.1:2181` | -| | | | +| `externalZookeeper.zookeeperRoot` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify Zookeeper root path for `master` and `worker` | `dolphinscheduler` | +| | | | +| `common.configmap.DOLPHINSCHEDULER_ENV_PATH` | Extra env file path. | `/tmp/dolphinscheduler/env` | +| `common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH` | File uploaded path of DS. | `/tmp/dolphinscheduler/files` | +| `common.configmap.RESOURCE_STORAGE_TYPE` | Resource Storate type, support type are: S3、HDFS、NONE. | `NONE` | +| `common.configmap.RESOURCE_UPLOAD_PATH` | The base path of resource. | `/ds` | +| `common.configmap.FS_DEFAULT_FS` | The default fs of resource, for s3 is the `s3a` prefix and bucket name. | `s3a://xxxx` | +| `common.configmap.FS_S3A_ENDPOINT` | If the resource type is `S3`, you should fill this filed, it's the endpoint of s3. | `s3.xxx.amazonaws.com` | +| `common.configmap.FS_S3A_ACCESS_KEY` | The access key for your s3 bucket. | `xxxxxxx` | +| `common.configmap.FS_S3A_SECRET_KEY` | The secret key for your s3 bucket. | `xxxxxxx` | | `master.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` | +| | | | | `master.replicas` | Replicas is the desired number of replicas of the given Template | `3` | | `master.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | | `master.tolerations` | If specified, the pod's tolerations | `{}` | | `master.affinity` | If specified, the pod's scheduling constraints | `{}` | +| `master.jvmOptions` | The JVM options for master server. | `""` | +| `master.resources` | The `resource` limit and request config for master server. | `{}` | +| `master.annotations` | The `annotations` for master server. | `{}` | | `master.configmap.MASTER_EXEC_THREADS` | Master execute thread num | `100` | | `master.configmap.MASTER_EXEC_TASK_NUM` | Master execute task number in parallel | `20` | | `master.configmap.MASTER_HEARTBEAT_INTERVAL` | Master heartbeat interval | `10` | @@ -97,12 +113,15 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `master.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | | `master.persistentVolumeClaim.storageClassName` | `Master` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | | `master.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` | -| | | | +| | | | | `worker.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` | | `worker.replicas` | Replicas is the desired number of replicas of the given Template | `3` | | `worker.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | | `worker.tolerations` | If specified, the pod's tolerations | `{}` | | `worker.affinity` | If specified, the pod's scheduling constraints | `{}` | +| `worker.jvmOptions` | The JVM options for worker server. | `""` | +| `worker.resources` | The `resource` limit and request config for worker server. | `{}` | +| `worker.annotations` | The `annotations` for worker server. | `{}` | | `worker.configmap.WORKER_EXEC_THREADS` | Worker execute thread num | `100` | | `worker.configmap.WORKER_HEARTBEAT_INTERVAL` | Worker heartbeat interval | `10` | | `worker.configmap.WORKER_FETCH_TASK_NUM` | Submit the number of tasks at a time | `3` | @@ -131,7 +150,7 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `worker.persistentVolumeClaim.logsPersistentVolume.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | | `worker.persistentVolumeClaim.logsPersistentVolume.storageClassName` | `Worker` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | | `worker.persistentVolumeClaim.logsPersistentVolume.storage` | `PersistentVolumeClaim` Size | `20Gi` | -| | | | +| | | | | `alert.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` | | `alert.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` | | `alert.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` | @@ -139,6 +158,10 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `alert.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | | `alert.tolerations` | If specified, the pod's tolerations | `{}` | | `alert.affinity` | If specified, the pod's scheduling constraints | `{}` | +| `alert.jvmOptions` | The JVM options for alert server. | `""` | +| `alert.resources` | The `resource` limit and request config for alert server. | `{}` | +| `alert.annotations` | The `annotations` for alert server. | `{}` | +| `alert.configmap.ALERT_PLUGIN_DIR` | Alert plugin path. | `/opt/dolphinscheduler/alert/plugin` | | `alert.configmap.XLS_FILE_PATH` | XLS file path | `/tmp/xls` | | `alert.configmap.MAIL_SERVER_HOST` | Mail `SERVER HOST ` | `nil` | | `alert.configmap.MAIL_SERVER_PORT` | Mail `SERVER PORT` | `nil` | @@ -177,6 +200,9 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `api.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | | `api.tolerations` | If specified, the pod's tolerations | `{}` | | `api.affinity` | If specified, the pod's scheduling constraints | `{}` | +| `api.jvmOptions` | The JVM options for api server. | `""` | +| `api.resources` | The `resource` limit and request config for api server. | `{}` | +| `api.annotations` | The `annotations` for api server. | `{}` | | `api.livenessProbe.enabled` | Turn on and off liveness probe | `true` | | `api.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | | `api.livenessProbe.periodSeconds` | How often to perform the probe | `30` | @@ -201,6 +227,8 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `frontend.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | | `frontend.tolerations` | If specified, the pod's tolerations | `{}` | | `frontend.affinity` | If specified, the pod's scheduling constraints | `{}` | +| `frontend.resources` | The `resource` limit and request config for frontend server. | `{}` | +| `frontend.annotations` | The `annotations` for frontend server. | `{}` | | `frontend.livenessProbe.enabled` | Turn on and off liveness probe | `true` | | `frontend.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | | `frontend.livenessProbe.periodSeconds` | How often to perform the probe | `30` | diff --git a/docker/kubernetes/dolphinscheduler/requirements.yaml b/docker/kubernetes/dolphinscheduler/requirements.yaml deleted file mode 100644 index e219975995..0000000000 --- a/docker/kubernetes/dolphinscheduler/requirements.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -dependencies: -- name: postgresql - version: 8.x.x - repository: https://charts.bitnami.com/bitnami - condition: postgresql.enabled -- name: zookeeper - version: 5.x.x - repository: https://charts.bitnami.com/bitnami - condition: redis.enabled \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/NOTES.txt b/docker/kubernetes/dolphinscheduler/templates/NOTES.txt index eb3a9cfc52..256c53ca57 100644 --- a/docker/kubernetes/dolphinscheduler/templates/NOTES.txt +++ b/docker/kubernetes/dolphinscheduler/templates/NOTES.txt @@ -29,16 +29,3 @@ kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-frontend 8888:8888 {{- end }} - -2. Get the Dolphinscheduler URL by running: - -{{- if .Values.ingress.enabled }} - - export HOSTNAME=$(kubectl get ingress --namespace {{ .Release.Namespace }} {{ template "dolphinscheduler.fullname" . }} -o jsonpath='{.spec.rules[0].host}') - echo "Dolphinscheduler URL: http://$HOSTNAME/" - -{{- else }} - - kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-frontend 8888:8888 - -{{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl index 9ba290b771..1f121dfea8 100644 --- a/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl +++ b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl @@ -96,6 +96,14 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- printf "%s/%s:%s" (include "dolphinscheduler.image.registry" .) .Values.image.repository .Values.image.tag -}} {{- end -}} +{{/* +Create a default image pull secrects. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "dolphinscheduler.image.pullSecrets" -}} +{{- default nil .Values.image.pullSecrets -}} +{{- end -}} + {{/* Create a default fully qualified postgresql name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml index 76daad8568..2c5b76c3f0 100644 --- a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml @@ -24,6 +24,7 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} data: + ALERT_PLUGIN_DIR: {{ .Values.alert.configmap.ALERT_PLUGIN_DIR | quote }} XLS_FILE_PATH: {{ .Values.alert.configmap.XLS_FILE_PATH | quote }} MAIL_SERVER_HOST: {{ .Values.alert.configmap.MAIL_SERVER_HOST | quote }} MAIL_SERVER_PORT: {{ .Values.alert.configmap.MAIL_SERVER_PORT | quote }} diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml new file mode 100644 index 0000000000..931f6d48e4 --- /dev/null +++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- if .Values.common.configmap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-common + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-common + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +data: + DOLPHINSCHEDULER_ENV_PATH: {{ .Values.common.configmap.DOLPHINSCHEDULER_ENV_PATH | quote }} + DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ .Values.common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH | quote }} + RESOURCE_STORAGE_TYPE: {{ .Values.common.configmap.RESOURCE_STORAGE_TYPE | quote }} + RESOURCE_UPLOAD_PATH: {{ .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }} + FS_DEFAULT_FS: {{ .Values.common.configmap.FS_DEFAULT_FS | quote }} + FS_S3A_ENDPOINT: {{ .Values.common.configmap.FS_S3A_ENDPOINT | quote }} + FS_S3A_ACCESS_KEY: {{ .Values.common.configmap.FS_S3A_ACCESS_KEY | quote }} + FS_S3A_SECRET_KEY: {{ .Values.common.configmap.FS_S3A_SECRET_KEY | quote }} +{{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml index 69662e95d9..fb9a23786b 100644 --- a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml @@ -43,6 +43,10 @@ spec: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/component: alert + {{- if .Values.alert.annotations }} + annotations: + {{- toYaml .Values.alert.annotations | nindent 8 }} + {{- end }} spec: {{- if .Values.alert.affinity }} affinity: {{- toYaml .Values.alert.affinity | nindent 8 }} @@ -54,34 +58,38 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} initContainers: - - name: init-postgresql + - name: init-database image: busybox:1.31.0 command: - /bin/sh - -ec - | - while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do + while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do counter=$((counter+1)) if [ $counter == 5 ]; then - echo "Error: Couldn't connect to postgresql." + echo "Error: Couldn't connect to database." exit 1 fi - echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter." + echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter." sleep 60 done env: - - name: POSTGRESQL_HOST + - name: DATABASE_HOST {{- if .Values.postgresql.enabled }} value: {{ template "dolphinscheduler.postgresql.fullname" . }} {{- else }} value: {{ .Values.externalDatabase.host | quote }} {{- end }} - - name: POSTGRESQL_PORT + - name: DATABASE_PORT {{- if .Values.postgresql.enabled }} value: "5432" {{- else }} - value: {{ .Values.externalDatabase.port }} + value: {{ .Values.externalDatabase.port | quote }} {{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + - name: {{ include "dolphinscheduler.image.pullSecrets" . }} + {{- end }} containers: - name: {{ include "dolphinscheduler.fullname" . }}-alert image: {{ include "dolphinscheduler.image.repository" . | quote }} @@ -89,8 +97,15 @@ spec: - "alert-server" imagePullPolicy: {{ .Values.image.pullPolicy }} env: + - name: DOLPHINSCHEDULER_OPTS + value: {{ default "" .Values.alert.jvmOptions }} - name: TZ value: {{ .Values.timezone }} + - name: ALERT_PLUGIN_DIR + valueFrom: + configMapKeyRef: + key: ALERT_PLUGIN_DIR + name: {{ include "dolphinscheduler.fullname" . }}-alert - name: XLS_FILE_PATH valueFrom: configMapKeyRef: @@ -161,25 +176,37 @@ spec: configMapKeyRef: key: ENTERPRISE_WECHAT_USERS name: {{ include "dolphinscheduler.fullname" . }}-alert - - name: POSTGRESQL_HOST + - name: DATABASE_TYPE + {{- if .Values.postgresql.enabled }} + value: "postgresql" + {{- else }} + value: {{ .Values.externalDatabase.type | quote }} + {{- end }} + - name: DATABASE_DRIVER + {{- if .Values.postgresql.enabled }} + value: "org.postgresql.Driver" + {{- else }} + value: {{ .Values.externalDatabase.driver | quote }} + {{- end }} + - name: DATABASE_HOST {{- if .Values.postgresql.enabled }} value: {{ template "dolphinscheduler.postgresql.fullname" . }} {{- else }} value: {{ .Values.externalDatabase.host | quote }} {{- end }} - - name: POSTGRESQL_PORT + - name: DATABASE_PORT {{- if .Values.postgresql.enabled }} value: "5432" {{- else }} - value: {{ .Values.externalDatabase.port }} + value: {{ .Values.externalDatabase.port | quote }} {{- end }} - - name: POSTGRESQL_USERNAME + - name: DATABASE_USERNAME {{- if .Values.postgresql.enabled }} value: {{ .Values.postgresql.postgresqlUsername }} {{- else }} value: {{ .Values.externalDatabase.username | quote }} {{- end }} - - name: POSTGRESQL_PASSWORD + - name: DATABASE_PASSWORD valueFrom: secretKeyRef: {{- if .Values.postgresql.enabled }} @@ -189,12 +216,57 @@ spec: name: {{ printf "%s-%s" .Release.Name "externaldb" }} key: db-password {{- end }} - - name: POSTGRESQL_DATABASE + - name: DATABASE_DATABASE {{- if .Values.postgresql.enabled }} value: {{ .Values.postgresql.postgresqlDatabase }} {{- else }} value: {{ .Values.externalDatabase.database | quote }} {{- end }} + - name: DATABASE_PARAMS + {{- if .Values.postgresql.enabled }} + value: "characterEncoding=utf8" + {{- else }} + value: {{ .Values.externalDatabase.params | quote }} + {{- end }} + - name: RESOURCE_STORAGE_TYPE + valueFrom: + configMapKeyRef: + key: RESOURCE_STORAGE_TYPE + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: RESOURCE_UPLOAD_PATH + valueFrom: + configMapKeyRef: + key: RESOURCE_UPLOAD_PATH + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_DEFAULT_FS + valueFrom: + configMapKeyRef: + key: FS_DEFAULT_FS + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_ENDPOINT + valueFrom: + configMapKeyRef: + key: FS_S3A_ENDPOINT + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_ACCESS_KEY + valueFrom: + configMapKeyRef: + key: FS_S3A_ACCESS_KEY + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_SECRET_KEY + valueFrom: + configMapKeyRef: + key: FS_S3A_SECRET_KEY + name: {{ include "dolphinscheduler.fullname" . }}-common + {{- if .Values.alert.resources }} + resources: + limits: + memory: {{ .Values.alert.resources.limits.memory }} + cpu: {{ .Values.alert.resources.limits.cpu }} + requests: + memory: {{ .Values.alert.resources.requests.memory }} + cpu: {{ .Values.alert.resources.requests.cpu }} + {{- end }} {{- if .Values.alert.livenessProbe.enabled }} livenessProbe: exec: diff --git a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml index 487ede0b8f..51a83bcfa7 100644 --- a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml @@ -43,6 +43,10 @@ spec: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/component: api + {{- if .Values.alert.annotations }} + annotations: + {{- toYaml .Values.alert.annotations | nindent 8 }} + {{- end }} spec: {{- if .Values.api.affinity }} affinity: {{- toYaml .Values.api.affinity | nindent 8 }} @@ -54,34 +58,38 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} initContainers: - - name: init-postgresql + - name: init-database image: busybox:1.31.0 command: - /bin/sh - -ec - | - while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do + while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do counter=$((counter+1)) if [ $counter == 5 ]; then - echo "Error: Couldn't connect to postgresql." + echo "Error: Couldn't connect to database." exit 1 fi - echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter." + echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter." sleep 60 done env: - - name: POSTGRESQL_HOST + - name: DATABASE_HOST {{- if .Values.postgresql.enabled }} value: {{ template "dolphinscheduler.postgresql.fullname" . }} {{- else }} value: {{ .Values.externalDatabase.host | quote }} {{- end }} - - name: POSTGRESQL_PORT + - name: DATABASE_PORT {{- if .Values.postgresql.enabled }} value: "5432" {{- else }} - value: {{ .Values.externalDatabase.port }} + value: {{ .Values.externalDatabase.port | quote }} {{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + - name: {{ include "dolphinscheduler.image.pullSecrets" . }} + {{- end }} containers: - name: {{ include "dolphinscheduler.fullname" . }}-api image: {{ include "dolphinscheduler.image.repository" . | quote }} @@ -92,27 +100,41 @@ spec: name: tcp-port imagePullPolicy: {{ .Values.image.pullPolicy }} env: + - name: DOLPHINSCHEDULER_OPTS + value: {{ default "" .Values.api.jvmOptions }} - name: TZ value: {{ .Values.timezone }} - - name: POSTGRESQL_HOST + - name: DATABASE_TYPE + {{- if .Values.postgresql.enabled }} + value: "postgresql" + {{- else }} + value: {{ .Values.externalDatabase.type | quote }} + {{- end }} + - name: DATABASE_DRIVER + {{- if .Values.postgresql.enabled }} + value: "org.postgresql.Driver" + {{- else }} + value: {{ .Values.externalDatabase.driver | quote }} + {{- end }} + - name: DATABASE_HOST {{- if .Values.postgresql.enabled }} value: {{ template "dolphinscheduler.postgresql.fullname" . }} {{- else }} value: {{ .Values.externalDatabase.host | quote }} {{- end }} - - name: POSTGRESQL_PORT + - name: DATABASE_PORT {{- if .Values.postgresql.enabled }} value: "5432" {{- else }} - value: {{ .Values.externalDatabase.port }} + value: {{ .Values.externalDatabase.port | quote }} {{- end }} - - name: POSTGRESQL_USERNAME + - name: DATABASE_USERNAME {{- if .Values.postgresql.enabled }} value: {{ .Values.postgresql.postgresqlUsername }} {{- else }} value: {{ .Values.externalDatabase.username | quote }} {{- end }} - - name: POSTGRESQL_PASSWORD + - name: DATABASE_PASSWORD valueFrom: secretKeyRef: {{- if .Values.postgresql.enabled }} @@ -122,18 +144,63 @@ spec: name: {{ printf "%s-%s" .Release.Name "externaldb" }} key: db-password {{- end }} - - name: POSTGRESQL_DATABASE + - name: DATABASE_DATABASE {{- if .Values.postgresql.enabled }} value: {{ .Values.postgresql.postgresqlDatabase }} {{- else }} value: {{ .Values.externalDatabase.database | quote }} {{- end }} + - name: DATABASE_PARAMS + {{- if .Values.postgresql.enabled }} + value: "characterEncoding=utf8" + {{- else }} + value: {{ .Values.externalDatabase.params | quote }} + {{- end }} - name: ZOOKEEPER_QUORUM {{- if .Values.zookeeper.enabled }} value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}" {{- else }} value: {{ .Values.externalZookeeper.zookeeperQuorum }} {{- end }} + - name: RESOURCE_STORAGE_TYPE + valueFrom: + configMapKeyRef: + key: RESOURCE_STORAGE_TYPE + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: RESOURCE_UPLOAD_PATH + valueFrom: + configMapKeyRef: + key: RESOURCE_UPLOAD_PATH + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_DEFAULT_FS + valueFrom: + configMapKeyRef: + key: FS_DEFAULT_FS + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_ENDPOINT + valueFrom: + configMapKeyRef: + key: FS_S3A_ENDPOINT + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_ACCESS_KEY + valueFrom: + configMapKeyRef: + key: FS_S3A_ACCESS_KEY + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_SECRET_KEY + valueFrom: + configMapKeyRef: + key: FS_S3A_SECRET_KEY + name: {{ include "dolphinscheduler.fullname" . }}-common + {{- if .Values.api.resources }} + resources: + limits: + memory: {{ .Values.api.resources.limits.memory | quote }} + cpu: {{ .Values.api.resources.limits.cpu | quote }} + requests: + memory: {{ .Values.api.resources.requests.memory | quote }} + cpu: {{ .Values.api.resources.requests.cpu | quote }} + {{- end }} {{- if .Values.api.livenessProbe.enabled }} livenessProbe: tcpSocket: diff --git a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml index aea09f107f..04b9408d9f 100644 --- a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml @@ -43,6 +43,10 @@ spec: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/component: frontend + {{- if .Values.alert.annotations }} + annotations: + {{- toYaml .Values.alert.annotations | nindent 8 }} + {{- end }} spec: {{- if .Values.frontend.affinity }} affinity: {{- toYaml .Values.frontend.affinity | nindent 8 }} @@ -53,6 +57,10 @@ spec: {{- if .Values.frontend.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + - name: {{ include "dolphinscheduler.image.pullSecrets" . }} + {{- end }} containers: - name: {{ include "dolphinscheduler.fullname" . }}-frontend image: {{ include "dolphinscheduler.image.repository" . | quote }} @@ -69,6 +77,15 @@ spec: value: '{{ include "dolphinscheduler.fullname" . }}-api' - name: FRONTEND_API_SERVER_PORT value: "12345" + {{- if .Values.frontend.resources }} + resources: + limits: + memory: {{ .Values.frontend.resources.limits.memory | quote }} + cpu: {{ .Values.frontend.resources.limits.cpu | quote }} + requests: + memory: {{ .Values.frontend.resources.requests.memory | quote }} + cpu: {{ .Values.frontend.resources.requests.cpu | quote }} + {{- end }} {{- if .Values.frontend.livenessProbe.enabled }} livenessProbe: tcpSocket: diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml index b142fac4fe..0949127dda 100644 --- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml @@ -40,6 +40,10 @@ spec: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/component: master + {{- if .Values.alert.annotations }} + annotations: + {{- toYaml .Values.alert.annotations | nindent 8 }} + {{- end }} spec: {{- if .Values.master.affinity }} affinity: {{- toYaml .Values.master.affinity | nindent 8 }} @@ -75,34 +79,38 @@ spec: {{- else }} value: {{ .Values.externalZookeeper.zookeeperQuorum }} {{- end }} - - name: init-postgresql + - name: init-database image: busybox:1.31.0 command: - /bin/sh - -ec - | - while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do + while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do counter=$((counter+1)) if [ $counter == 5 ]; then - echo "Error: Couldn't connect to postgresql." + echo "Error: Couldn't connect to database." exit 1 fi - echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter." + echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter." sleep 60 done env: - - name: POSTGRESQL_HOST + - name: DATABASE_HOST {{- if .Values.postgresql.enabled }} value: {{ template "dolphinscheduler.postgresql.fullname" . }} {{- else }} value: {{ .Values.externalDatabase.host | quote }} {{- end }} - - name: POSTGRESQL_PORT + - name: DATABASE_PORT {{- if .Values.postgresql.enabled }} value: "5432" {{- else }} - value: {{ .Values.externalDatabase.port }} + value: {{ .Values.externalDatabase.port | quote }} {{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + - name: {{ include "dolphinscheduler.image.pullSecrets" . }} + {{- end }} containers: - name: {{ include "dolphinscheduler.fullname" . }}-master image: {{ include "dolphinscheduler.image.repository" . | quote }} @@ -113,6 +121,8 @@ spec: name: "master-port" imagePullPolicy: {{ .Values.image.pullPolicy }} env: + - name: DOLPHINSCHEDULER_OPTS + value: {{ default "" .Values.master.jvmOptions }} - name: TZ value: {{ .Values.timezone }} - name: MASTER_EXEC_THREADS @@ -160,25 +170,37 @@ spec: configMapKeyRef: name: {{ include "dolphinscheduler.fullname" . }}-master key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH - - name: POSTGRESQL_HOST + - name: DATABASE_TYPE + {{- if .Values.postgresql.enabled }} + value: "postgresql" + {{- else }} + value: {{ .Values.externalDatabase.type | quote }} + {{- end }} + - name: DATABASE_DRIVER + {{- if .Values.postgresql.enabled }} + value: "org.postgresql.Driver" + {{- else }} + value: {{ .Values.externalDatabase.driver | quote }} + {{- end }} + - name: DATABASE_HOST {{- if .Values.postgresql.enabled }} value: {{ template "dolphinscheduler.postgresql.fullname" . }} {{- else }} value: {{ .Values.externalDatabase.host | quote }} {{- end }} - - name: POSTGRESQL_PORT + - name: DATABASE_PORT {{- if .Values.postgresql.enabled }} value: "5432" {{- else }} - value: {{ .Values.externalDatabase.port }} + value: {{ .Values.externalDatabase.port | quote }} {{- end }} - - name: POSTGRESQL_USERNAME + - name: DATABASE_USERNAME {{- if .Values.postgresql.enabled }} value: {{ .Values.postgresql.postgresqlUsername }} {{- else }} value: {{ .Values.externalDatabase.username | quote }} {{- end }} - - name: POSTGRESQL_PASSWORD + - name: DATABASE_PASSWORD valueFrom: secretKeyRef: {{- if .Values.postgresql.enabled }} @@ -188,18 +210,63 @@ spec: name: {{ printf "%s-%s" .Release.Name "externaldb" }} key: db-password {{- end }} - - name: POSTGRESQL_DATABASE + - name: DATABASE_DATABASE {{- if .Values.postgresql.enabled }} value: {{ .Values.postgresql.postgresqlDatabase }} {{- else }} value: {{ .Values.externalDatabase.database | quote }} {{- end }} + - name: DATABASE_PARAMS + {{- if .Values.postgresql.enabled }} + value: "characterEncoding=utf8" + {{- else }} + value: {{ .Values.externalDatabase.params | quote }} + {{- end }} - name: ZOOKEEPER_QUORUM {{- if .Values.zookeeper.enabled }} value: {{ template "dolphinscheduler.zookeeper.quorum" . }} {{- else }} value: {{ .Values.externalZookeeper.zookeeperQuorum }} {{- end }} + - name: RESOURCE_STORAGE_TYPE + valueFrom: + configMapKeyRef: + key: RESOURCE_STORAGE_TYPE + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: RESOURCE_UPLOAD_PATH + valueFrom: + configMapKeyRef: + key: RESOURCE_UPLOAD_PATH + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_DEFAULT_FS + valueFrom: + configMapKeyRef: + key: FS_DEFAULT_FS + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_ENDPOINT + valueFrom: + configMapKeyRef: + key: FS_S3A_ENDPOINT + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_ACCESS_KEY + valueFrom: + configMapKeyRef: + key: FS_S3A_ACCESS_KEY + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_SECRET_KEY + valueFrom: + configMapKeyRef: + key: FS_S3A_SECRET_KEY + name: {{ include "dolphinscheduler.fullname" . }}-common + {{- if .Values.master.resources }} + resources: + limits: + memory: {{ .Values.master.resources.limits.memory | quote }} + cpu: {{ .Values.master.resources.limits.cpu | quote }} + requests: + memory: {{ .Values.master.resources.requests.memory | quote }} + cpu: {{ .Values.master.resources.requests.cpu | quote }} + {{- end }} {{- if .Values.master.livenessProbe.enabled }} livenessProbe: exec: diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml index 198cef43b6..097f8d8580 100644 --- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml @@ -40,6 +40,10 @@ spec: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/component: worker + {{- if .Values.alert.annotations }} + annotations: + {{- toYaml .Values.alert.annotations | nindent 8 }} + {{- end }} spec: {{- if .Values.worker.affinity }} affinity: {{- toYaml .Values.worker.affinity | nindent 8 }} @@ -75,34 +79,38 @@ spec: {{- else }} value: {{ .Values.externalZookeeper.zookeeperQuorum }} {{- end }} - - name: init-postgresql + - name: init-database image: busybox:1.31.0 command: - /bin/sh - -ec - | - while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do + while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do counter=$((counter+1)) if [ $counter == 5 ]; then - echo "Error: Couldn't connect to postgresql." + echo "Error: Couldn't connect to database." exit 1 fi - echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter." + echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter." sleep 60 done env: - - name: POSTGRESQL_HOST + - name: DATABASE_HOST {{- if .Values.postgresql.enabled }} value: {{ template "dolphinscheduler.postgresql.fullname" . }} {{- else }} value: {{ .Values.externalDatabase.host | quote }} {{- end }} - - name: POSTGRESQL_PORT + - name: DATABASE_PORT {{- if .Values.postgresql.enabled }} value: "5432" {{- else }} - value: {{ .Values.externalDatabase.port }} + value: {{ .Values.externalDatabase.port | quote }} {{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + - name: {{ include "dolphinscheduler.image.pullSecrets" . }} + {{- end }} containers: - name: {{ include "dolphinscheduler.fullname" . }}-worker image: {{ include "dolphinscheduler.image.repository" . | quote }} @@ -115,6 +123,8 @@ spec: name: "logs-port" imagePullPolicy: {{ .Values.image.pullPolicy }} env: + - name: DOLPHINSCHEDULER_OPTS + value: {{ default "" .Values.worker.jvmOptions }} - name: TZ value: {{ .Values.timezone }} - name: WORKER_EXEC_THREADS @@ -157,25 +167,37 @@ spec: configMapKeyRef: name: {{ include "dolphinscheduler.fullname" . }}-master key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH - - name: POSTGRESQL_HOST + - name: DATABASE_TYPE + {{- if .Values.postgresql.enabled }} + value: "postgresql" + {{- else }} + value: {{ .Values.externalDatabase.type | quote }} + {{- end }} + - name: DATABASE_DRIVER + {{- if .Values.postgresql.enabled }} + value: "org.postgresql.Driver" + {{- else }} + value: {{ .Values.externalDatabase.driver | quote }} + {{- end }} + - name: DATABASE_HOST {{- if .Values.postgresql.enabled }} value: {{ template "dolphinscheduler.postgresql.fullname" . }} {{- else }} value: {{ .Values.externalDatabase.host | quote }} {{- end }} - - name: POSTGRESQL_PORT + - name: DATABASE_PORT {{- if .Values.postgresql.enabled }} value: "5432" {{- else }} - value: {{ .Values.externalDatabase.port }} + value: {{ .Values.externalDatabase.port | quote }} {{- end }} - - name: POSTGRESQL_USERNAME + - name: DATABASE_USERNAME {{- if .Values.postgresql.enabled }} value: {{ .Values.postgresql.postgresqlUsername }} {{- else }} value: {{ .Values.externalDatabase.username | quote }} {{- end }} - - name: POSTGRESQL_PASSWORD + - name: DATABASE_PASSWORD valueFrom: secretKeyRef: {{- if .Values.postgresql.enabled }} @@ -185,18 +207,133 @@ spec: name: {{ printf "%s-%s" .Release.Name "externaldb" }} key: db-password {{- end }} - - name: POSTGRESQL_DATABASE + - name: DATABASE_DATABASE {{- if .Values.postgresql.enabled }} value: {{ .Values.postgresql.postgresqlDatabase }} {{- else }} value: {{ .Values.externalDatabase.database | quote }} {{- end }} + - name: DATABASE_PARAMS + {{- if .Values.postgresql.enabled }} + value: "characterEncoding=utf8" + {{- else }} + value: {{ .Values.externalDatabase.params | quote }} + {{- end }} - name: ZOOKEEPER_QUORUM {{- if .Values.zookeeper.enabled }} value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}" {{- else }} value: {{ .Values.externalZookeeper.zookeeperQuorum }} {{- end }} + - name: RESOURCE_STORAGE_TYPE + valueFrom: + configMapKeyRef: + key: RESOURCE_STORAGE_TYPE + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: RESOURCE_UPLOAD_PATH + valueFrom: + configMapKeyRef: + key: RESOURCE_UPLOAD_PATH + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_DEFAULT_FS + valueFrom: + configMapKeyRef: + key: FS_DEFAULT_FS + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_ENDPOINT + valueFrom: + configMapKeyRef: + key: FS_S3A_ENDPOINT + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_ACCESS_KEY + valueFrom: + configMapKeyRef: + key: FS_S3A_ACCESS_KEY + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_SECRET_KEY + valueFrom: + configMapKeyRef: + key: FS_S3A_SECRET_KEY + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: XLS_FILE_PATH + valueFrom: + configMapKeyRef: + key: XLS_FILE_PATH + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SERVER_HOST + valueFrom: + configMapKeyRef: + key: MAIL_SERVER_HOST + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SERVER_PORT + valueFrom: + configMapKeyRef: + key: MAIL_SERVER_PORT + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SENDER + valueFrom: + configMapKeyRef: + key: MAIL_SENDER + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_USER + valueFrom: + configMapKeyRef: + key: MAIL_USER + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_PASSWD + valueFrom: + configMapKeyRef: + key: MAIL_PASSWD + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SMTP_STARTTLS_ENABLE + valueFrom: + configMapKeyRef: + key: MAIL_SMTP_STARTTLS_ENABLE + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SMTP_SSL_ENABLE + valueFrom: + configMapKeyRef: + key: MAIL_SMTP_SSL_ENABLE + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SMTP_SSL_TRUST + valueFrom: + configMapKeyRef: + key: MAIL_SMTP_SSL_TRUST + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: ENTERPRISE_WECHAT_ENABLE + valueFrom: + configMapKeyRef: + key: ENTERPRISE_WECHAT_ENABLE + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: ENTERPRISE_WECHAT_CORP_ID + valueFrom: + configMapKeyRef: + key: ENTERPRISE_WECHAT_CORP_ID + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: ENTERPRISE_WECHAT_SECRET + valueFrom: + configMapKeyRef: + key: ENTERPRISE_WECHAT_SECRET + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: ENTERPRISE_WECHAT_AGENT_ID + valueFrom: + configMapKeyRef: + key: ENTERPRISE_WECHAT_AGENT_ID + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: ENTERPRISE_WECHAT_USERS + valueFrom: + configMapKeyRef: + key: ENTERPRISE_WECHAT_USERS + name: {{ include "dolphinscheduler.fullname" . }}-alert + {{- if .Values.worker.resources }} + resources: + limits: + memory: {{ .Values.worker.resources.limits.memory | quote }} + cpu: {{ .Values.worker.resources.limits.cpu | quote }} + requests: + memory: {{ .Values.worker.resources.requests.memory | quote }} + cpu: {{ .Values.worker.resources.requests.cpu | quote }} + {{- end }} {{- if .Values.worker.livenessProbe.enabled }} livenessProbe: exec: diff --git a/docker/kubernetes/dolphinscheduler/values.yaml b/docker/kubernetes/dolphinscheduler/values.yaml index 3cb35c19e0..8acb1d326a 100644 --- a/docker/kubernetes/dolphinscheduler/values.yaml +++ b/docker/kubernetes/dolphinscheduler/values.yaml @@ -29,10 +29,9 @@ image: repository: "dolphinscheduler" tag: "latest" pullPolicy: "IfNotPresent" + pullSecrets: [] -imagePullSecrets: [] - -# If not exists external postgresql, by default, Dolphinscheduler's database will use it. +# If not exists external database, by default, Dolphinscheduler's database will use it. postgresql: enabled: true postgresqlUsername: "root" @@ -43,21 +42,24 @@ postgresql: size: "20Gi" storageClass: "-" -# If exists external postgresql, and set postgresql.enable value to false. -# If postgresql.enable is false, Dolphinscheduler's database will use it. +# If exists external database, and set postgresql.enable value to false. +# external database will be used, otherwise Dolphinscheduler's database will be used. externalDatabase: + type: "postgresql" + driver: "org.postgresql.Driver" host: "localhost" port: "5432" username: "root" password: "root" database: "dolphinscheduler" + # multi params should join with & char + params: "characterEncoding=utf8" # If not exists external zookeeper, by default, Dolphinscheduler's zookeeper will use it. zookeeper: enabled: true taskQueue: "zookeeper" - config: - ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons + config: null service: port: "2181" persistence: @@ -70,6 +72,18 @@ zookeeper: externalZookeeper: taskQueue: "zookeeper" zookeeperQuorum: "127.0.0.1:2181" + zookeeperRoot: "/dolphinscheduler" + +common: + configmap: + DOLPHINSCHEDULER_ENV_PATH: "/tmp/dolphinscheduler/env" + DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler/files" + RESOURCE_STORAGE_TYPE: "NONE" + RESOURCE_UPLOAD_PATH: "/ds" + FS_DEFAULT_FS: "s3a://xxxx" + FS_S3A_ENDPOINT: "s3.xxx.amazonaws.com" + FS_S3A_ACCESS_KEY: "xxxxxxx" + FS_S3A_SECRET_KEY: "xxxxxxx" master: podManagementPolicy: "Parallel" @@ -85,6 +99,18 @@ master: # If specified, the pod's scheduling constraints. # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core affinity: {} + # The jvm options for java instance startup + jvmOptions: "" + resources: {} + # limits: + # memory: "18Gi" + # cpu: "4" + # requests: + # memory: "2Gi" + # cpu: "500m" + # You can use annotations to attach arbitrary non-identifying metadata to objects. + # Clients such as tools and libraries can retrieve this metadata. + annotations: {} ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes configmap: @@ -137,6 +163,18 @@ worker: # If specified, the pod's scheduling constraints. # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core affinity: {} + # The jvm options for java instance startup + jvmOptions: "" + resources: {} + # limits: + # memory: "18Gi" + # cpu: "4" + # requests: + # memory: "2Gi" + # cpu: "500m" + # You can use annotations to attach arbitrary non-identifying metadata to objects. + # Clients such as tools and libraries can retrieve this metadata. + annotations: {} ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: @@ -213,9 +251,22 @@ alert: # If specified, the pod's scheduling constraints. # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core affinity: {} + # The jvm options for java instance startup + jvmOptions: "" + resources: {} + # limits: + # memory: "4Gi" + # cpu: "1" + # requests: + # memory: "2Gi" + # cpu: "500m" + # You can use annotations to attach arbitrary non-identifying metadata to objects. + # Clients such as tools and libraries can retrieve this metadata. + annotations: {} ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes configmap: + ALERT_PLUGIN_DIR: "/opt/dolphinscheduler/alert/plugin" XLS_FILE_PATH: "/tmp/xls" MAIL_SERVER_HOST: "" MAIL_SERVER_PORT: "" @@ -275,6 +326,18 @@ api: # If specified, the pod's scheduling constraints. # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core affinity: {} + # The jvm options for java instance startup + jvmOptions: "" + resources: {} + # limits: + # memory: "4Gi" + # cpu: "2" + # requests: + # memory: "2Gi" + # cpu: "500m" + # You can use annotations to attach arbitrary non-identifying metadata to objects. + # Clients such as tools and libraries can retrieve this metadata. + annotations: {} ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: @@ -322,6 +385,16 @@ frontend: # If specified, the pod's scheduling constraints. # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core affinity: {} + resources: {} + # limits: + # memory: "256Mi" + # cpu: "1" + # requests: + # memory: "256Mi" + # cpu: "500m" + # You can use annotations to attach arbitrary non-identifying metadata to objects. + # Clients such as tools and libraries can retrieve this metadata. + annotations: {} ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java index 38dc38b4a8..36f903c25d 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java @@ -48,8 +48,8 @@ public class EnterpriseWeChatUtils { private static final String ENTERPRISE_WE_CHAT_TOKEN_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TOKEN_URL); private static final String ENTERPRISE_WE_CHAT_TOKEN_URL_REPLACE = ENTERPRISE_WE_CHAT_TOKEN_URL == null ? null : ENTERPRISE_WE_CHAT_TOKEN_URL - .replaceAll("\\$corpId", ENTERPRISE_WE_CHAT_CORP_ID) - .replaceAll("\\$secret", ENTERPRISE_WE_CHAT_SECRET); + .replaceAll("\\{corpId\\}", ENTERPRISE_WE_CHAT_CORP_ID) + .replaceAll("\\{secret\\}", ENTERPRISE_WE_CHAT_SECRET); private static final String ENTERPRISE_WE_CHAT_PUSH_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_PUSH_URL); @@ -120,9 +120,9 @@ public class EnterpriseWeChatUtils { * @return Enterprise WeChat send message */ public static String makeTeamSendMsg(String toParty, String agentId, String msg) { - return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\$toParty", toParty) - .replaceAll("\\$agentId", agentId) - .replaceAll("\\$msg", msg); + return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty\\}", toParty) + .replaceAll("\\{agentId\\}", agentId) + .replaceAll("\\{msg\\}", msg); } /** @@ -135,9 +135,9 @@ public class EnterpriseWeChatUtils { */ public static String makeTeamSendMsg(Collection toParty, String agentId, String msg) { String listParty = FuncUtils.mkString(toParty, "|"); - return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\$toParty", listParty) - .replaceAll("\\$agentId", agentId) - .replaceAll("\\$msg", msg); + return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty\\}", listParty) + .replaceAll("\\{agentId\\}", agentId) + .replaceAll("\\{msg\\}", msg); } /** @@ -149,9 +149,9 @@ public class EnterpriseWeChatUtils { * @return Enterprise WeChat send message */ public static String makeUserSendMsg(String toUser, String agentId, String msg) { - return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\$toUser", toUser) - .replaceAll("\\$agentId", agentId) - .replaceAll("\\$msg", msg); + return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\{toUser\\}", toUser) + .replaceAll("\\{agentId\\}", agentId) + .replaceAll("\\{msg\\}", msg); } /** @@ -164,9 +164,9 @@ public class EnterpriseWeChatUtils { */ public static String makeUserSendMsg(Collection toUser, String agentId, String msg) { String listUser = FuncUtils.mkString(toUser, "|"); - return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\$toUser", listUser) - .replaceAll("\\$agentId", agentId) - .replaceAll("\\$msg", msg); + return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\{toUser\\}", listUser) + .replaceAll("\\{agentId\\}", agentId) + .replaceAll("\\{msg\\}", msg); } /** @@ -179,7 +179,7 @@ public class EnterpriseWeChatUtils { * @throws IOException the IOException */ public static String sendEnterpriseWeChat(String charset, String data, String token) throws IOException { - String enterpriseWeChatPushUrlReplace = ENTERPRISE_WE_CHAT_PUSH_URL.replaceAll("\\$token", token); + String enterpriseWeChatPushUrlReplace = ENTERPRISE_WE_CHAT_PUSH_URL.replaceAll("\\{token\\}", token); CloseableHttpClient httpClient = HttpClients.createDefault(); try { diff --git a/dolphinscheduler-alert/src/main/resources/alert.properties b/dolphinscheduler-alert/src/main/resources/alert.properties index 4e48e7d0e1..6eb701841c 100644 --- a/dolphinscheduler-alert/src/main/resources/alert.properties +++ b/dolphinscheduler-alert/src/main/resources/alert.properties @@ -41,10 +41,10 @@ enterprise.wechat.enable=false #enterprise.wechat.secret=xxxxxxx #enterprise.wechat.agent.id=xxxxxxx #enterprise.wechat.users=xxxxxxx -#enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret -#enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token -#enterprise.wechat.team.send.msg={\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"} -#enterprise.wechat.user.send.msg={\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}} +#enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid={corpId}&corpsecret={secret} +#enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={token} +#enterprise.wechat.team.send.msg={\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"} +#enterprise.wechat.user.send.msg={\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}} plugin.dir=/Users/xx/your/path/to/plugin/dir diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java index 01b81101fd..7b6cdd013b 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java @@ -55,8 +55,8 @@ public class EnterpriseWeChatUtilsTest { private static final String enterpriseWechatUsers="LiGang,journey"; private static final String msg = "hello world"; - private static final String enterpriseWechatTeamSendMsg = "{\\\"toparty\\\":\\\"$toParty\\\",\\\"agentid\\\":\\\"$agentId\\\",\\\"msgtype\\\":\\\"text\\\",\\\"text\\\":{\\\"content\\\":\\\"$msg\\\"},\\\"safe\\\":\\\"0\\\"}"; - private static final String enterpriseWechatUserSendMsg = "{\\\"touser\\\":\\\"$toUser\\\",\\\"agentid\\\":\\\"$agentId\\\",\\\"msgtype\\\":\\\"markdown\\\",\\\"markdown\\\":{\\\"content\\\":\\\"$msg\\\"}}"; + private static final String enterpriseWechatTeamSendMsg = "{\\\"toparty\\\":\\\"{toParty}\\\",\\\"agentid\\\":\\\"{agentId}\\\",\\\"msgtype\\\":\\\"text\\\",\\\"text\\\":{\\\"content\\\":\\\"{msg}\\\"},\\\"safe\\\":\\\"0\\\"}"; + private static final String enterpriseWechatUserSendMsg = "{\\\"touser\\\":\\\"{toUser}\\\",\\\"agentid\\\":\\\"{agentId}\\\",\\\"msgtype\\\":\\\"markdown\\\",\\\"markdown\\\":{\\\"content\\\":\\\"{msg}\\\"}}"; @Before public void init(){ diff --git a/dolphinscheduler-ui/package.json b/dolphinscheduler-ui/package.json index 1f5170637e..a5642c8f97 100644 --- a/dolphinscheduler-ui/package.json +++ b/dolphinscheduler-ui/package.json @@ -63,7 +63,7 @@ "html-loader": "^0.5.5", "html-webpack-plugin": "^3.2.0", "mini-css-extract-plugin": "^0.8.2", - "node-sass": "^4.14.0", + "node-sass": "^4.14.1", "postcss-loader": "^3.0.0", "progress-bar-webpack-plugin": "^1.12.1", "rimraf": "^2.6.2", diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index b0437a30c2..4ffd7fe4c2 100644 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -46,7 +46,7 @@ export DOLPHINSCHEDULER_LOG_DIR=$DOLPHINSCHEDULER_HOME/logs export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/* -export DOLPHINSCHEDULER_OPTS="-server -Xmx16g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" +export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-"-server -Xmx16g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"} export STOP_TIMEOUT=5 if [ ! -d "$DOLPHINSCHEDULER_LOG_DIR" ]; then