From a7aa58e6ca258c817ec53d8ae27743e4904189c7 Mon Sep 17 00:00:00 2001 From: Rubik-W <39549317+Rubik-W@users.noreply.github.com> Date: Wed, 15 Jul 2020 11:21:38 +0800 Subject: [PATCH 1/6] Feature add user state (#2710) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feature: #1923 * fix：docker sql script add column * fix: update sql script * fix: UT bug * fix: UT bug * fix: UT bug * fix: e2e UT bug * fix: e2e button location * fix：sql init script to add t_ds_user.state column Co-authored-by: Rubik-W Co-authored-by: dailidong Co-authored-by: qiaozhanwei --- sql/dolphinscheduler-postgre.sql | 3 ++- sql/dolphinscheduler_mysql.sql | 2 +- sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_ddl.sql | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sql/dolphinscheduler-postgre.sql b/sql/dolphinscheduler-postgre.sql index 3ec802242b..1b27c7b733 100644 --- a/sql/dolphinscheduler-postgre.sql +++ b/sql/dolphinscheduler-postgre.sql @@ -627,6 +627,7 @@ CREATE TABLE t_ds_user ( state int DEFAULT 1 , PRIMARY KEY (id) ); +comment on column t_ds_user.state is 'state 0:disable 1:enable'; -- -- Table structure for table t_ds_version @@ -751,7 +752,7 @@ ALTER TABLE t_ds_worker_server ALTER COLUMN id SET DEFAULT NEXTVAL('t_ds_worker_ -- Records of t_ds_user?user : admin , password : dolphinscheduler123 -INSERT INTO t_ds_user(user_name,user_password,user_type,email,phone,tenant_id,create_time,update_time) VALUES ('admin', '7ad2410b2f4c074479a8937a28a22b8f', '0', 'xxx@qq.com', 'xx', '0', '2018-03-27 15:48:50', '2018-10-24 17:40:22'); +INSERT INTO t_ds_user(user_name,user_password,user_type,email,phone,tenant_id,state,create_time,update_time) VALUES ('admin', '7ad2410b2f4c074479a8937a28a22b8f', '0', 'xxx@qq.com', 'xx', '0', 1, '2018-03-27 15:48:50', '2018-10-24 17:40:22'); -- Records of t_ds_alertgroup，dolphinscheduler warning group INSERT INTO t_ds_alertgroup(group_name,group_type,description,create_time,update_time) VALUES ('dolphinscheduler warning group', '0', 'dolphinscheduler warning group','2018-11-29 10:20:39', '2018-11-29 10:20:39'); diff --git a/sql/dolphinscheduler_mysql.sql b/sql/dolphinscheduler_mysql.sql index d4fa011301..cac32b63e8 100644 --- a/sql/dolphinscheduler_mysql.sql +++ b/sql/dolphinscheduler_mysql.sql @@ -815,4 +815,4 @@ INSERT INTO `t_ds_relation_user_alertgroup` VALUES ('1', '1', '1', '2018-11-29 1 -- ---------------------------- -- Records of t_ds_user -- ---------------------------- -INSERT INTO `t_ds_user` VALUES ('1', 'admin', '7ad2410b2f4c074479a8937a28a22b8f', '0', 'xxx@qq.com', 'xx', '0', '2018-03-27 15:48:50', '2018-10-24 17:40:22', null,1); +INSERT INTO `t_ds_user` VALUES ('1', 'admin', '7ad2410b2f4c074479a8937a28a22b8f', '0', 'xxx@qq.com', 'xx', '0', '2018-03-27 15:48:50', '2018-10-24 17:40:22', null, 1); diff --git a/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_ddl.sql b/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_ddl.sql index 7ea4a342a1..a9e7525ef8 100644 --- a/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_ddl.sql @@ -296,6 +296,7 @@ BEGIN AND COLUMN_NAME ='state') THEN ALTER TABLE t_ds_user ADD COLUMN state int DEFAULT 1; + comment on column t_ds_user.state is 'state 0:disable 1:enable'; END IF; END; $$ LANGUAGE plpgsql; From 3aa34bc72363eab368ade91b6e5be77b67897b02 Mon Sep 17 00:00:00 2001 From: muzhongjiang Date: Thu, 16 Jul 2020 14:56:53 +0800 Subject: [PATCH 2/6] [bugfix] "getNotifyGroupList" cache bug #3179 (#3200) * add state * fixed bug "jackson enum conversion : InvalidFormatException" * Word spelling modification Comment modification Word spelling modification,Comment modification,Log level modification * Update EmailManager.java * Update FlinkParameters.java * Update SqlTask.java * fixed "getNotifyGroupList cache" bug Co-authored-by: mzjnumber1@163.com Co-authored-by: dailidong --- .../js/conf/home/pages/dag/_source/startingParam/index.vue | 5 ----- .../projects/pages/definition/pages/list/_source/start.vue | 6 ------ .../projects/pages/definition/pages/list/_source/timing.vue | 6 ------ 3 files changed, 17 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/startingParam/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/startingParam/index.vue index bea5d92227..516f03bc2f 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/startingParam/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/startingParam/index.vue @@ -76,14 +76,9 @@ return '-' }, _getNotifyGroupList () { - let notifyGroupListS = _.cloneDeep(this.store.state.dag.notifyGroupListS) || [] - if (!notifyGroupListS.length) { this.store.dispatch('dag/getNotifyGroupList').then(res => { this.notifyGroupList = res }) - } else { - this.notifyGroupList = notifyGroupListS - } }, _getWorkerGroupList () { let stateWorkerGroupsList = this.store.state.security.workerGroupsListAll || [] diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue index e47d8c757f..87c0f62cdb 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue @@ -243,16 +243,10 @@ }, _getNotifyGroupList () { return new Promise((resolve, reject) => { - let notifyGroupListS = _.cloneDeep(this.store.state.dag.notifyGroupListS) || [] - if (!notifyGroupListS.length) { this.store.dispatch('dag/getNotifyGroupList').then(res => { this.notifyGroupList = res resolve() }) - } else { - this.notifyGroupList = notifyGroupListS - resolve() - } }) }, _getReceiver () { diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue index 82a82dec45..65b9176c18 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/timing.vue @@ -280,8 +280,6 @@ _getNotifyGroupList () { return new Promise((resolve, reject) => { - let notifyGroupListS = _.cloneDeep(this.store.state.dag.notifyGroupListS) || [] - if (!notifyGroupListS.length) { this.store.dispatch('dag/getNotifyGroupList').then(res => { this.notifyGroupList = res if (this.notifyGroupList.length) { @@ -290,10 +288,6 @@ reject(new Error(0)) } }) - } else { - this.notifyGroupList = notifyGroupListS - resolve() - } }) }, ok () { From b6a0d55949d8379ff5649993361aed858a98ca13 Mon Sep 17 00:00:00 2001 From: rockxsj Date: Thu, 16 Jul 2020 16:36:29 +0800 Subject: [PATCH 3/6] fix and update k8s deploy (#3215) * Modify the AWS S3 request encryption method to V4. * * update Dockerfile to add mysql-client for mysql database service test * update wechat alert config replace placeholder from to {xx} because of the will be treat as env variable * fix unit test of EnterpriseWeChatUtilsTest * update values.yaml default config for helm deploy DS cluster * update values.yaml default config for custom install * update templates of helm for custom install * update some name of env varibale, for example change from POSTGRESQL_HOST to DATABASE_HOST * delele the redundancy requirements.yaml for heml * update README* in the docker dir and subdir * update startup scripts for support mysql as backend database * * fix the ENV name wrong in docker-compose.yml and docker-stack.yml * remove the didn't need port mapping in postgresql & zookeeper & dolphinscheduler-api * * keep expose the port of postgresql & zk & api because of unit test needed * * remove mysql-client package from dockerfile Co-authored-by: rockxsj --- .../DOLPHIN/1.2.1/package/scripts/params.py | 4 +- .../DOLPHIN/1.3.0/package/scripts/params.py | 4 +- docker/build/Dockerfile | 2 +- docker/build/README.md | 66 ++++--- docker/build/README_zh_CN.md | 66 ++++--- .../dolphinscheduler/alert.properties.tpl | 14 +- .../dolphinscheduler/common.properties.tpl | 12 +- .../datasource.properties.tpl | 15 +- .../logback/logback-alert.xml | 2 +- .../dolphinscheduler/zookeeper.properties.tpl | 2 +- docker/build/startup-init-conf.sh | 23 ++- docker/build/startup.sh | 40 +++-- docker/docker-swarm/docker-compose.yml | 40 ++--- docker/docker-swarm/docker-stack.yml | 52 +++--- docker/kubernetes/dolphinscheduler/Chart.yaml | 2 +- docker/kubernetes/dolphinscheduler/README.md | 42 ++++- .../dolphinscheduler/requirements.yaml | 25 --- .../dolphinscheduler/templates/NOTES.txt | 13 -- .../dolphinscheduler/templates/_helpers.tpl | 8 + .../configmap-dolphinscheduler-alert.yaml | 1 + .../configmap-dolphinscheduler-common.yaml | 35 ++++ .../deployment-dolphinscheduler-alert.yaml | 98 +++++++++-- .../deployment-dolphinscheduler-api.yaml | 93 ++++++++-- .../deployment-dolphinscheduler-frontend.yaml | 17 ++ .../statefulset-dolphinscheduler-master.yaml | 93 ++++++++-- .../statefulset-dolphinscheduler-worker.yaml | 163 ++++++++++++++++-- .../kubernetes/dolphinscheduler/values.yaml | 87 +++++++++- .../alert/utils/EnterpriseWeChatUtils.java | 30 ++-- .../src/main/resources/alert.properties | 8 +- .../utils/EnterpriseWeChatUtilsTest.java | 4 +- dolphinscheduler-ui/package.json | 2 +- script/dolphinscheduler-daemon.sh | 2 +- 32 files changed, 795 insertions(+), 270 deletions(-) delete mode 100644 docker/kubernetes/dolphinscheduler/requirements.yaml create mode 100644 docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py b/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py index 93b3249614..049b2cf3ae 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py +++ b/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py @@ -76,8 +76,8 @@ else: dolphin_alert_map = {} wechat_push_url = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token' wechat_token_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret' -wechat_team_send_msg = '{\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"}' -wechat_user_send_msg = '{\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}}' +wechat_team_send_msg = '{\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"}' +wechat_user_send_msg = '{\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}}' dolphin_alert_map['enterprise.wechat.push.ur'] = wechat_push_url dolphin_alert_map['enterprise.wechat.token.url'] = wechat_token_url diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/params.py b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/params.py index 3ca46f2182..5a9994f559 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/params.py +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/params.py @@ -77,8 +77,8 @@ else: dolphin_alert_map = {} wechat_push_url = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token' wechat_token_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret' -wechat_team_send_msg = '{\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"}' -wechat_user_send_msg = '{\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}}' +wechat_team_send_msg = '{\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"}' +wechat_user_send_msg = '{\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}}' dolphin_alert_config_map = config['configurations']['dolphin-alert'] diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile index bb69347797..d0f16d5d0d 100644 --- a/docker/build/Dockerfile +++ b/docker/build/Dockerfile @@ -42,7 +42,7 @@ ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ /opt/dolphinscheduler/ ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler -#4. install pg +#4. install database, if use mysql as your backend database, the `mysql-client` package should be installed RUN apk add postgresql postgresql-contrib #5. modify nginx RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \ diff --git a/docker/build/README.md b/docker/build/README.md index 5725ec56ce..bc516bc214 100644 --- a/docker/build/README.md +++ b/docker/build/README.md @@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org #### You can start a dolphinscheduler instance ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \ +-e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \ -p 8888:8888 \ dolphinscheduler all ``` @@ -25,14 +25,14 @@ The default postgres user `root`, postgres password `root` and database `dolphin The default zookeeper is created in the `startup.sh`. -#### Or via Environment Variables **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`POSTGRESQL_DATABASE`** **`ZOOKEEPER_QUORUM`** +#### Or via Environment Variables **`DATABASE_HOST`** **`DATABASE_PORT`** **`DATABASE_DATABASE`** **`ZOOKEEPER_QUORUM`** You can specify **existing postgres service**. Example: ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ -p 8888:8888 \ dolphinscheduler all ``` @@ -42,7 +42,7 @@ You can specify **existing zookeeper service**. Example: ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \ -p 8888:8888 \ dolphinscheduler all ``` @@ -56,8 +56,8 @@ You can start a standalone dolphinscheduler server. ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ dolphinscheduler master-server ``` @@ -66,8 +66,8 @@ dolphinscheduler master-server ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ dolphinscheduler worker-server ``` @@ -75,8 +75,8 @@ dolphinscheduler worker-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ -p 12345:12345 \ dolphinscheduler api-server ``` @@ -85,8 +85,8 @@ dolphinscheduler api-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ dolphinscheduler alert-server ``` @@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \ dolphinscheduler frontend ``` -**Note**: You must be specify `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server. +**Note**: You must be specify `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server. ## How to build a docker image @@ -124,33 +124,51 @@ Please read `./docker/build/hooks/build` `./docker/build/hooks/build.bat` script The Dolphin Scheduler image uses several environment variables which are easy to miss. While none of the variables are required, they may significantly aid you in using the image. -**`POSTGRESQL_HOST`** +**`DATABASE_TYPE`** -This environment variable sets the host for PostgreSQL. The default value is `127.0.0.1`. +This environment variable sets the type for database. The default value is `postgresql`. **Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. -**`POSTGRESQL_PORT`** +**`DATABASE_DRIVER`** -This environment variable sets the port for PostgreSQL. The default value is `5432`. +This environment variable sets the type for database. The default value is `org.postgresql.Driver`. + +**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. + +**`DATABASE_HOST`** + +This environment variable sets the host for database. The default value is `127.0.0.1`. + +**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. + +**`DATABASE_PORT`** + +This environment variable sets the port for database. The default value is `5432`. **Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. -**`POSTGRESQL_USERNAME`** +**`DATABASE_USERNAME`** + +This environment variable sets the username for database. The default value is `root`. + +**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. + +**`DATABASE_PASSWORD`** -This environment variable sets the username for PostgreSQL. The default value is `root`. +This environment variable sets the password for database. The default value is `root`. **Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. -**`POSTGRESQL_PASSWORD`** +**`DATABASE_DATABASE`** -This environment variable sets the password for PostgreSQL. The default value is `root`. +This environment variable sets the database for database. The default value is `dolphinscheduler`. **Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. -**`POSTGRESQL_DATABASE`** +**`DATABASE_PARAMS`** -This environment variable sets the database for PostgreSQL. The default value is `dolphinscheduler`. +This environment variable sets the database for database. The default value is `characterEncoding=utf8`. **Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md index 0605bef547..c2affc0691 100644 --- a/docker/build/README_zh_CN.md +++ b/docker/build/README_zh_CN.md @@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org #### 你可以运行一个dolphinscheduler实例 ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \ +-e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \ -p 8888:8888 \ dolphinscheduler all ``` @@ -25,14 +25,14 @@ dolphinscheduler all 同时，默认的`Zookeeper`也会在`startup.sh`脚本中被创建。 -#### 或者通过环境变量 **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`ZOOKEEPER_QUORUM`** 使用已存在的服务 +#### 或者通过环境变量 **`DATABASE_HOST`** **`DATABASE_PORT`** **`ZOOKEEPER_QUORUM`** 使用已存在的服务 你可以指定一个已经存在的 **`Postgres`** 服务. 如下: ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ -p 8888:8888 \ dolphinscheduler all ``` @@ -42,7 +42,7 @@ dolphinscheduler all ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \ -p 8888:8888 \ dolphinscheduler all ``` @@ -56,8 +56,8 @@ dolphinscheduler all ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ dolphinscheduler master-server ``` @@ -66,8 +66,8 @@ dolphinscheduler master-server ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ dolphinscheduler worker-server ``` @@ -75,8 +75,8 @@ dolphinscheduler worker-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ -p 12345:12345 \ dolphinscheduler api-server ``` @@ -85,8 +85,8 @@ dolphinscheduler api-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \ +-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \ dolphinscheduler alert-server ``` @@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \ dolphinscheduler frontend ``` -**注意**: 当你运行dolphinscheduler中的部分服务时，你必须指定这些环境变量 `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM`。 +**注意**: 当你运行dolphinscheduler中的部分服务时，你必须指定这些环境变量 `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM`。 ## 如何构建一个docker镜像 @@ -124,33 +124,51 @@ c:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些变量不是必须的，但是可以帮助你更容易配置镜像并根据你的需求定义相应的服务配置。 -**`POSTGRESQL_HOST`** +**`DATABASE_TYPE`** -配置`PostgreSQL`的`HOST`， 默认值 `127.0.0.1`。 +配置`database`的`TYPE`， 默认值 `postgresql`。 **注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 -**`POSTGRESQL_PORT`** +**`DATABASE_DRIVER`** -配置`PostgreSQL`的`PORT`， 默认值 `5432`。 +配置`database`的`DRIVER`， 默认值 `org.postgresql.Driver`。 **注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 -**`POSTGRESQL_USERNAME`** +**`DATABASE_HOST`** -配置`PostgreSQL`的`USERNAME`， 默认值 `root`。 +配置`database`的`HOST`， 默认值 `127.0.0.1`。 **注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 -**`POSTGRESQL_PASSWORD`** +**`DATABASE_PORT`** -配置`PostgreSQL`的`PASSWORD`， 默认值 `root`。 +配置`database`的`PORT`， 默认值 `5432`。 **注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 -**`POSTGRESQL_DATABASE`** +**`DATABASE_USERNAME`** -配置`PostgreSQL`的`DATABASE`， 默认值 `dolphinscheduler`。 +配置`database`的`USERNAME`， 默认值 `root`。 + +**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 + +**`DATABASE_PASSWORD`** + +配置`database`的`PASSWORD`， 默认值 `root`。 + +**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 + +**`DATABASE_DATABASE`** + +配置`database`的`DATABASE`， 默认值 `dolphinscheduler`。 + +**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 + +**`DATABASE_PARAMS`** + +配置`database`的`PARAMS`， 默认值 `characterEncoding=utf8`。 **注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 diff --git a/docker/build/conf/dolphinscheduler/alert.properties.tpl b/docker/build/conf/dolphinscheduler/alert.properties.tpl index b940ecd203..6d7450e13d 100644 --- a/docker/build/conf/dolphinscheduler/alert.properties.tpl +++ b/docker/build/conf/dolphinscheduler/alert.properties.tpl @@ -35,16 +35,22 @@ mail.smtp.ssl.trust=${MAIL_SMTP_SSL_TRUST} #xls file path,need create if not exist xls.file.path=${XLS_FILE_PATH} +# plugins dir +plugin.dir=${ALERT_PLUGIN_DIR} + # Enterprise WeChat configuration enterprise.wechat.enable=${ENTERPRISE_WECHAT_ENABLE} enterprise.wechat.corp.id=${ENTERPRISE_WECHAT_CORP_ID} enterprise.wechat.secret=${ENTERPRISE_WECHAT_SECRET} enterprise.wechat.agent.id=${ENTERPRISE_WECHAT_AGENT_ID} enterprise.wechat.users=${ENTERPRISE_WECHAT_USERS} -enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret -enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token -enterprise.wechat.team.send.msg={\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"} -enterprise.wechat.user.send.msg={\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}} +enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid={corpId}&corpsecret={secret} +enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={token} +enterprise.wechat.team.send.msg={\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"} +enterprise.wechat.user.send.msg={\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}} + + + diff --git a/docker/build/conf/dolphinscheduler/common.properties.tpl b/docker/build/conf/dolphinscheduler/common.properties.tpl index f318ff8414..ff74598fd4 100644 --- a/docker/build/conf/dolphinscheduler/common.properties.tpl +++ b/docker/build/conf/dolphinscheduler/common.properties.tpl @@ -25,13 +25,13 @@ dolphinscheduler.env.path=${DOLPHINSCHEDULER_ENV_PATH} data.basedir.path=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH} # resource upload startup type : HDFS,S3,NONE -resource.storage.type=NONE +resource.storage.type=${RESOURCE_STORAGE_TYPE} #============================================================================ # HDFS #============================================================================ # resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended -#resource.upload.path=/dolphinscheduler +resource.upload.path=${RESOURCE_UPLOAD_PATH} # whether kerberos starts #hadoop.security.authentication.startup.state=false @@ -58,16 +58,16 @@ kerberos.expire.time=7 # S3 #============================================================================ # if resource.storage.type=S3，the value like: s3a://dolphinscheduler ; if resource.storage.type=HDFS, When namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir -fs.defaultFS=hdfs://mycluster:8020 +fs.defaultFS=${FS_DEFAULT_FS} # if resource.storage.type=S3，s3 endpoint -#fs.s3a.endpoint=http://192.168.199.91:9010 +fs.s3a.endpoint=${FS_S3A_ENDPOINT} # if resource.storage.type=S3，s3 access key -#fs.s3a.access.key=A3DXS30FO22544RE +fs.s3a.access.key=${FS_S3A_ACCESS_KEY} # if resource.storage.type=S3，s3 secret key -#fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK +fs.s3a.secret.key=${FS_S3A_SECRET_KEY} # if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty TODO yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx diff --git a/docker/build/conf/dolphinscheduler/datasource.properties.tpl b/docker/build/conf/dolphinscheduler/datasource.properties.tpl index aefb9e3b0b..f7c5ee6881 100644 --- a/docker/build/conf/dolphinscheduler/datasource.properties.tpl +++ b/docker/build/conf/dolphinscheduler/datasource.properties.tpl @@ -15,16 +15,11 @@ # limitations under the License. # - -# mysql -#spring.datasource.driver-class-name=com.mysql.jdbc.Driver -#spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 - -# postgre -spring.datasource.driver-class-name=org.postgresql.Driver -spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8 -spring.datasource.username=${POSTGRESQL_USERNAME} -spring.datasource.password=${POSTGRESQL_PASSWORD} +# db +spring.datasource.driver-class-name=${DATABASE_DRIVER} +spring.datasource.url=jdbc:${DATABASE_TYPE}://${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DATABASE}?${DATABASE_PARAMS} +spring.datasource.username=${DATABASE_USERNAME} +spring.datasource.password=${DATABASE_PASSWORD} ## base spring data source configuration todo need to remove #spring.datasource.type=com.alibaba.druid.pool.DruidDataSource diff --git a/docker/build/conf/dolphinscheduler/logback/logback-alert.xml b/docker/build/conf/dolphinscheduler/logback/logback-alert.xml index 5d1c07858d..1718947dd1 100644 --- a/docker/build/conf/dolphinscheduler/logback/logback-alert.xml +++ b/docker/build/conf/dolphinscheduler/logback/logback-alert.xml @@ -46,7 +46,7 @@ - + \ No newline at end of file diff --git a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl b/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl index 362a4e85ad..51540aa345 100644 --- a/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl +++ b/docker/build/conf/dolphinscheduler/zookeeper.properties.tpl @@ -19,7 +19,7 @@ zookeeper.quorum=${ZOOKEEPER_QUORUM} # dolphinscheduler root directory -#zookeeper.dolphinscheduler.root=/dolphinscheduler +zookeeper.dolphinscheduler.root=${ZOOKEEPER_ROOT} # dolphinscheduler failover directory #zookeeper.session.timeout=60000 diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh index da6eb21b7d..73fdad6798 100644 --- a/docker/build/startup-init-conf.sh +++ b/docker/build/startup-init-conf.sh @@ -24,22 +24,33 @@ echo "init env variables" #============================================================================ # Database Source #============================================================================ -export POSTGRESQL_HOST=${POSTGRESQL_HOST:-"127.0.0.1"} -export POSTGRESQL_PORT=${POSTGRESQL_PORT:-"5432"} -export POSTGRESQL_USERNAME=${POSTGRESQL_USERNAME:-"root"} -export POSTGRESQL_PASSWORD=${POSTGRESQL_PASSWORD:-"root"} -export POSTGRESQL_DATABASE=${POSTGRESQL_DATABASE:-"dolphinscheduler"} +export DATABASE_HOST=${DATABASE_HOST:-"127.0.0.1"} +export DATABASE_PORT=${DATABASE_PORT:-"5432"} +export DATABASE_USERNAME=${DATABASE_USERNAME:-"root"} +export DATABASE_PASSWORD=${DATABASE_PASSWORD:-"root"} +export DATABASE_DATABASE=${DATABASE_DATABASE:-"dolphinscheduler"} +export DATABASE_TYPE=${DATABASE_TYPE:-"postgresql"} +export DATABASE_DRIVER=${DATABASE_DRIVER:-"org.postgresql.Driver"} +export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"} #============================================================================ # System #============================================================================ export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"} export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"} +export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""} +export RESOURCE_STORAGE_TYPE=${RESOURCE_STORAGE_TYPE:-"NONE"} +export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/ds"} +export FS_DEFAULT_FS=${FS_DEFAULT_FS:-"s3a://xxxx"} +export FS_S3A_ENDPOINT=${FS_S3A_ENDPOINT:-"s3.xxx.amazonaws.com"} +export FS_S3A_ACCESS_KEY=${FS_S3A_ACCESS_KEY:-"xxxxxxx"} +export FS_S3A_SECRET_KEY=${FS_S3A_SECRET_KEY:-"xxxxxxx"} #============================================================================ # Zookeeper #============================================================================ export ZOOKEEPER_QUORUM=${ZOOKEEPER_QUORUM:-"127.0.0.1:2181"} +export ZOOKEEPER_ROOT=${ZOOKEEPER_ROOT:-"/dolphinscheduler"} #============================================================================ # Master Server @@ -67,6 +78,8 @@ export WORKER_GROUP=${WORKER_GROUP:-"default"} #============================================================================ # Alert Server #============================================================================ +# alert plugin dir +export ALERT_PLUGIN_DIR=${ALERT_PLUGIN_DIR:-"/opt/dolphinscheduler"} # XLS FILE export XLS_FILE_PATH=${XLS_FILE_PATH:-"/tmp/xls"} # mail diff --git a/docker/build/startup.sh b/docker/build/startup.sh index 00bb9cdc52..ea4fbc0cd1 100644 --- a/docker/build/startup.sh +++ b/docker/build/startup.sh @@ -22,24 +22,32 @@ DOLPHINSCHEDULER_BIN=${DOLPHINSCHEDULER_HOME}/bin DOLPHINSCHEDULER_SCRIPT=${DOLPHINSCHEDULER_HOME}/script DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs -# start postgresql -initPostgreSQL() { - echo "test postgresql service" - while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do +# start database +initDatabase() { + echo "test ${DATABASE_TYPE} service" + while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do counter=$((counter+1)) if [ $counter == 30 ]; then - echo "Error: Couldn't connect to postgresql." + echo "Error: Couldn't connect to ${DATABASE_TYPE}." exit 1 fi - echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter." + echo "Trying to connect to ${DATABASE_TYPE} at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter." sleep 5 done - echo "connect postgresql service" - v=$(sudo -u postgres PGPASSWORD=${POSTGRESQL_PASSWORD} psql -h ${POSTGRESQL_HOST} -p ${POSTGRESQL_PORT} -U ${POSTGRESQL_USERNAME} -d dolphinscheduler -tAc "select 1") - if [ "$(echo '${v}' | grep 'FATAL' | wc -l)" -eq 1 ]; then - echo "Error: Can't connect to database...${v}" - exit 1 + echo "connect ${DATABASE_TYPE} service" + if [ ${DATABASE_TYPE} = "mysql" ]; then + v=$(mysql -h${DATABASE_HOST} -P${DATABASE_PORT} -u${DATABASE_USERNAME} --password=${DATABASE_PASSWORD} -D ${DATABASE_DATABASE} -e "select 1" 2>&1) + if [ "$(echo '${v}' | grep 'ERROR' | wc -l)" -eq 1 ]; then + echo "Error: Can't connect to database...${v}" + exit 1 + fi + else + v=$(sudo -u postgres PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "select 1") + if [ "$(echo '${v}' | grep 'FATAL' | wc -l)" -eq 1 ]; then + echo "Error: Can't connect to database...${v}" + exit 1 + fi fi echo "import sql data" @@ -123,7 +131,7 @@ LOGFILE=/var/log/nginx/access.log case "$1" in (all) initZK - initPostgreSQL + initDatabase initMasterServer initWorkerServer initApiServer @@ -134,25 +142,25 @@ case "$1" in ;; (master-server) initZK - initPostgreSQL + initDatabase initMasterServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-master.log ;; (worker-server) initZK - initPostgreSQL + initDatabase initWorkerServer initLoggerServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log ;; (api-server) initZK - initPostgreSQL + initDatabase initApiServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log ;; (alert-server) - initPostgreSQL + initDatabase initAlertServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-alert.log ;; diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index 5cb6717f15..51eb0aeaa5 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -56,11 +56,11 @@ services: - 12345:12345 environment: TZ: Asia/Shanghai - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 healthcheck: test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] @@ -119,11 +119,11 @@ services: ENTERPRISE_WECHAT_SECRET: "" ENTERPRISE_WECHAT_AGENT_ID: "" ENTERPRISE_WECHAT_USERS: "" - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler healthcheck: test: ["CMD", "/root/checkpoint.sh", "AlertServer"] interval: 30s @@ -152,11 +152,11 @@ services: MASTER_TASK_COMMIT_INTERVAL: "1000" MASTER_MAX_CPULOAD_AVG: "100" MASTER_RESERVED_MEMORY: "0.1" - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 healthcheck: test: ["CMD", "/root/checkpoint.sh", "MasterServer"] @@ -188,11 +188,11 @@ services: WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUP: "default" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 healthcheck: test: ["CMD", "/root/checkpoint.sh", "WorkerServer"] diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml index 5c3873fb07..ca9f7c88c7 100644 --- a/docker/docker-swarm/docker-stack.yml +++ b/docker/docker-swarm/docker-stack.yml @@ -20,13 +20,13 @@ services: dolphinscheduler-postgresql: image: bitnami/postgresql:latest - ports: - - 5432:5432 environment: TZ: Asia/Shanghai POSTGRESQL_USERNAME: root POSTGRESQL_PASSWORD: root POSTGRESQL_DATABASE: dolphinscheduler + ports: + - 5432:5432 volumes: - dolphinscheduler-postgresql:/bitnami/postgresql networks: @@ -37,12 +37,12 @@ services: dolphinscheduler-zookeeper: image: bitnami/zookeeper:latest - ports: - - 2181:2181 environment: TZ: Asia/Shanghai ALLOW_ANONYMOUS_LOGIN: "yes" ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons + ports: + - 2181:2181 volumes: - dolphinscheduler-zookeeper:/bitnami/zookeeper networks: @@ -54,16 +54,16 @@ services: dolphinscheduler-api: image: apache/dolphinscheduler:latest command: ["api-server"] - ports: - - 12345:12345 environment: TZ: Asia/Shanghai - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 + ports: + - 12345:12345 healthcheck: test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"] interval: 30 @@ -120,11 +120,11 @@ services: ENTERPRISE_WECHAT_SECRET: "" ENTERPRISE_WECHAT_AGENT_ID: "" ENTERPRISE_WECHAT_USERS: "" - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler healthcheck: test: ["CMD", "/root/checkpoint.sh", "AlertServer"] interval: 30 @@ -153,11 +153,11 @@ services: MASTER_TASK_COMMIT_INTERVAL: "1000" MASTER_MAX_CPULOAD_AVG: "100" MASTER_RESERVED_MEMORY: "0.1" - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 healthcheck: test: ["CMD", "/root/checkpoint.sh", "MasterServer"] @@ -188,11 +188,11 @@ services: WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUP: "default" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" - POSTGRESQL_HOST: dolphinscheduler-postgresql - POSTGRESQL_PORT: 5432 - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler + DATABASE_HOST: dolphinscheduler-postgresql + DATABASE_PORT: 5432 + DATABASE_USERNAME: root + DATABASE_PASSWORD: root + DATABASE_DATABASE: dolphinscheduler ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 healthcheck: test: ["CMD", "/root/checkpoint.sh", "WorkerServer"] diff --git a/docker/kubernetes/dolphinscheduler/Chart.yaml b/docker/kubernetes/dolphinscheduler/Chart.yaml index ac989d571f..9d640869dd 100644 --- a/docker/kubernetes/dolphinscheduler/Chart.yaml +++ b/docker/kubernetes/dolphinscheduler/Chart.yaml @@ -49,4 +49,4 @@ dependencies: - name: zookeeper version: 5.x.x repository: https://charts.bitnami.com/bitnami - condition: redis.enabled + condition: zookeeper.enabled diff --git a/docker/kubernetes/dolphinscheduler/README.md b/docker/kubernetes/dolphinscheduler/README.md index 9e6d1c6448..0691b1ed2e 100644 --- a/docker/kubernetes/dolphinscheduler/README.md +++ b/docker/kubernetes/dolphinscheduler/README.md @@ -46,8 +46,8 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `image.repository` | Docker image repository for the Dolphins Scheduler | `dolphinscheduler` | | `image.tag` | Docker image version for the Dolphins Scheduler | `1.2.1` | | `image.imagePullPolicy` | Image pull policy. One of Always, Never, IfNotPresent | `IfNotPresent` | -| `imagePullSecrets` | ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images | `[]` | -| | | | +| `image.pullSecres` | PullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images | `[]` | +| | | | | `postgresql.enabled` | If not exists external PostgreSQL, by default, the Dolphins Scheduler will use a internal PostgreSQL | `true` | | `postgresql.postgresqlUsername` | The username for internal PostgreSQL | `root` | | `postgresql.postgresqlPassword` | The password for internal PostgreSQL | `root` | @@ -55,12 +55,15 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `postgresql.persistence.enabled` | Set `postgresql.persistence.enabled` to `true` to mount a new volume for internal PostgreSQL | `false` | | `postgresql.persistence.size` | `PersistentVolumeClaim` Size | `20Gi` | | `postgresql.persistence.storageClass` | PostgreSQL data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | -| `externalDatabase.host` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database host will use it. | `localhost` | +| `externalDatabase.type` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database type will use it. | `postgresql` | +| `externalDatabase.driver` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database driver will use it. | `org.postgresql.Driver` | +| `externalDatabase.host` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database host will use it. | `localhost` | | `externalDatabase.port` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database port will use it. | `5432` | | `externalDatabase.username` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database username will use it. | `root` | | `externalDatabase.password` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database password will use it. | `root` | | `externalDatabase.database` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database database will use it. | `dolphinscheduler` | -| | | | +| `externalDatabase.params` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database params will use it. | `characterEncoding=utf8` | +| | | | | `zookeeper.enabled` | If not exists external Zookeeper, by default, the Dolphin Scheduler will use a internal Zookeeper | `true` | | `zookeeper.taskQueue` | Specify task queue for `master` and `worker` | `zookeeper` | | `zookeeper.persistence.enabled` | Set `zookeeper.persistence.enabled` to `true` to mount a new volume for internal Zookeeper | `false` | @@ -68,12 +71,25 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `zookeeper.persistence.storageClass` | Zookeeper data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | | `externalZookeeper.taskQueue` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify task queue for `master` and `worker` | `zookeeper` | | `externalZookeeper.zookeeperQuorum` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify Zookeeper quorum | `127.0.0.1:2181` | -| | | | +| `externalZookeeper.zookeeperRoot` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify Zookeeper root path for `master` and `worker` | `dolphinscheduler` | +| | | | +| `common.configmap.DOLPHINSCHEDULER_ENV_PATH` | Extra env file path. | `/tmp/dolphinscheduler/env` | +| `common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH` | File uploaded path of DS. | `/tmp/dolphinscheduler/files` | +| `common.configmap.RESOURCE_STORAGE_TYPE` | Resource Storate type, support type are: S3、HDFS、NONE. | `NONE` | +| `common.configmap.RESOURCE_UPLOAD_PATH` | The base path of resource. | `/ds` | +| `common.configmap.FS_DEFAULT_FS` | The default fs of resource, for s3 is the `s3a` prefix and bucket name. | `s3a://xxxx` | +| `common.configmap.FS_S3A_ENDPOINT` | If the resource type is `S3`, you should fill this filed, it's the endpoint of s3. | `s3.xxx.amazonaws.com` | +| `common.configmap.FS_S3A_ACCESS_KEY` | The access key for your s3 bucket. | `xxxxxxx` | +| `common.configmap.FS_S3A_SECRET_KEY` | The secret key for your s3 bucket. | `xxxxxxx` | | `master.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` | +| | | | | `master.replicas` | Replicas is the desired number of replicas of the given Template | `3` | | `master.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | | `master.tolerations` | If specified, the pod's tolerations | `{}` | | `master.affinity` | If specified, the pod's scheduling constraints | `{}` | +| `master.jvmOptions` | The JVM options for master server. | `""` | +| `master.resources` | The `resource` limit and request config for master server. | `{}` | +| `master.annotations` | The `annotations` for master server. | `{}` | | `master.configmap.MASTER_EXEC_THREADS` | Master execute thread num | `100` | | `master.configmap.MASTER_EXEC_TASK_NUM` | Master execute task number in parallel | `20` | | `master.configmap.MASTER_HEARTBEAT_INTERVAL` | Master heartbeat interval | `10` | @@ -97,12 +113,15 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `master.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | | `master.persistentVolumeClaim.storageClassName` | `Master` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | | `master.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` | -| | | | +| | | | | `worker.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` | | `worker.replicas` | Replicas is the desired number of replicas of the given Template | `3` | | `worker.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | | `worker.tolerations` | If specified, the pod's tolerations | `{}` | | `worker.affinity` | If specified, the pod's scheduling constraints | `{}` | +| `worker.jvmOptions` | The JVM options for worker server. | `""` | +| `worker.resources` | The `resource` limit and request config for worker server. | `{}` | +| `worker.annotations` | The `annotations` for worker server. | `{}` | | `worker.configmap.WORKER_EXEC_THREADS` | Worker execute thread num | `100` | | `worker.configmap.WORKER_HEARTBEAT_INTERVAL` | Worker heartbeat interval | `10` | | `worker.configmap.WORKER_FETCH_TASK_NUM` | Submit the number of tasks at a time | `3` | @@ -131,7 +150,7 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `worker.persistentVolumeClaim.logsPersistentVolume.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | | `worker.persistentVolumeClaim.logsPersistentVolume.storageClassName` | `Worker` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | | `worker.persistentVolumeClaim.logsPersistentVolume.storage` | `PersistentVolumeClaim` Size | `20Gi` | -| | | | +| | | | | `alert.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` | | `alert.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` | | `alert.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` | @@ -139,6 +158,10 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `alert.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | | `alert.tolerations` | If specified, the pod's tolerations | `{}` | | `alert.affinity` | If specified, the pod's scheduling constraints | `{}` | +| `alert.jvmOptions` | The JVM options for alert server. | `""` | +| `alert.resources` | The `resource` limit and request config for alert server. | `{}` | +| `alert.annotations` | The `annotations` for alert server. | `{}` | +| `alert.configmap.ALERT_PLUGIN_DIR` | Alert plugin path. | `/opt/dolphinscheduler/alert/plugin` | | `alert.configmap.XLS_FILE_PATH` | XLS file path | `/tmp/xls` | | `alert.configmap.MAIL_SERVER_HOST` | Mail `SERVER HOST ` | `nil` | | `alert.configmap.MAIL_SERVER_PORT` | Mail `SERVER PORT` | `nil` | @@ -177,6 +200,9 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `api.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | | `api.tolerations` | If specified, the pod's tolerations | `{}` | | `api.affinity` | If specified, the pod's scheduling constraints | `{}` | +| `api.jvmOptions` | The JVM options for api server. | `""` | +| `api.resources` | The `resource` limit and request config for api server. | `{}` | +| `api.annotations` | The `annotations` for api server. | `{}` | | `api.livenessProbe.enabled` | Turn on and off liveness probe | `true` | | `api.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | | `api.livenessProbe.periodSeconds` | How often to perform the probe | `30` | @@ -201,6 +227,8 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `frontend.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | | `frontend.tolerations` | If specified, the pod's tolerations | `{}` | | `frontend.affinity` | If specified, the pod's scheduling constraints | `{}` | +| `frontend.resources` | The `resource` limit and request config for frontend server. | `{}` | +| `frontend.annotations` | The `annotations` for frontend server. | `{}` | | `frontend.livenessProbe.enabled` | Turn on and off liveness probe | `true` | | `frontend.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | | `frontend.livenessProbe.periodSeconds` | How often to perform the probe | `30` | diff --git a/docker/kubernetes/dolphinscheduler/requirements.yaml b/docker/kubernetes/dolphinscheduler/requirements.yaml deleted file mode 100644 index e219975995..0000000000 --- a/docker/kubernetes/dolphinscheduler/requirements.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -dependencies: -- name: postgresql - version: 8.x.x - repository: https://charts.bitnami.com/bitnami - condition: postgresql.enabled -- name: zookeeper - version: 5.x.x - repository: https://charts.bitnami.com/bitnami - condition: redis.enabled \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/NOTES.txt b/docker/kubernetes/dolphinscheduler/templates/NOTES.txt index eb3a9cfc52..256c53ca57 100644 --- a/docker/kubernetes/dolphinscheduler/templates/NOTES.txt +++ b/docker/kubernetes/dolphinscheduler/templates/NOTES.txt @@ -29,16 +29,3 @@ kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-frontend 8888:8888 {{- end }} - -2. Get the Dolphinscheduler URL by running: - -{{- if .Values.ingress.enabled }} - - export HOSTNAME=$(kubectl get ingress --namespace {{ .Release.Namespace }} {{ template "dolphinscheduler.fullname" . }} -o jsonpath='{.spec.rules[0].host}') - echo "Dolphinscheduler URL: http://$HOSTNAME/" - -{{- else }} - - kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "dolphinscheduler.fullname" . }}-frontend 8888:8888 - -{{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl index 9ba290b771..1f121dfea8 100644 --- a/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl +++ b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl @@ -96,6 +96,14 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- printf "%s/%s:%s" (include "dolphinscheduler.image.registry" .) .Values.image.repository .Values.image.tag -}} {{- end -}} +{{/* +Create a default image pull secrects. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "dolphinscheduler.image.pullSecrets" -}} +{{- default nil .Values.image.pullSecrets -}} +{{- end -}} + {{/* Create a default fully qualified postgresql name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml index 76daad8568..2c5b76c3f0 100644 --- a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml @@ -24,6 +24,7 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} data: + ALERT_PLUGIN_DIR: {{ .Values.alert.configmap.ALERT_PLUGIN_DIR | quote }} XLS_FILE_PATH: {{ .Values.alert.configmap.XLS_FILE_PATH | quote }} MAIL_SERVER_HOST: {{ .Values.alert.configmap.MAIL_SERVER_HOST | quote }} MAIL_SERVER_PORT: {{ .Values.alert.configmap.MAIL_SERVER_PORT | quote }} diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml new file mode 100644 index 0000000000..931f6d48e4 --- /dev/null +++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-common.yaml @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- if .Values.common.configmap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "dolphinscheduler.fullname" . }}-common + labels: + app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-common + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +data: + DOLPHINSCHEDULER_ENV_PATH: {{ .Values.common.configmap.DOLPHINSCHEDULER_ENV_PATH | quote }} + DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ .Values.common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH | quote }} + RESOURCE_STORAGE_TYPE: {{ .Values.common.configmap.RESOURCE_STORAGE_TYPE | quote }} + RESOURCE_UPLOAD_PATH: {{ .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }} + FS_DEFAULT_FS: {{ .Values.common.configmap.FS_DEFAULT_FS | quote }} + FS_S3A_ENDPOINT: {{ .Values.common.configmap.FS_S3A_ENDPOINT | quote }} + FS_S3A_ACCESS_KEY: {{ .Values.common.configmap.FS_S3A_ACCESS_KEY | quote }} + FS_S3A_SECRET_KEY: {{ .Values.common.configmap.FS_S3A_SECRET_KEY | quote }} +{{- end }} \ No newline at end of file diff --git a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml index 69662e95d9..fb9a23786b 100644 --- a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml @@ -43,6 +43,10 @@ spec: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/component: alert + {{- if .Values.alert.annotations }} + annotations: + {{- toYaml .Values.alert.annotations | nindent 8 }} + {{- end }} spec: {{- if .Values.alert.affinity }} affinity: {{- toYaml .Values.alert.affinity | nindent 8 }} @@ -54,34 +58,38 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} initContainers: - - name: init-postgresql + - name: init-database image: busybox:1.31.0 command: - /bin/sh - -ec - | - while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do + while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do counter=$((counter+1)) if [ $counter == 5 ]; then - echo "Error: Couldn't connect to postgresql." + echo "Error: Couldn't connect to database." exit 1 fi - echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter." + echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter." sleep 60 done env: - - name: POSTGRESQL_HOST + - name: DATABASE_HOST {{- if .Values.postgresql.enabled }} value: {{ template "dolphinscheduler.postgresql.fullname" . }} {{- else }} value: {{ .Values.externalDatabase.host | quote }} {{- end }} - - name: POSTGRESQL_PORT + - name: DATABASE_PORT {{- if .Values.postgresql.enabled }} value: "5432" {{- else }} - value: {{ .Values.externalDatabase.port }} + value: {{ .Values.externalDatabase.port | quote }} {{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + - name: {{ include "dolphinscheduler.image.pullSecrets" . }} + {{- end }} containers: - name: {{ include "dolphinscheduler.fullname" . }}-alert image: {{ include "dolphinscheduler.image.repository" . | quote }} @@ -89,8 +97,15 @@ spec: - "alert-server" imagePullPolicy: {{ .Values.image.pullPolicy }} env: + - name: DOLPHINSCHEDULER_OPTS + value: {{ default "" .Values.alert.jvmOptions }} - name: TZ value: {{ .Values.timezone }} + - name: ALERT_PLUGIN_DIR + valueFrom: + configMapKeyRef: + key: ALERT_PLUGIN_DIR + name: {{ include "dolphinscheduler.fullname" . }}-alert - name: XLS_FILE_PATH valueFrom: configMapKeyRef: @@ -161,25 +176,37 @@ spec: configMapKeyRef: key: ENTERPRISE_WECHAT_USERS name: {{ include "dolphinscheduler.fullname" . }}-alert - - name: POSTGRESQL_HOST + - name: DATABASE_TYPE + {{- if .Values.postgresql.enabled }} + value: "postgresql" + {{- else }} + value: {{ .Values.externalDatabase.type | quote }} + {{- end }} + - name: DATABASE_DRIVER + {{- if .Values.postgresql.enabled }} + value: "org.postgresql.Driver" + {{- else }} + value: {{ .Values.externalDatabase.driver | quote }} + {{- end }} + - name: DATABASE_HOST {{- if .Values.postgresql.enabled }} value: {{ template "dolphinscheduler.postgresql.fullname" . }} {{- else }} value: {{ .Values.externalDatabase.host | quote }} {{- end }} - - name: POSTGRESQL_PORT + - name: DATABASE_PORT {{- if .Values.postgresql.enabled }} value: "5432" {{- else }} - value: {{ .Values.externalDatabase.port }} + value: {{ .Values.externalDatabase.port | quote }} {{- end }} - - name: POSTGRESQL_USERNAME + - name: DATABASE_USERNAME {{- if .Values.postgresql.enabled }} value: {{ .Values.postgresql.postgresqlUsername }} {{- else }} value: {{ .Values.externalDatabase.username | quote }} {{- end }} - - name: POSTGRESQL_PASSWORD + - name: DATABASE_PASSWORD valueFrom: secretKeyRef: {{- if .Values.postgresql.enabled }} @@ -189,12 +216,57 @@ spec: name: {{ printf "%s-%s" .Release.Name "externaldb" }} key: db-password {{- end }} - - name: POSTGRESQL_DATABASE + - name: DATABASE_DATABASE {{- if .Values.postgresql.enabled }} value: {{ .Values.postgresql.postgresqlDatabase }} {{- else }} value: {{ .Values.externalDatabase.database | quote }} {{- end }} + - name: DATABASE_PARAMS + {{- if .Values.postgresql.enabled }} + value: "characterEncoding=utf8" + {{- else }} + value: {{ .Values.externalDatabase.params | quote }} + {{- end }} + - name: RESOURCE_STORAGE_TYPE + valueFrom: + configMapKeyRef: + key: RESOURCE_STORAGE_TYPE + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: RESOURCE_UPLOAD_PATH + valueFrom: + configMapKeyRef: + key: RESOURCE_UPLOAD_PATH + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_DEFAULT_FS + valueFrom: + configMapKeyRef: + key: FS_DEFAULT_FS + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_ENDPOINT + valueFrom: + configMapKeyRef: + key: FS_S3A_ENDPOINT + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_ACCESS_KEY + valueFrom: + configMapKeyRef: + key: FS_S3A_ACCESS_KEY + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_SECRET_KEY + valueFrom: + configMapKeyRef: + key: FS_S3A_SECRET_KEY + name: {{ include "dolphinscheduler.fullname" . }}-common + {{- if .Values.alert.resources }} + resources: + limits: + memory: {{ .Values.alert.resources.limits.memory }} + cpu: {{ .Values.alert.resources.limits.cpu }} + requests: + memory: {{ .Values.alert.resources.requests.memory }} + cpu: {{ .Values.alert.resources.requests.cpu }} + {{- end }} {{- if .Values.alert.livenessProbe.enabled }} livenessProbe: exec: diff --git a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml index 487ede0b8f..51a83bcfa7 100644 --- a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml @@ -43,6 +43,10 @@ spec: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/component: api + {{- if .Values.alert.annotations }} + annotations: + {{- toYaml .Values.alert.annotations | nindent 8 }} + {{- end }} spec: {{- if .Values.api.affinity }} affinity: {{- toYaml .Values.api.affinity | nindent 8 }} @@ -54,34 +58,38 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} initContainers: - - name: init-postgresql + - name: init-database image: busybox:1.31.0 command: - /bin/sh - -ec - | - while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do + while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do counter=$((counter+1)) if [ $counter == 5 ]; then - echo "Error: Couldn't connect to postgresql." + echo "Error: Couldn't connect to database." exit 1 fi - echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter." + echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter." sleep 60 done env: - - name: POSTGRESQL_HOST + - name: DATABASE_HOST {{- if .Values.postgresql.enabled }} value: {{ template "dolphinscheduler.postgresql.fullname" . }} {{- else }} value: {{ .Values.externalDatabase.host | quote }} {{- end }} - - name: POSTGRESQL_PORT + - name: DATABASE_PORT {{- if .Values.postgresql.enabled }} value: "5432" {{- else }} - value: {{ .Values.externalDatabase.port }} + value: {{ .Values.externalDatabase.port | quote }} {{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + - name: {{ include "dolphinscheduler.image.pullSecrets" . }} + {{- end }} containers: - name: {{ include "dolphinscheduler.fullname" . }}-api image: {{ include "dolphinscheduler.image.repository" . | quote }} @@ -92,27 +100,41 @@ spec: name: tcp-port imagePullPolicy: {{ .Values.image.pullPolicy }} env: + - name: DOLPHINSCHEDULER_OPTS + value: {{ default "" .Values.api.jvmOptions }} - name: TZ value: {{ .Values.timezone }} - - name: POSTGRESQL_HOST + - name: DATABASE_TYPE + {{- if .Values.postgresql.enabled }} + value: "postgresql" + {{- else }} + value: {{ .Values.externalDatabase.type | quote }} + {{- end }} + - name: DATABASE_DRIVER + {{- if .Values.postgresql.enabled }} + value: "org.postgresql.Driver" + {{- else }} + value: {{ .Values.externalDatabase.driver | quote }} + {{- end }} + - name: DATABASE_HOST {{- if .Values.postgresql.enabled }} value: {{ template "dolphinscheduler.postgresql.fullname" . }} {{- else }} value: {{ .Values.externalDatabase.host | quote }} {{- end }} - - name: POSTGRESQL_PORT + - name: DATABASE_PORT {{- if .Values.postgresql.enabled }} value: "5432" {{- else }} - value: {{ .Values.externalDatabase.port }} + value: {{ .Values.externalDatabase.port | quote }} {{- end }} - - name: POSTGRESQL_USERNAME + - name: DATABASE_USERNAME {{- if .Values.postgresql.enabled }} value: {{ .Values.postgresql.postgresqlUsername }} {{- else }} value: {{ .Values.externalDatabase.username | quote }} {{- end }} - - name: POSTGRESQL_PASSWORD + - name: DATABASE_PASSWORD valueFrom: secretKeyRef: {{- if .Values.postgresql.enabled }} @@ -122,18 +144,63 @@ spec: name: {{ printf "%s-%s" .Release.Name "externaldb" }} key: db-password {{- end }} - - name: POSTGRESQL_DATABASE + - name: DATABASE_DATABASE {{- if .Values.postgresql.enabled }} value: {{ .Values.postgresql.postgresqlDatabase }} {{- else }} value: {{ .Values.externalDatabase.database | quote }} {{- end }} + - name: DATABASE_PARAMS + {{- if .Values.postgresql.enabled }} + value: "characterEncoding=utf8" + {{- else }} + value: {{ .Values.externalDatabase.params | quote }} + {{- end }} - name: ZOOKEEPER_QUORUM {{- if .Values.zookeeper.enabled }} value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}" {{- else }} value: {{ .Values.externalZookeeper.zookeeperQuorum }} {{- end }} + - name: RESOURCE_STORAGE_TYPE + valueFrom: + configMapKeyRef: + key: RESOURCE_STORAGE_TYPE + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: RESOURCE_UPLOAD_PATH + valueFrom: + configMapKeyRef: + key: RESOURCE_UPLOAD_PATH + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_DEFAULT_FS + valueFrom: + configMapKeyRef: + key: FS_DEFAULT_FS + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_ENDPOINT + valueFrom: + configMapKeyRef: + key: FS_S3A_ENDPOINT + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_ACCESS_KEY + valueFrom: + configMapKeyRef: + key: FS_S3A_ACCESS_KEY + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_SECRET_KEY + valueFrom: + configMapKeyRef: + key: FS_S3A_SECRET_KEY + name: {{ include "dolphinscheduler.fullname" . }}-common + {{- if .Values.api.resources }} + resources: + limits: + memory: {{ .Values.api.resources.limits.memory | quote }} + cpu: {{ .Values.api.resources.limits.cpu | quote }} + requests: + memory: {{ .Values.api.resources.requests.memory | quote }} + cpu: {{ .Values.api.resources.requests.cpu | quote }} + {{- end }} {{- if .Values.api.livenessProbe.enabled }} livenessProbe: tcpSocket: diff --git a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml index aea09f107f..04b9408d9f 100644 --- a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-frontend.yaml @@ -43,6 +43,10 @@ spec: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/component: frontend + {{- if .Values.alert.annotations }} + annotations: + {{- toYaml .Values.alert.annotations | nindent 8 }} + {{- end }} spec: {{- if .Values.frontend.affinity }} affinity: {{- toYaml .Values.frontend.affinity | nindent 8 }} @@ -53,6 +57,10 @@ spec: {{- if .Values.frontend.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + - name: {{ include "dolphinscheduler.image.pullSecrets" . }} + {{- end }} containers: - name: {{ include "dolphinscheduler.fullname" . }}-frontend image: {{ include "dolphinscheduler.image.repository" . | quote }} @@ -69,6 +77,15 @@ spec: value: '{{ include "dolphinscheduler.fullname" . }}-api' - name: FRONTEND_API_SERVER_PORT value: "12345" + {{- if .Values.frontend.resources }} + resources: + limits: + memory: {{ .Values.frontend.resources.limits.memory | quote }} + cpu: {{ .Values.frontend.resources.limits.cpu | quote }} + requests: + memory: {{ .Values.frontend.resources.requests.memory | quote }} + cpu: {{ .Values.frontend.resources.requests.cpu | quote }} + {{- end }} {{- if .Values.frontend.livenessProbe.enabled }} livenessProbe: tcpSocket: diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml index b142fac4fe..0949127dda 100644 --- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml @@ -40,6 +40,10 @@ spec: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/component: master + {{- if .Values.alert.annotations }} + annotations: + {{- toYaml .Values.alert.annotations | nindent 8 }} + {{- end }} spec: {{- if .Values.master.affinity }} affinity: {{- toYaml .Values.master.affinity | nindent 8 }} @@ -75,34 +79,38 @@ spec: {{- else }} value: {{ .Values.externalZookeeper.zookeeperQuorum }} {{- end }} - - name: init-postgresql + - name: init-database image: busybox:1.31.0 command: - /bin/sh - -ec - | - while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do + while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do counter=$((counter+1)) if [ $counter == 5 ]; then - echo "Error: Couldn't connect to postgresql." + echo "Error: Couldn't connect to database." exit 1 fi - echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter." + echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter." sleep 60 done env: - - name: POSTGRESQL_HOST + - name: DATABASE_HOST {{- if .Values.postgresql.enabled }} value: {{ template "dolphinscheduler.postgresql.fullname" . }} {{- else }} value: {{ .Values.externalDatabase.host | quote }} {{- end }} - - name: POSTGRESQL_PORT + - name: DATABASE_PORT {{- if .Values.postgresql.enabled }} value: "5432" {{- else }} - value: {{ .Values.externalDatabase.port }} + value: {{ .Values.externalDatabase.port | quote }} {{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + - name: {{ include "dolphinscheduler.image.pullSecrets" . }} + {{- end }} containers: - name: {{ include "dolphinscheduler.fullname" . }}-master image: {{ include "dolphinscheduler.image.repository" . | quote }} @@ -113,6 +121,8 @@ spec: name: "master-port" imagePullPolicy: {{ .Values.image.pullPolicy }} env: + - name: DOLPHINSCHEDULER_OPTS + value: {{ default "" .Values.master.jvmOptions }} - name: TZ value: {{ .Values.timezone }} - name: MASTER_EXEC_THREADS @@ -160,25 +170,37 @@ spec: configMapKeyRef: name: {{ include "dolphinscheduler.fullname" . }}-master key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH - - name: POSTGRESQL_HOST + - name: DATABASE_TYPE + {{- if .Values.postgresql.enabled }} + value: "postgresql" + {{- else }} + value: {{ .Values.externalDatabase.type | quote }} + {{- end }} + - name: DATABASE_DRIVER + {{- if .Values.postgresql.enabled }} + value: "org.postgresql.Driver" + {{- else }} + value: {{ .Values.externalDatabase.driver | quote }} + {{- end }} + - name: DATABASE_HOST {{- if .Values.postgresql.enabled }} value: {{ template "dolphinscheduler.postgresql.fullname" . }} {{- else }} value: {{ .Values.externalDatabase.host | quote }} {{- end }} - - name: POSTGRESQL_PORT + - name: DATABASE_PORT {{- if .Values.postgresql.enabled }} value: "5432" {{- else }} - value: {{ .Values.externalDatabase.port }} + value: {{ .Values.externalDatabase.port | quote }} {{- end }} - - name: POSTGRESQL_USERNAME + - name: DATABASE_USERNAME {{- if .Values.postgresql.enabled }} value: {{ .Values.postgresql.postgresqlUsername }} {{- else }} value: {{ .Values.externalDatabase.username | quote }} {{- end }} - - name: POSTGRESQL_PASSWORD + - name: DATABASE_PASSWORD valueFrom: secretKeyRef: {{- if .Values.postgresql.enabled }} @@ -188,18 +210,63 @@ spec: name: {{ printf "%s-%s" .Release.Name "externaldb" }} key: db-password {{- end }} - - name: POSTGRESQL_DATABASE + - name: DATABASE_DATABASE {{- if .Values.postgresql.enabled }} value: {{ .Values.postgresql.postgresqlDatabase }} {{- else }} value: {{ .Values.externalDatabase.database | quote }} {{- end }} + - name: DATABASE_PARAMS + {{- if .Values.postgresql.enabled }} + value: "characterEncoding=utf8" + {{- else }} + value: {{ .Values.externalDatabase.params | quote }} + {{- end }} - name: ZOOKEEPER_QUORUM {{- if .Values.zookeeper.enabled }} value: {{ template "dolphinscheduler.zookeeper.quorum" . }} {{- else }} value: {{ .Values.externalZookeeper.zookeeperQuorum }} {{- end }} + - name: RESOURCE_STORAGE_TYPE + valueFrom: + configMapKeyRef: + key: RESOURCE_STORAGE_TYPE + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: RESOURCE_UPLOAD_PATH + valueFrom: + configMapKeyRef: + key: RESOURCE_UPLOAD_PATH + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_DEFAULT_FS + valueFrom: + configMapKeyRef: + key: FS_DEFAULT_FS + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_ENDPOINT + valueFrom: + configMapKeyRef: + key: FS_S3A_ENDPOINT + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_ACCESS_KEY + valueFrom: + configMapKeyRef: + key: FS_S3A_ACCESS_KEY + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_SECRET_KEY + valueFrom: + configMapKeyRef: + key: FS_S3A_SECRET_KEY + name: {{ include "dolphinscheduler.fullname" . }}-common + {{- if .Values.master.resources }} + resources: + limits: + memory: {{ .Values.master.resources.limits.memory | quote }} + cpu: {{ .Values.master.resources.limits.cpu | quote }} + requests: + memory: {{ .Values.master.resources.requests.memory | quote }} + cpu: {{ .Values.master.resources.requests.cpu | quote }} + {{- end }} {{- if .Values.master.livenessProbe.enabled }} livenessProbe: exec: diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml index 198cef43b6..097f8d8580 100644 --- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml @@ -40,6 +40,10 @@ spec: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/component: worker + {{- if .Values.alert.annotations }} + annotations: + {{- toYaml .Values.alert.annotations | nindent 8 }} + {{- end }} spec: {{- if .Values.worker.affinity }} affinity: {{- toYaml .Values.worker.affinity | nindent 8 }} @@ -75,34 +79,38 @@ spec: {{- else }} value: {{ .Values.externalZookeeper.zookeeperQuorum }} {{- end }} - - name: init-postgresql + - name: init-database image: busybox:1.31.0 command: - /bin/sh - -ec - | - while ! nc -z ${POSTGRESQL_HOST} ${POSTGRESQL_PORT}; do + while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do counter=$((counter+1)) if [ $counter == 5 ]; then - echo "Error: Couldn't connect to postgresql." + echo "Error: Couldn't connect to database." exit 1 fi - echo "Trying to connect to postgresql at ${POSTGRESQL_HOST}:${POSTGRESQL_PORT}. Attempt $counter." + echo "Trying to connect to database at ${DATABASE_HOST}:${DATABASE_PORT}. Attempt $counter." sleep 60 done env: - - name: POSTGRESQL_HOST + - name: DATABASE_HOST {{- if .Values.postgresql.enabled }} value: {{ template "dolphinscheduler.postgresql.fullname" . }} {{- else }} value: {{ .Values.externalDatabase.host | quote }} {{- end }} - - name: POSTGRESQL_PORT + - name: DATABASE_PORT {{- if .Values.postgresql.enabled }} value: "5432" {{- else }} - value: {{ .Values.externalDatabase.port }} + value: {{ .Values.externalDatabase.port | quote }} {{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + - name: {{ include "dolphinscheduler.image.pullSecrets" . }} + {{- end }} containers: - name: {{ include "dolphinscheduler.fullname" . }}-worker image: {{ include "dolphinscheduler.image.repository" . | quote }} @@ -115,6 +123,8 @@ spec: name: "logs-port" imagePullPolicy: {{ .Values.image.pullPolicy }} env: + - name: DOLPHINSCHEDULER_OPTS + value: {{ default "" .Values.worker.jvmOptions }} - name: TZ value: {{ .Values.timezone }} - name: WORKER_EXEC_THREADS @@ -157,25 +167,37 @@ spec: configMapKeyRef: name: {{ include "dolphinscheduler.fullname" . }}-master key: DOLPHINSCHEDULER_DATA_BASEDIR_PATH - - name: POSTGRESQL_HOST + - name: DATABASE_TYPE + {{- if .Values.postgresql.enabled }} + value: "postgresql" + {{- else }} + value: {{ .Values.externalDatabase.type | quote }} + {{- end }} + - name: DATABASE_DRIVER + {{- if .Values.postgresql.enabled }} + value: "org.postgresql.Driver" + {{- else }} + value: {{ .Values.externalDatabase.driver | quote }} + {{- end }} + - name: DATABASE_HOST {{- if .Values.postgresql.enabled }} value: {{ template "dolphinscheduler.postgresql.fullname" . }} {{- else }} value: {{ .Values.externalDatabase.host | quote }} {{- end }} - - name: POSTGRESQL_PORT + - name: DATABASE_PORT {{- if .Values.postgresql.enabled }} value: "5432" {{- else }} - value: {{ .Values.externalDatabase.port }} + value: {{ .Values.externalDatabase.port | quote }} {{- end }} - - name: POSTGRESQL_USERNAME + - name: DATABASE_USERNAME {{- if .Values.postgresql.enabled }} value: {{ .Values.postgresql.postgresqlUsername }} {{- else }} value: {{ .Values.externalDatabase.username | quote }} {{- end }} - - name: POSTGRESQL_PASSWORD + - name: DATABASE_PASSWORD valueFrom: secretKeyRef: {{- if .Values.postgresql.enabled }} @@ -185,18 +207,133 @@ spec: name: {{ printf "%s-%s" .Release.Name "externaldb" }} key: db-password {{- end }} - - name: POSTGRESQL_DATABASE + - name: DATABASE_DATABASE {{- if .Values.postgresql.enabled }} value: {{ .Values.postgresql.postgresqlDatabase }} {{- else }} value: {{ .Values.externalDatabase.database | quote }} {{- end }} + - name: DATABASE_PARAMS + {{- if .Values.postgresql.enabled }} + value: "characterEncoding=utf8" + {{- else }} + value: {{ .Values.externalDatabase.params | quote }} + {{- end }} - name: ZOOKEEPER_QUORUM {{- if .Values.zookeeper.enabled }} value: "{{ template "dolphinscheduler.zookeeper.quorum" . }}" {{- else }} value: {{ .Values.externalZookeeper.zookeeperQuorum }} {{- end }} + - name: RESOURCE_STORAGE_TYPE + valueFrom: + configMapKeyRef: + key: RESOURCE_STORAGE_TYPE + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: RESOURCE_UPLOAD_PATH + valueFrom: + configMapKeyRef: + key: RESOURCE_UPLOAD_PATH + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_DEFAULT_FS + valueFrom: + configMapKeyRef: + key: FS_DEFAULT_FS + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_ENDPOINT + valueFrom: + configMapKeyRef: + key: FS_S3A_ENDPOINT + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_ACCESS_KEY + valueFrom: + configMapKeyRef: + key: FS_S3A_ACCESS_KEY + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: FS_S3A_SECRET_KEY + valueFrom: + configMapKeyRef: + key: FS_S3A_SECRET_KEY + name: {{ include "dolphinscheduler.fullname" . }}-common + - name: XLS_FILE_PATH + valueFrom: + configMapKeyRef: + key: XLS_FILE_PATH + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SERVER_HOST + valueFrom: + configMapKeyRef: + key: MAIL_SERVER_HOST + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SERVER_PORT + valueFrom: + configMapKeyRef: + key: MAIL_SERVER_PORT + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SENDER + valueFrom: + configMapKeyRef: + key: MAIL_SENDER + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_USER + valueFrom: + configMapKeyRef: + key: MAIL_USER + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_PASSWD + valueFrom: + configMapKeyRef: + key: MAIL_PASSWD + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SMTP_STARTTLS_ENABLE + valueFrom: + configMapKeyRef: + key: MAIL_SMTP_STARTTLS_ENABLE + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SMTP_SSL_ENABLE + valueFrom: + configMapKeyRef: + key: MAIL_SMTP_SSL_ENABLE + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: MAIL_SMTP_SSL_TRUST + valueFrom: + configMapKeyRef: + key: MAIL_SMTP_SSL_TRUST + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: ENTERPRISE_WECHAT_ENABLE + valueFrom: + configMapKeyRef: + key: ENTERPRISE_WECHAT_ENABLE + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: ENTERPRISE_WECHAT_CORP_ID + valueFrom: + configMapKeyRef: + key: ENTERPRISE_WECHAT_CORP_ID + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: ENTERPRISE_WECHAT_SECRET + valueFrom: + configMapKeyRef: + key: ENTERPRISE_WECHAT_SECRET + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: ENTERPRISE_WECHAT_AGENT_ID + valueFrom: + configMapKeyRef: + key: ENTERPRISE_WECHAT_AGENT_ID + name: {{ include "dolphinscheduler.fullname" . }}-alert + - name: ENTERPRISE_WECHAT_USERS + valueFrom: + configMapKeyRef: + key: ENTERPRISE_WECHAT_USERS + name: {{ include "dolphinscheduler.fullname" . }}-alert + {{- if .Values.worker.resources }} + resources: + limits: + memory: {{ .Values.worker.resources.limits.memory | quote }} + cpu: {{ .Values.worker.resources.limits.cpu | quote }} + requests: + memory: {{ .Values.worker.resources.requests.memory | quote }} + cpu: {{ .Values.worker.resources.requests.cpu | quote }} + {{- end }} {{- if .Values.worker.livenessProbe.enabled }} livenessProbe: exec: diff --git a/docker/kubernetes/dolphinscheduler/values.yaml b/docker/kubernetes/dolphinscheduler/values.yaml index 3cb35c19e0..8acb1d326a 100644 --- a/docker/kubernetes/dolphinscheduler/values.yaml +++ b/docker/kubernetes/dolphinscheduler/values.yaml @@ -29,10 +29,9 @@ image: repository: "dolphinscheduler" tag: "latest" pullPolicy: "IfNotPresent" + pullSecrets: [] -imagePullSecrets: [] - -# If not exists external postgresql, by default, Dolphinscheduler's database will use it. +# If not exists external database, by default, Dolphinscheduler's database will use it. postgresql: enabled: true postgresqlUsername: "root" @@ -43,21 +42,24 @@ postgresql: size: "20Gi" storageClass: "-" -# If exists external postgresql, and set postgresql.enable value to false. -# If postgresql.enable is false, Dolphinscheduler's database will use it. +# If exists external database, and set postgresql.enable value to false. +# external database will be used, otherwise Dolphinscheduler's database will be used. externalDatabase: + type: "postgresql" + driver: "org.postgresql.Driver" host: "localhost" port: "5432" username: "root" password: "root" database: "dolphinscheduler" + # multi params should join with & char + params: "characterEncoding=utf8" # If not exists external zookeeper, by default, Dolphinscheduler's zookeeper will use it. zookeeper: enabled: true taskQueue: "zookeeper" - config: - ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons + config: null service: port: "2181" persistence: @@ -70,6 +72,18 @@ zookeeper: externalZookeeper: taskQueue: "zookeeper" zookeeperQuorum: "127.0.0.1:2181" + zookeeperRoot: "/dolphinscheduler" + +common: + configmap: + DOLPHINSCHEDULER_ENV_PATH: "/tmp/dolphinscheduler/env" + DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler/files" + RESOURCE_STORAGE_TYPE: "NONE" + RESOURCE_UPLOAD_PATH: "/ds" + FS_DEFAULT_FS: "s3a://xxxx" + FS_S3A_ENDPOINT: "s3.xxx.amazonaws.com" + FS_S3A_ACCESS_KEY: "xxxxxxx" + FS_S3A_SECRET_KEY: "xxxxxxx" master: podManagementPolicy: "Parallel" @@ -85,6 +99,18 @@ master: # If specified, the pod's scheduling constraints. # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core affinity: {} + # The jvm options for java instance startup + jvmOptions: "" + resources: {} + # limits: + # memory: "18Gi" + # cpu: "4" + # requests: + # memory: "2Gi" + # cpu: "500m" + # You can use annotations to attach arbitrary non-identifying metadata to objects. + # Clients such as tools and libraries can retrieve this metadata. + annotations: {} ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes configmap: @@ -137,6 +163,18 @@ worker: # If specified, the pod's scheduling constraints. # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core affinity: {} + # The jvm options for java instance startup + jvmOptions: "" + resources: {} + # limits: + # memory: "18Gi" + # cpu: "4" + # requests: + # memory: "2Gi" + # cpu: "500m" + # You can use annotations to attach arbitrary non-identifying metadata to objects. + # Clients such as tools and libraries can retrieve this metadata. + annotations: {} ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: @@ -213,9 +251,22 @@ alert: # If specified, the pod's scheduling constraints. # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core affinity: {} + # The jvm options for java instance startup + jvmOptions: "" + resources: {} + # limits: + # memory: "4Gi" + # cpu: "1" + # requests: + # memory: "2Gi" + # cpu: "500m" + # You can use annotations to attach arbitrary non-identifying metadata to objects. + # Clients such as tools and libraries can retrieve this metadata. + annotations: {} ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes configmap: + ALERT_PLUGIN_DIR: "/opt/dolphinscheduler/alert/plugin" XLS_FILE_PATH: "/tmp/xls" MAIL_SERVER_HOST: "" MAIL_SERVER_PORT: "" @@ -275,6 +326,18 @@ api: # If specified, the pod's scheduling constraints. # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core affinity: {} + # The jvm options for java instance startup + jvmOptions: "" + resources: {} + # limits: + # memory: "4Gi" + # cpu: "2" + # requests: + # memory: "2Gi" + # cpu: "500m" + # You can use annotations to attach arbitrary non-identifying metadata to objects. + # Clients such as tools and libraries can retrieve this metadata. + annotations: {} ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: @@ -322,6 +385,16 @@ frontend: # If specified, the pod's scheduling constraints. # More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core affinity: {} + resources: {} + # limits: + # memory: "256Mi" + # cpu: "1" + # requests: + # memory: "256Mi" + # cpu: "500m" + # You can use annotations to attach arbitrary non-identifying metadata to objects. + # Clients such as tools and libraries can retrieve this metadata. + annotations: {} ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes livenessProbe: diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java index 38dc38b4a8..36f903c25d 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java @@ -48,8 +48,8 @@ public class EnterpriseWeChatUtils { private static final String ENTERPRISE_WE_CHAT_TOKEN_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TOKEN_URL); private static final String ENTERPRISE_WE_CHAT_TOKEN_URL_REPLACE = ENTERPRISE_WE_CHAT_TOKEN_URL == null ? null : ENTERPRISE_WE_CHAT_TOKEN_URL - .replaceAll("\\$corpId", ENTERPRISE_WE_CHAT_CORP_ID) - .replaceAll("\\$secret", ENTERPRISE_WE_CHAT_SECRET); + .replaceAll("\\{corpId\\}", ENTERPRISE_WE_CHAT_CORP_ID) + .replaceAll("\\{secret\\}", ENTERPRISE_WE_CHAT_SECRET); private static final String ENTERPRISE_WE_CHAT_PUSH_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_PUSH_URL); @@ -120,9 +120,9 @@ public class EnterpriseWeChatUtils { * @return Enterprise WeChat send message */ public static String makeTeamSendMsg(String toParty, String agentId, String msg) { - return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\$toParty", toParty) - .replaceAll("\\$agentId", agentId) - .replaceAll("\\$msg", msg); + return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty\\}", toParty) + .replaceAll("\\{agentId\\}", agentId) + .replaceAll("\\{msg\\}", msg); } /** @@ -135,9 +135,9 @@ public class EnterpriseWeChatUtils { */ public static String makeTeamSendMsg(Collection toParty, String agentId, String msg) { String listParty = FuncUtils.mkString(toParty, "|"); - return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\$toParty", listParty) - .replaceAll("\\$agentId", agentId) - .replaceAll("\\$msg", msg); + return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty\\}", listParty) + .replaceAll("\\{agentId\\}", agentId) + .replaceAll("\\{msg\\}", msg); } /** @@ -149,9 +149,9 @@ public class EnterpriseWeChatUtils { * @return Enterprise WeChat send message */ public static String makeUserSendMsg(String toUser, String agentId, String msg) { - return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\$toUser", toUser) - .replaceAll("\\$agentId", agentId) - .replaceAll("\\$msg", msg); + return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\{toUser\\}", toUser) + .replaceAll("\\{agentId\\}", agentId) + .replaceAll("\\{msg\\}", msg); } /** @@ -164,9 +164,9 @@ public class EnterpriseWeChatUtils { */ public static String makeUserSendMsg(Collection toUser, String agentId, String msg) { String listUser = FuncUtils.mkString(toUser, "|"); - return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\$toUser", listUser) - .replaceAll("\\$agentId", agentId) - .replaceAll("\\$msg", msg); + return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\{toUser\\}", listUser) + .replaceAll("\\{agentId\\}", agentId) + .replaceAll("\\{msg\\}", msg); } /** @@ -179,7 +179,7 @@ public class EnterpriseWeChatUtils { * @throws IOException the IOException */ public static String sendEnterpriseWeChat(String charset, String data, String token) throws IOException { - String enterpriseWeChatPushUrlReplace = ENTERPRISE_WE_CHAT_PUSH_URL.replaceAll("\\$token", token); + String enterpriseWeChatPushUrlReplace = ENTERPRISE_WE_CHAT_PUSH_URL.replaceAll("\\{token\\}", token); CloseableHttpClient httpClient = HttpClients.createDefault(); try { diff --git a/dolphinscheduler-alert/src/main/resources/alert.properties b/dolphinscheduler-alert/src/main/resources/alert.properties index 4e48e7d0e1..6eb701841c 100644 --- a/dolphinscheduler-alert/src/main/resources/alert.properties +++ b/dolphinscheduler-alert/src/main/resources/alert.properties @@ -41,10 +41,10 @@ enterprise.wechat.enable=false #enterprise.wechat.secret=xxxxxxx #enterprise.wechat.agent.id=xxxxxxx #enterprise.wechat.users=xxxxxxx -#enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&corpsecret=$secret -#enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token -#enterprise.wechat.team.send.msg={\"toparty\":\"$toParty\",\"agentid\":\"$agentId\",\"msgtype\":\"text\",\"text\":{\"content\":\"$msg\"},\"safe\":\"0\"} -#enterprise.wechat.user.send.msg={\"touser\":\"$toUser\",\"agentid\":\"$agentId\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"$msg\"}} +#enterprise.wechat.token.url=https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid={corpId}&corpsecret={secret} +#enterprise.wechat.push.url=https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={token} +#enterprise.wechat.team.send.msg={\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"} +#enterprise.wechat.user.send.msg={\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}} plugin.dir=/Users/xx/your/path/to/plugin/dir diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java index 01b81101fd..7b6cdd013b 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java @@ -55,8 +55,8 @@ public class EnterpriseWeChatUtilsTest { private static final String enterpriseWechatUsers="LiGang,journey"; private static final String msg = "hello world"; - private static final String enterpriseWechatTeamSendMsg = "{\\\"toparty\\\":\\\"$toParty\\\",\\\"agentid\\\":\\\"$agentId\\\",\\\"msgtype\\\":\\\"text\\\",\\\"text\\\":{\\\"content\\\":\\\"$msg\\\"},\\\"safe\\\":\\\"0\\\"}"; - private static final String enterpriseWechatUserSendMsg = "{\\\"touser\\\":\\\"$toUser\\\",\\\"agentid\\\":\\\"$agentId\\\",\\\"msgtype\\\":\\\"markdown\\\",\\\"markdown\\\":{\\\"content\\\":\\\"$msg\\\"}}"; + private static final String enterpriseWechatTeamSendMsg = "{\\\"toparty\\\":\\\"{toParty}\\\",\\\"agentid\\\":\\\"{agentId}\\\",\\\"msgtype\\\":\\\"text\\\",\\\"text\\\":{\\\"content\\\":\\\"{msg}\\\"},\\\"safe\\\":\\\"0\\\"}"; + private static final String enterpriseWechatUserSendMsg = "{\\\"touser\\\":\\\"{toUser}\\\",\\\"agentid\\\":\\\"{agentId}\\\",\\\"msgtype\\\":\\\"markdown\\\",\\\"markdown\\\":{\\\"content\\\":\\\"{msg}\\\"}}"; @Before public void init(){ diff --git a/dolphinscheduler-ui/package.json b/dolphinscheduler-ui/package.json index 1f5170637e..a5642c8f97 100644 --- a/dolphinscheduler-ui/package.json +++ b/dolphinscheduler-ui/package.json @@ -63,7 +63,7 @@ "html-loader": "^0.5.5", "html-webpack-plugin": "^3.2.0", "mini-css-extract-plugin": "^0.8.2", - "node-sass": "^4.14.0", + "node-sass": "^4.14.1", "postcss-loader": "^3.0.0", "progress-bar-webpack-plugin": "^1.12.1", "rimraf": "^2.6.2", diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index b0437a30c2..4ffd7fe4c2 100644 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -46,7 +46,7 @@ export DOLPHINSCHEDULER_LOG_DIR=$DOLPHINSCHEDULER_HOME/logs export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/* -export DOLPHINSCHEDULER_OPTS="-server -Xmx16g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" +export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-"-server -Xmx16g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"} export STOP_TIMEOUT=5 if [ ! -d "$DOLPHINSCHEDULER_LOG_DIR" ]; then From 50fcc492c1649720b8f20e8f750808fa4ca7c199 Mon Sep 17 00:00:00 2001 From: hcz Date: Fri, 17 Jul 2020 14:26:46 +0800 Subject: [PATCH 4/6] Fix typo (#3230) --- .../src/main/resources/config/install_config.conf | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/dolphinscheduler-server/src/main/resources/config/install_config.conf index eeff0308b1..6da5fbed55 100644 --- a/dolphinscheduler-server/src/main/resources/config/install_config.conf +++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf @@ -16,7 +16,7 @@ # -# NOTICE : If the following config has special characters in the variable `.*[]^${}\+?|()@#&`, Please escape, for example, `[` escape to `\[` +# NOTICE: If the following config has special characters in the variable `.*[]^${}\+?|()@#&`, Please escape, for example, `[` escape to `\[` # postgresql or mysql dbtype="mysql" @@ -27,12 +27,13 @@ dbhost="192.168.xx.xx:3306" # db username username="xx" +# db password +# NOTICE: if there are special characters, please use the \ to escape, for example, `[` escape to `\[` +password="xx" + # database name dbname="dolphinscheduler" -# db passwprd -# NOTICE: if there are special characters, please use the \ to escape, for example, `[` escape to `\[` -password="xx" # zk cluster zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181" @@ -137,4 +138,4 @@ alertServer="ds3" # run api machine # note: list of machine hostnames for deploying api server -apiServers="ds1" \ No newline at end of file +apiServers="ds1" From 178d8f829a3b1e08052927783801c0afb1895489 Mon Sep 17 00:00:00 2001 From: amenglu <1175433513@qq.com> Date: Fri, 17 Jul 2020 16:54:52 +0800 Subject: [PATCH 5/6] Update MasterSchedulerService.java (#3226) --- .../server/master/runner/MasterSchedulerService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java index ea6dbcc060..b0c0ccb2df 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java @@ -46,7 +46,7 @@ import java.util.concurrent.TimeUnit; public class MasterSchedulerService extends Thread { /** - * logger of MasterSchedulerThread + * logger of MasterSchedulerService */ private static final Logger logger = LoggerFactory.getLogger(MasterSchedulerService.class); @@ -80,7 +80,7 @@ public class MasterSchedulerService extends Thread { /** - * constructor of MasterSchedulerThread + * constructor of MasterSchedulerService */ @PostConstruct public void init(){ @@ -91,7 +91,7 @@ public class MasterSchedulerService extends Thread { @Override public void start(){ - super.setName("MasterSchedulerThread"); + super.setName("MasterSchedulerService"); super.start(); } @@ -109,7 +109,7 @@ public class MasterSchedulerService extends Thread { } /** - * run of MasterSchedulerThread + * run of MasterSchedulerService */ @Override public void run() { From 1b54c76ed8981e9feea1f172e2c81c5b3e0cd789 Mon Sep 17 00:00:00 2001 From: xingchun-chen <55787491+xingchun-chen@users.noreply.github.com> Date: Fri, 17 Jul 2020 17:04:28 +0800 Subject: [PATCH 6/6] [test-2995][e2e]add rerun workflow and optimize testCase (#3220) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add LoginTest license * Delete useless packages * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * e2e add project and workflow case * Update worker_group_id to worker_group in init.sql * Update worker_group_id to worker_group in init.sql * Update worker_group_id to worker_group * Increase dataX environment variable, sslTrust default value * modify dataX environment variable * update e2e chrome version * add alert manage testcase and Optimize test case execution order * add e2e to create and to edit queue * update e2e chrome version * add e2e token testcase * add e2e token testcase * add e2e to run workflow testcase * add e2e timing testcase * add e2e timing testcase * add e2e timing testcase * add e2e timing testcase * Merge remote-tracking branch 'upstream/dev' into dev # Conflicts: # e2e/src/test/java/org/apache/dolphinscheduler/page/security/AlertManagePage.java # e2e/src/test/java/org/apache/dolphinscheduler/page/security/QueueManagePage.java * Merge remote-tracking branch 'upstream/dev' into dev # Conflicts: # e2e/src/test/java/org/apache/dolphinscheduler/page/security/AlertManagePage.java # e2e/src/test/java/org/apache/dolphinscheduler/page/security/QueueManagePage.java * Merge remote-tracking branch 'upstream/dev' into dev # Conflicts: # e2e/src/test/java/org/apache/dolphinscheduler/page/security/AlertManagePage.java # e2e/src/test/java/org/apache/dolphinscheduler/page/security/QueueManagePage.java * Merge remote-tracking branch 'upstream/dev' into dev # Conflicts: # e2e/src/test/java/org/apache/dolphinscheduler/page/security/AlertManagePage.java # e2e/src/test/java/org/apache/dolphinscheduler/page/security/QueueManagePage.java * Merge remote-tracking branch 'upstream/dev' into dev # Conflicts: # e2e/src/test/java/org/apache/dolphinscheduler/page/security/AlertManagePage.java # e2e/src/test/java/org/apache/dolphinscheduler/page/security/QueueManagePage.java * Merge remote-tracking branch 'upstream/dev' into dev # Conflicts: # e2e/src/test/java/org/apache/dolphinscheduler/page/security/AlertManagePage.java # e2e/src/test/java/org/apache/dolphinscheduler/page/security/QueueManagePage.java * Merge remote-tracking branch 'upstream/dev' into dev # Conflicts: # e2e/src/test/java/org/apache/dolphinscheduler/page/security/AlertManagePage.java # e2e/src/test/java/org/apache/dolphinscheduler/page/security/QueueManagePage.java * Merge remote-tracking branch 'upstream/dev' into dev # Conflicts: # e2e/src/test/java/org/apache/dolphinscheduler/page/security/AlertManagePage.java # e2e/src/test/java/org/apache/dolphinscheduler/page/security/QueueManagePage.java * Merge remote-tracking branch 'upstream/dev' into dev # Conflicts: # e2e/src/test/java/org/apache/dolphinscheduler/page/security/AlertManagePage.java # e2e/src/test/java/org/apache/dolphinscheduler/page/security/QueueManagePage.java * Merge remote-tracking branch 'upstream/dev' into dev # Conflicts: # e2e/src/test/java/org/apache/dolphinscheduler/page/security/AlertManagePage.java # e2e/src/test/java/org/apache/dolphinscheduler/page/security/QueueManagePage.java * Merge remote-tracking branch 'upstream/dev' into dev # Conflicts: # e2e/src/test/java/org/apache/dolphinscheduler/page/security/AlertManagePage.java # e2e/src/test/java/org/apache/dolphinscheduler/page/security/QueueManagePage.java * Merge remote-tracking branch 'upstream/dev' into dev # Conflicts: # e2e/src/test/java/org/apache/dolphinscheduler/page/security/AlertManagePage.java # e2e/src/test/java/org/apache/dolphinscheduler/page/security/QueueManagePage.java * Merge remote-tracking branch 'upstream/dev' into dev # Conflicts: # e2e/src/test/java/org/apache/dolphinscheduler/page/security/AlertManagePage.java # e2e/src/test/java/org/apache/dolphinscheduler/page/security/QueueManagePage.java * edit user * edit user * edit user * edit user * edit user * Update TimingLocator.java * modify ci log * Update ci_e2e.yml modify ci logs * Update ci_e2e.yml modify ci logs * Update ci_e2e.yml modify ci logs * Update ci_e2e.yml * Update ci_e2e.yml modify ci logs * Update ci_e2e.yml modify ci logs * Update ci_e2e.yml modify ci logs * Update ci_e2e.yml modify ci logs * Update ci_e2e.yml modify artifact version * Update ci_e2e.yml * Update ci_e2e.yml modify /var/lib/docker/volumes permissions * Update ci_e2e.yml modify ci logs * Update ci_e2e.yml Modify log file permissions * Update ci_e2e.yml modify logs * Update ci_e2e.yml modify logs * Update ci_e2e.yml modify logs * modify ci logs * modify ci logs * modify ci logs * modify ci logs * Update ci_e2e.yml modify ci logs * modify ci logs * modify ci logs * modify ci logs * modify ci logs * modify ci logs * modify ci logs * modify ci logs * Update ci_e2e.yml modify ci logs * modify ci logs * modify ci logs * modify ci logs * modify ci logs * modify ci logs * modify ci logs * modify ci logs * modify ci logs * modify ci logs * modify ci logs * modify tenant * modify alert wait time * modify queue wait time * modify alert wait time * modify alert wait time * modify alert wait time * modify alert wait time * modify alert wait time * modify alert wait time * modify button xpath * modify queue xpath * Ignore edit user * modify alert wait time * print delete user log * modify the xpath of delete user button * modify timing wait time * add process instance * add process instance * modify timing wait time * modify rerun assert * modify alert wait time * Optimize test cases   Co-authored-by: chenxingchun <438044805@qq.com> Co-authored-by: dailidong --- .../data/project/ProcessInstanceData.java | 25 ++ ...reateProjectData.java => ProjectData.java} | 2 +- ...kflowData.java => WorkflowDefineData.java} | 2 +- .../project/ProcessInstanceLocator.java | 31 +++ ...rojectLocator.java => ProjectLocator.java} | 9 +- .../locator/project/TimingLocator.java | 4 +- ...ocator.java => WorkflowDefineLocator.java} | 2 +- .../page/project/CreateWorkflowPage.java | 226 ------------------ .../page/project/ProcessInstancePage.java | 40 ++++ ...reateProjectPage.java => ProjectPage.java} | 29 +-- .../page/project/TimingPage.java | 4 +- .../page/project/WorkflowDefinePage.java | 226 ++++++++++++++++++ .../testDeleteData/TestDeleteProject.java | 6 +- .../testDeleteData/TestDeleteWorkflow.java | 26 +- .../testProject/TestProcessInstance.java | 35 +++ ...estCreateProject.java => TestProject.java} | 12 +- .../testcase/testProject/TestRunWorkflow.java | 10 +- ...eWorkflow.java => TestWorkflowDefine.java} | 36 +-- e2e/testng.xml | 29 ++- 19 files changed, 457 insertions(+), 297 deletions(-) create mode 100644 e2e/src/test/java/org/apache/dolphinscheduler/data/project/ProcessInstanceData.java rename e2e/src/test/java/org/apache/dolphinscheduler/data/project/{CreateProjectData.java => ProjectData.java} (97%) rename e2e/src/test/java/org/apache/dolphinscheduler/data/project/{CreateWorkflowData.java => WorkflowDefineData.java} (98%) create mode 100644 e2e/src/test/java/org/apache/dolphinscheduler/locator/project/ProcessInstanceLocator.java rename e2e/src/test/java/org/apache/dolphinscheduler/locator/project/{CreateProjectLocator.java => ProjectLocator.java} (78%) rename e2e/src/test/java/org/apache/dolphinscheduler/locator/project/{CreateWorkflowLocator.java => WorkflowDefineLocator.java} (99%) delete mode 100644 e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateWorkflowPage.java create mode 100644 e2e/src/test/java/org/apache/dolphinscheduler/page/project/ProcessInstancePage.java rename e2e/src/test/java/org/apache/dolphinscheduler/page/project/{CreateProjectPage.java => ProjectPage.java} (66%) create mode 100644 e2e/src/test/java/org/apache/dolphinscheduler/page/project/WorkflowDefinePage.java create mode 100644 e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestProcessInstance.java rename e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/{TestCreateProject.java => TestProject.java} (81%) rename e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/{TestCreateWorkflow.java => TestWorkflowDefine.java} (59%) diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/ProcessInstanceData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/ProcessInstanceData.java new file mode 100644 index 0000000000..7114851d25 --- /dev/null +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/ProcessInstanceData.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.data.project; + +public class ProcessInstanceData { + //Process Instance page title + public static final String PROCESS_INSTANCE_TITLE = "工作流实例 - DolphinScheduler"; + public static final String RERUN_TYPE= "重跑"; + + +} diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreateProjectData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/ProjectData.java similarity index 97% rename from e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreateProjectData.java rename to e2e/src/test/java/org/apache/dolphinscheduler/data/project/ProjectData.java index 004004c54f..fce247aa4d 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreateProjectData.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/ProjectData.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.data.project; -public class CreateProjectData { +public class ProjectData { // create project name public static final String PROJECT_NAME = "selenium_project_1"; // create project description diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreateWorkflowData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/WorkflowDefineData.java similarity index 98% rename from e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreateWorkflowData.java rename to e2e/src/test/java/org/apache/dolphinscheduler/data/project/WorkflowDefineData.java index 0ec5f05834..0b3645f329 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreateWorkflowData.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/WorkflowDefineData.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.data.project; -public class CreateWorkflowData { +public class WorkflowDefineData { /** * create workflow data */ diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/ProcessInstanceLocator.java b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/ProcessInstanceLocator.java new file mode 100644 index 0000000000..1af9d56274 --- /dev/null +++ b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/ProcessInstanceLocator.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.locator.project; + +import org.openqa.selenium.By; + +public class ProcessInstanceLocator { + // jump Process Instance page + //click Process Instance name + public static final By CLICK_PROCESS_INSTANCE_NAME = By.xpath("//div[3]/div/ul/li[2]"); + + // click rerun button + public static final By CLICK_RERUN_BUTTON = By.xpath("//tr[2]/td[14]/div[1]/button[2]"); + + //assert rerun type + public static final By RUNNING_TYPE = By.xpath("//tr[2]/td[5]/span"); +} diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateProjectLocator.java b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/ProjectLocator.java similarity index 78% rename from e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateProjectLocator.java rename to e2e/src/test/java/org/apache/dolphinscheduler/locator/project/ProjectLocator.java index 3d96f24288..8b26e673a6 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateProjectLocator.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/ProjectLocator.java @@ -18,17 +18,12 @@ package org.apache.dolphinscheduler.locator.project; import org.openqa.selenium.By; -public class CreateProjectLocator { +public class ProjectLocator { //click project manage -// public static final By PROJECT_MANAGE = By.xpath("//div[@class='clearfix list'][2]"); public static final By PROJECT_MANAGE = By.xpath("//div[2]/div[2]/div/a/span"); - // public static final By SECURITY_MANAGE = By.xpath("//div[@class='m-top']/div/div[2]/dev[@class='clearfix list'][6]"); - public static final By SECURITY_MANAGE = By.xpath("//div[2]/div[6]/div/a/span"); - - //click create project button - public static final By CREATE_PROJECT_BUTTON = By.xpath("//div[2]/div/div[1]/button/span"); + public static final By CREATE_PROJECT_BUTTON = By.xpath("//button/span"); //input project name public static final By PROJECT_NAME = By.xpath("//div[2]/div/div/div[2]/div/input"); diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/TimingLocator.java b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/TimingLocator.java index d9559b1236..2605636f16 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/TimingLocator.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/TimingLocator.java @@ -45,9 +45,9 @@ public class TimingLocator { public static final By CLICK_CREATE_BUTTON = By.xpath("//div[12]/button[2]/span"); //edit timing - public static final By CLICK_TIMING_MANAGEMENT_BUTTON = By.xpath("//div[2]/div/div[3]/div[1]/div/table/tr[2]/td[10]/button[6]"); + public static final By CLICK_TIMING_MANAGEMENT_BUTTON = By.xpath("//tr[2]/td[10]/button[6]"); - public static final By CLICK_EDIT_TIMING_BUTTON = By.xpath("//div[2]/div[3]/div/div[2]/div[1]/table/tr[2]/td[10]/button[1]"); + public static final By CLICK_EDIT_TIMING_BUTTON = By.xpath("//tr[2]/td[10]/button[1]/i"); //online timing public static final By CLICK_ONLINE_TIMING_BUTTON = By.xpath("//td[10]/button[2]"); diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateWorkflowLocator.java b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/WorkflowDefineLocator.java similarity index 99% rename from e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateWorkflowLocator.java rename to e2e/src/test/java/org/apache/dolphinscheduler/locator/project/WorkflowDefineLocator.java index ad18b6217c..bfb237ccd2 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateWorkflowLocator.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/WorkflowDefineLocator.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.locator.project; import org.openqa.selenium.By; -public class CreateWorkflowLocator { +public class WorkflowDefineLocator { /** * jump workflow define page diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateWorkflowPage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateWorkflowPage.java deleted file mode 100644 index ed978a80ed..0000000000 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateWorkflowPage.java +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.page.project; - -import org.apache.dolphinscheduler.common.PageCommon; -import org.apache.dolphinscheduler.constant.TestConstant; -import org.apache.dolphinscheduler.data.project.CreateWorkflowData; -import org.apache.dolphinscheduler.locator.project.CreateWorkflowLocator; -import org.apache.dolphinscheduler.locator.project.RunWorkflowLocator; -import org.openqa.selenium.WebDriver; - -public class CreateWorkflowPage extends PageCommon { - public CreateWorkflowPage(WebDriver driver) { - super(driver); - } - - /** - * jump create workflow page - */ - - public boolean jumpWorkflowPage() throws InterruptedException { - // click project name - clickElement(CreateWorkflowLocator.CLICK_PROJECT_NAME); - Thread.sleep(TestConstant.ONE_THOUSAND); - - System.out.println("Click on workflow define to jump to workflow define page"); - // click workflow define - clickElement(CreateWorkflowLocator.CLICK_WORKFLOW_DEFINE); - - return ifTitleContains(CreateWorkflowData.WORKFLOW_TITLE); - } - - public boolean createWorkflow() throws InterruptedException { - System.out.println("Click create workflow button"); - // click create workflow button - clickElement(CreateWorkflowLocator.CLICK_CREATE_WORKFLOW_BUTTON); - - System.out.println("drag shell task"); - //drag shell_task - dragAndDrop(CreateWorkflowLocator.MOUSE_DOWN_AT_SHELL,CreateWorkflowLocator.MOUSE_MOVE_SHELL_AT_DAG); - - //input shell task _name - sendInput(CreateWorkflowLocator.INPUT_SHELL_TASK_NAME , CreateWorkflowData.SHELL_TASK_NAME); - - //click stop run type - clickElement(CreateWorkflowLocator.CLICK_STOP_RUN_TYPE); - - //click normal run type - clickElement(CreateWorkflowLocator.CLICK_NORMAL_RUN_TYPE); - - //input shell task description - sendInput(CreateWorkflowLocator.INPUT_SHELL_TASK_DESCRIPTION , CreateWorkflowData.SHELL_TASK_DESCRIPTION); - - //select task priority - clickElement(CreateWorkflowLocator.CLICK_TASK_PRIORITY); - clickElement(CreateWorkflowLocator.SELECT_TASK_PRIORITY); - - //select work group - clickElement(CreateWorkflowLocator.CLICK_WORK_GROUP); - clickElement(CreateWorkflowLocator.SELECT_WORK_GROUP); - - //select number of failed retries - clickElement(CreateWorkflowLocator.SELECT_FAIL_RETRIES_NUMBER); - - //select failed retry interval - clickElement(CreateWorkflowLocator.SELECT_FAIL_RETRIES_INTERVAL); - - //click timeout alarm - clickElement(CreateWorkflowLocator.CLICK_TIMEOUT_ALARM); - - //select timeout fail - clickElement(CreateWorkflowLocator.SELECT_TIMEOUT_FAIL); - - //cancel timeout alarm - clickElement(CreateWorkflowLocator.CANCEL_TIMEOUT_ALARM); - - //select timeout alarm - clickElement(CreateWorkflowLocator.SELECT_TIMEOUT_ALARM); - - //clear timeout - clearInput(CreateWorkflowLocator.SELECT_TIMEOUT); - clearInput(CreateWorkflowLocator.SELECT_TIMEOUT); - - //input timeout - sendInput(CreateWorkflowLocator.SELECT_TIMEOUT, CreateWorkflowData.INPUT_TIMEOUT); - - //click codeMirror and input script - inputCodeMirror(CreateWorkflowLocator.CLICK_CODE_MIRROR, CreateWorkflowLocator.INPUT_SCRIPT, CreateWorkflowData.SHELL_SCRIPT); - scrollToElementBottom(CreateWorkflowLocator.SCROLL_BOTTOM); - - //click custom parameters - clickElement(CreateWorkflowLocator.CLICK_CUSTOM_PARAMETERS); - - //input custom parameters - sendInput(CreateWorkflowLocator.INPUT_CUSTOM_PARAMETERS, CreateWorkflowData.INPUT_CUSTOM_PARAMETERS); - - //input custom parameters value - sendInput(CreateWorkflowLocator.INPUT_CUSTOM_PARAMETERS_VALUE, CreateWorkflowData.INPUT_CUSTOM_PARAMETERS_VALUE); - - //click add custom parameters - clickElement(CreateWorkflowLocator.CLICK_ADD_CUSTOM_PARAMETERS); - - scrollToElementBottom(CreateWorkflowLocator.SCROLL_BOTTOM); - - //input add custom parameters - sendInput(CreateWorkflowLocator.INPUT_ADD_CUSTOM_PARAMETERS, CreateWorkflowData.INPUT_ADD_CUSTOM_PARAMETERS); - - //input add custom parameters value - sendInput(CreateWorkflowLocator.INPUT_ADD_CUSTOM_PARAMETERS_VALUE, CreateWorkflowData.INPUT_ADD_CUSTOM_PARAMETERS_VALUE); - - //click delete custom parameters - clickElement(CreateWorkflowLocator.CLICK_DELETE_CUSTOM_PARAMETERS); - - //click submit button - clickElement(CreateWorkflowLocator.CLICK_SUBMIT_BUTTON); - Thread.sleep(TestConstant.ONE_THOUSAND); - System.out.println("Task node set up successfully"); - System.out.println("move to Dag Element "); - moveToDragElement(CreateWorkflowLocator.MOUSE_MOVE_SHELL_AT_DAG,-300,-100); - - return ifTitleContains(CreateWorkflowData.CREATE_WORKFLOW_TITLE); - } - - /** - * save workflow - */ - public boolean saveWorkflow() throws InterruptedException { - System.out.println("start to save workflow "); - - //click save workflow button - clickElement(CreateWorkflowLocator.CLICK_SAVE_WORKFLOW_BUTTON); - - //input workflow name - sendInput(CreateWorkflowLocator.INPUT_WORKFLOW_NAME, CreateWorkflowData.INPUT_WORKFLOW_NAME); - - //input workflow description - sendInput(CreateWorkflowLocator.INPUT_WORKFLOW_DESCRIPTION, CreateWorkflowData.INPUT_WORKFLOW_DESCRIPTION); - - //select tenant - clickElement(CreateWorkflowLocator.CLICK_TENANT); - clickElement(CreateWorkflowLocator.SELECT_TENANT); - - //click workflow timeout alarm - clickElement(CreateWorkflowLocator.CLICK_WORKFLOW_TIMEOUT_ALARM); - clearInput(CreateWorkflowLocator.INPUT_WORKFLOW_TIMEOUT); - - //input workflow timeout - sendInput(CreateWorkflowLocator.INPUT_WORKFLOW_TIMEOUT, CreateWorkflowData.INPUT_WORKFLOW_TIMEOUT); - - //click workflow global parameters - clickElement(CreateWorkflowLocator.CLICK_WORKFLOW_GLOBAL_PARAMETERS); - - //input workflow global parameters - sendInput(CreateWorkflowLocator.INPUT_WORKFLOW_GLOBAL_PARAMETERS, CreateWorkflowData.INPUT_WORKFLOW_GLOBAL_PARAMETERS); - - //input workflow global parameters value - sendInput(CreateWorkflowLocator.INPUT_WORKFLOW_GLOBAL_PARAMETERS_VALUES, CreateWorkflowData.INPUT_WORKFLOW_GLOBAL_PARAMETERS_VALUES); - - //click to add workflow global parameters - clickElement(CreateWorkflowLocator.CLICK_ADD_WORKFLOW_GLOBAL_PARAMETERS); - - //input to add workflow global parameters - sendInput(CreateWorkflowLocator.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS, CreateWorkflowData.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS); - - //input to add workflow global parameters value - sendInput(CreateWorkflowLocator.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS_VALUES, CreateWorkflowData.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS_VALUES); - - //delete workflow global parameters value - clickElement(CreateWorkflowLocator.CLICK_DELETE_WORKFLOW_GLOBAL_PARAMETERS); - Thread.sleep(TestConstant.ONE_THOUSAND); - - //click add button - System.out.println("submit workflow"); - clickButton(CreateWorkflowLocator.CLICK_ADD_BUTTON); - - return ifTitleContains(CreateWorkflowData.CREATE_WORKFLOW_TITLE); - } - - public boolean onlineWorkflow() throws InterruptedException { - clickElement(CreateWorkflowLocator.CLICK_WORKFLOW_DEFINE); - - // click online button - System.out.println("Click online workflow button"); - clickButton(CreateWorkflowLocator.CLICK_ONLINE_WORKFLOW_BUTTON); - - return ifTitleContains(CreateWorkflowData.WORKFLOW_TITLE); - } - - public boolean offlineWorkflow() throws InterruptedException { - clickElement(CreateWorkflowLocator.CLICK_WORKFLOW_DEFINE); - - // click offline button - System.out.println("offline workflow"); - Thread.sleep(500); - clickButton(CreateWorkflowLocator.CLICK_OFFLINE_WORKFLOW_BUTTON); - - return ifTitleContains(CreateWorkflowData.WORKFLOW_TITLE); - } - - - public boolean deleteWorkflow() throws InterruptedException { - //click delete workflow - Thread.sleep(500); - clickButton(CreateWorkflowLocator.DELETE_WORKFLOW_BOTTOM); - - //click confirm delete project - clickButton(CreateWorkflowLocator.CONFIRM_DELETE_WORKFLOW_BOTTOM); - - // Whether to enter the specified page after submit - return ifTitleContains(CreateWorkflowData.WORKFLOW_TITLE); - } -} diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/ProcessInstancePage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/ProcessInstancePage.java new file mode 100644 index 0000000000..f8503e51b9 --- /dev/null +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/ProcessInstancePage.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.page.project; + +import org.apache.dolphinscheduler.common.PageCommon; +import org.apache.dolphinscheduler.locator.project.ProcessInstanceLocator; +import org.openqa.selenium.WebDriver; +import org.apache.dolphinscheduler.data.project.ProcessInstanceData; + + +public class ProcessInstancePage extends PageCommon { + public ProcessInstancePage(WebDriver driver) { + super(driver); + } + + + /** + * rerun workflow page + */ + public boolean rerunWorkflowPage() throws InterruptedException { + clickTopElement(ProcessInstanceLocator.CLICK_PROCESS_INSTANCE_NAME); + clickTopElement(ProcessInstanceLocator.CLICK_RERUN_BUTTON); + return ifTitleContains(ProcessInstanceData.PROCESS_INSTANCE_TITLE); + } + +} diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateProjectPage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/ProjectPage.java similarity index 66% rename from e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateProjectPage.java rename to e2e/src/test/java/org/apache/dolphinscheduler/page/project/ProjectPage.java index cba62a4b34..5c591d0b5d 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateProjectPage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/ProjectPage.java @@ -18,12 +18,12 @@ package org.apache.dolphinscheduler.page.project; import org.apache.dolphinscheduler.common.PageCommon; import org.apache.dolphinscheduler.constant.TestConstant; -import org.apache.dolphinscheduler.data.project.CreateProjectData; -import org.apache.dolphinscheduler.locator.project.CreateProjectLocator; +import org.apache.dolphinscheduler.data.project.ProjectData; +import org.apache.dolphinscheduler.locator.project.ProjectLocator; import org.openqa.selenium.WebDriver; -public class CreateProjectPage extends PageCommon { - public CreateProjectPage(WebDriver driver) { +public class ProjectPage extends PageCommon { + public ProjectPage(WebDriver driver) { super(driver); } @@ -31,9 +31,9 @@ public class CreateProjectPage extends PageCommon { * jump to ProjectManagePage */ public boolean jumpProjectManagePage() throws InterruptedException { - clickTopElement(CreateProjectLocator.PROJECT_MANAGE); + clickTopElement(ProjectLocator.PROJECT_MANAGE); Thread.sleep(TestConstant.ONE_THOUSAND); - return ifTitleContains(CreateProjectData.PROJECT_TITLE); + return ifTitleContains(ProjectData.PROJECT_TITLE); } /** @@ -42,17 +42,18 @@ public class CreateProjectPage extends PageCommon { * @return Whether to enter the specified page after create project */ public boolean createProject() throws InterruptedException { - clickElement(CreateProjectLocator.CREATE_PROJECT_BUTTON); + Thread.sleep(500); + clickElement(ProjectLocator.CREATE_PROJECT_BUTTON); // input create project data - sendInput(CreateProjectLocator.PROJECT_NAME, CreateProjectData.PROJECT_NAME); - sendInput(CreateProjectLocator.PROJECT_DESCRIPTION, CreateProjectData.DESCRIPTION); + sendInput(ProjectLocator.PROJECT_NAME, ProjectData.PROJECT_NAME); + sendInput(ProjectLocator.PROJECT_DESCRIPTION, ProjectData.DESCRIPTION); // click submit button - clickButton(CreateProjectLocator.SUBMIT_BUTTON); + clickButton(ProjectLocator.SUBMIT_BUTTON); // Whether to enter the specified page after submit - return ifTitleContains(CreateProjectData.PROJECT_TITLE); + return ifTitleContains(ProjectData.PROJECT_TITLE); } /** @@ -62,12 +63,12 @@ public class CreateProjectPage extends PageCommon { */ public boolean deleteProject() throws InterruptedException { //click delete project - clickElement(CreateProjectLocator.DELETE_PROJECT_BUTTON); + clickElement(ProjectLocator.DELETE_PROJECT_BUTTON); //click confirm delete project - clickElement(CreateProjectLocator.CONFIRM_DELETE_PROJECT_BUTTON); + clickElement(ProjectLocator.CONFIRM_DELETE_PROJECT_BUTTON); // Whether to enter the specified page after submit - return ifTitleContains(CreateProjectData.PROJECT_TITLE); + return ifTitleContains(ProjectData.PROJECT_TITLE); } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/TimingPage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/TimingPage.java index f74f4608ae..e8989ff8e8 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/TimingPage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/TimingPage.java @@ -64,11 +64,11 @@ public class TimingPage extends PageCommon { public boolean editTiming() throws InterruptedException { // click timing button System.out.println("Click timing management button"); + Thread.sleep(1000); clickButton(TimingLocator.CLICK_TIMING_MANAGEMENT_BUTTON); - Thread.sleep(500); + Thread.sleep(1000); System.out.println("Click edit timing button"); clickButton(TimingLocator.CLICK_EDIT_TIMING_BUTTON); - Thread.sleep(500); System.out.println("Click execution timing button"); clickButton(TimingLocator.CLICK_EXECUTION_TIMING_BUTTON); Thread.sleep(1000); diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/WorkflowDefinePage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/WorkflowDefinePage.java new file mode 100644 index 0000000000..9e462805da --- /dev/null +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/WorkflowDefinePage.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.page.project; + +import org.apache.dolphinscheduler.common.PageCommon; +import org.apache.dolphinscheduler.constant.TestConstant; +import org.apache.dolphinscheduler.data.project.WorkflowDefineData; +import org.apache.dolphinscheduler.locator.project.WorkflowDefineLocator; +import org.openqa.selenium.WebDriver; + +public class WorkflowDefinePage extends PageCommon { + public WorkflowDefinePage(WebDriver driver) { + super(driver); + } + + /** + * jump create workflow page + */ + + public boolean jumpWorkflowPage() throws InterruptedException { + // click project name + clickElement(WorkflowDefineLocator.CLICK_PROJECT_NAME); + Thread.sleep(TestConstant.ONE_THOUSAND); + + System.out.println("Click on workflow define to jump to workflow define page"); + // click workflow define + clickElement(WorkflowDefineLocator.CLICK_WORKFLOW_DEFINE); + + return ifTitleContains(WorkflowDefineData.WORKFLOW_TITLE); + } + + public boolean createWorkflow() throws InterruptedException { + System.out.println("Click create workflow button"); + // click create workflow button + clickElement(WorkflowDefineLocator.CLICK_CREATE_WORKFLOW_BUTTON); + + System.out.println("drag shell task"); + //drag shell_task + dragAndDrop(WorkflowDefineLocator.MOUSE_DOWN_AT_SHELL, WorkflowDefineLocator.MOUSE_MOVE_SHELL_AT_DAG); + + //input shell task _name + sendInput(WorkflowDefineLocator.INPUT_SHELL_TASK_NAME , WorkflowDefineData.SHELL_TASK_NAME); + + //click stop run type + clickElement(WorkflowDefineLocator.CLICK_STOP_RUN_TYPE); + + //click normal run type + clickElement(WorkflowDefineLocator.CLICK_NORMAL_RUN_TYPE); + + //input shell task description + sendInput(WorkflowDefineLocator.INPUT_SHELL_TASK_DESCRIPTION , WorkflowDefineData.SHELL_TASK_DESCRIPTION); + + //select task priority + clickElement(WorkflowDefineLocator.CLICK_TASK_PRIORITY); + clickElement(WorkflowDefineLocator.SELECT_TASK_PRIORITY); + + //select work group + clickElement(WorkflowDefineLocator.CLICK_WORK_GROUP); + clickElement(WorkflowDefineLocator.SELECT_WORK_GROUP); + + //select number of failed retries + clickElement(WorkflowDefineLocator.SELECT_FAIL_RETRIES_NUMBER); + + //select failed retry interval + clickElement(WorkflowDefineLocator.SELECT_FAIL_RETRIES_INTERVAL); + + //click timeout alarm + clickElement(WorkflowDefineLocator.CLICK_TIMEOUT_ALARM); + + //select timeout fail + clickElement(WorkflowDefineLocator.SELECT_TIMEOUT_FAIL); + + //cancel timeout alarm + clickElement(WorkflowDefineLocator.CANCEL_TIMEOUT_ALARM); + + //select timeout alarm + clickElement(WorkflowDefineLocator.SELECT_TIMEOUT_ALARM); + + //clear timeout + clearInput(WorkflowDefineLocator.SELECT_TIMEOUT); + clearInput(WorkflowDefineLocator.SELECT_TIMEOUT); + + //input timeout + sendInput(WorkflowDefineLocator.SELECT_TIMEOUT, WorkflowDefineData.INPUT_TIMEOUT); + + //click codeMirror and input script + inputCodeMirror(WorkflowDefineLocator.CLICK_CODE_MIRROR, WorkflowDefineLocator.INPUT_SCRIPT, WorkflowDefineData.SHELL_SCRIPT); + scrollToElementBottom(WorkflowDefineLocator.SCROLL_BOTTOM); + + //click custom parameters + clickElement(WorkflowDefineLocator.CLICK_CUSTOM_PARAMETERS); + + //input custom parameters + sendInput(WorkflowDefineLocator.INPUT_CUSTOM_PARAMETERS, WorkflowDefineData.INPUT_CUSTOM_PARAMETERS); + + //input custom parameters value + sendInput(WorkflowDefineLocator.INPUT_CUSTOM_PARAMETERS_VALUE, WorkflowDefineData.INPUT_CUSTOM_PARAMETERS_VALUE); + + //click add custom parameters + clickElement(WorkflowDefineLocator.CLICK_ADD_CUSTOM_PARAMETERS); + + scrollToElementBottom(WorkflowDefineLocator.SCROLL_BOTTOM); + + //input add custom parameters + sendInput(WorkflowDefineLocator.INPUT_ADD_CUSTOM_PARAMETERS, WorkflowDefineData.INPUT_ADD_CUSTOM_PARAMETERS); + + //input add custom parameters value + sendInput(WorkflowDefineLocator.INPUT_ADD_CUSTOM_PARAMETERS_VALUE, WorkflowDefineData.INPUT_ADD_CUSTOM_PARAMETERS_VALUE); + + //click delete custom parameters + clickElement(WorkflowDefineLocator.CLICK_DELETE_CUSTOM_PARAMETERS); + + //click submit button + clickElement(WorkflowDefineLocator.CLICK_SUBMIT_BUTTON); + Thread.sleep(TestConstant.ONE_THOUSAND); + System.out.println("Task node set up successfully"); + System.out.println("move to Dag Element "); + moveToDragElement(WorkflowDefineLocator.MOUSE_MOVE_SHELL_AT_DAG,-300,-100); + + return ifTitleContains(WorkflowDefineData.CREATE_WORKFLOW_TITLE); + } + + /** + * save workflow + */ + public boolean saveWorkflow() throws InterruptedException { + System.out.println("start to save workflow "); + + //click save workflow button + clickElement(WorkflowDefineLocator.CLICK_SAVE_WORKFLOW_BUTTON); + + //input workflow name + sendInput(WorkflowDefineLocator.INPUT_WORKFLOW_NAME, WorkflowDefineData.INPUT_WORKFLOW_NAME); + + //input workflow description + sendInput(WorkflowDefineLocator.INPUT_WORKFLOW_DESCRIPTION, WorkflowDefineData.INPUT_WORKFLOW_DESCRIPTION); + + //select tenant + clickElement(WorkflowDefineLocator.CLICK_TENANT); + clickElement(WorkflowDefineLocator.SELECT_TENANT); + + //click workflow timeout alarm + clickElement(WorkflowDefineLocator.CLICK_WORKFLOW_TIMEOUT_ALARM); + clearInput(WorkflowDefineLocator.INPUT_WORKFLOW_TIMEOUT); + + //input workflow timeout + sendInput(WorkflowDefineLocator.INPUT_WORKFLOW_TIMEOUT, WorkflowDefineData.INPUT_WORKFLOW_TIMEOUT); + + //click workflow global parameters + clickElement(WorkflowDefineLocator.CLICK_WORKFLOW_GLOBAL_PARAMETERS); + + //input workflow global parameters + sendInput(WorkflowDefineLocator.INPUT_WORKFLOW_GLOBAL_PARAMETERS, WorkflowDefineData.INPUT_WORKFLOW_GLOBAL_PARAMETERS); + + //input workflow global parameters value + sendInput(WorkflowDefineLocator.INPUT_WORKFLOW_GLOBAL_PARAMETERS_VALUES, WorkflowDefineData.INPUT_WORKFLOW_GLOBAL_PARAMETERS_VALUES); + + //click to add workflow global parameters + clickElement(WorkflowDefineLocator.CLICK_ADD_WORKFLOW_GLOBAL_PARAMETERS); + + //input to add workflow global parameters + sendInput(WorkflowDefineLocator.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS, WorkflowDefineData.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS); + + //input to add workflow global parameters value + sendInput(WorkflowDefineLocator.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS_VALUES, WorkflowDefineData.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS_VALUES); + + //delete workflow global parameters value + clickElement(WorkflowDefineLocator.CLICK_DELETE_WORKFLOW_GLOBAL_PARAMETERS); + Thread.sleep(TestConstant.ONE_THOUSAND); + + //click add button + System.out.println("submit workflow"); + clickButton(WorkflowDefineLocator.CLICK_ADD_BUTTON); + + return ifTitleContains(WorkflowDefineData.CREATE_WORKFLOW_TITLE); + } + + public boolean onlineWorkflow() throws InterruptedException { + clickElement(WorkflowDefineLocator.CLICK_WORKFLOW_DEFINE); + + // click online button + System.out.println("Click online workflow button"); + clickButton(WorkflowDefineLocator.CLICK_ONLINE_WORKFLOW_BUTTON); + + return ifTitleContains(WorkflowDefineData.WORKFLOW_TITLE); + } + + public boolean offlineWorkflow() throws InterruptedException { + clickElement(WorkflowDefineLocator.CLICK_WORKFLOW_DEFINE); + + // click offline button + System.out.println("offline workflow"); + Thread.sleep(500); + clickButton(WorkflowDefineLocator.CLICK_OFFLINE_WORKFLOW_BUTTON); + + return ifTitleContains(WorkflowDefineData.WORKFLOW_TITLE); + } + + + public boolean deleteWorkflow() throws InterruptedException { + //click delete workflow + Thread.sleep(500); + clickElement(WorkflowDefineLocator.CLICK_WORKFLOW_DEFINE); + clickButton(WorkflowDefineLocator.DELETE_WORKFLOW_BOTTOM); + + //click confirm delete project + clickButton(WorkflowDefineLocator.CONFIRM_DELETE_WORKFLOW_BOTTOM); + + // Whether to enter the specified page after submit + return ifTitleContains(WorkflowDefineData.WORKFLOW_TITLE); + } +} diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/testDeleteData/TestDeleteProject.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/testDeleteData/TestDeleteProject.java index bcd4291e0e..9a1374ca34 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/testDeleteData/TestDeleteProject.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/testDeleteData/TestDeleteProject.java @@ -17,15 +17,15 @@ package org.apache.dolphinscheduler.testcase.testDeleteData; import org.apache.dolphinscheduler.base.BaseTest; -import org.apache.dolphinscheduler.page.project.CreateProjectPage; +import org.apache.dolphinscheduler.page.project.ProjectPage; import org.testng.annotations.Test; public class TestDeleteProject extends BaseTest { - private CreateProjectPage createProjectPage; + private ProjectPage createProjectPage; @Test(groups={"functionTests"},dependsOnGroups = { "login","project"},description = "TestDeleteProject") public void testDeleteProject() throws InterruptedException { - createProjectPage = new CreateProjectPage(driver); + createProjectPage = new ProjectPage(driver); //jump to project manage page System.out.println("jump to the project manage page to delete project"); createProjectPage.jumpProjectManagePage(); diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/testDeleteData/TestDeleteWorkflow.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/testDeleteData/TestDeleteWorkflow.java index 2d7714e4e9..3544b19f3c 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/testDeleteData/TestDeleteWorkflow.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/testDeleteData/TestDeleteWorkflow.java @@ -17,27 +17,29 @@ package org.apache.dolphinscheduler.testcase.testDeleteData; import org.apache.dolphinscheduler.base.BaseTest; -import org.apache.dolphinscheduler.page.project.CreateProjectPage; -import org.apache.dolphinscheduler.page.project.CreateWorkflowPage; +import org.apache.dolphinscheduler.page.project.ProjectPage; +import org.apache.dolphinscheduler.page.project.WorkflowDefinePage; import org.testng.annotations.Test; public class TestDeleteWorkflow extends BaseTest { - private CreateWorkflowPage createWorkflowPage; - private CreateProjectPage createProjectPage; + private WorkflowDefinePage createWorkflowPage; + private ProjectPage createProjectPage; + /** + * offline workflow + * @throws InterruptedException + */ @Test(groups={"functionTests"},dependsOnGroups = { "login","workflow"},description = "TestDeleteWorkflow") - public void testDeleteWorkflow() throws InterruptedException { - createProjectPage = new CreateProjectPage(driver); - //jump to project manage page -// System.out.println("jump to the project manage page to delete workflow"); -// createProjectPage.jumpProjectManagePage(); - - createWorkflowPage = new CreateWorkflowPage(driver); -// createWorkflowPage.jumpWorkflowPage(); + public void testOfflineWorkflow() throws InterruptedException { + createWorkflowPage = new WorkflowDefinePage(driver); System.out.println("start offline workflow"); assert createWorkflowPage.offlineWorkflow(); System.out.println("end offline workflow"); + System.out.println("==================================="); + } + @Test(groups={"functionTests"},dependsOnGroups = { "login","workflow"},description = "TestDeleteWorkflow") + public void testDeleteWorkflow() throws InterruptedException { System.out.println("start delete workflow"); assert createWorkflowPage.deleteWorkflow(); System.out.println("end delete workflow"); diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestProcessInstance.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestProcessInstance.java new file mode 100644 index 0000000000..dc77e5f27b --- /dev/null +++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestProcessInstance.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.testcase.testProject; + +import org.apache.dolphinscheduler.base.BaseTest; +import org.apache.dolphinscheduler.page.project.ProcessInstancePage; +import org.testng.annotations.Test; + +public class TestProcessInstance extends BaseTest { + private ProcessInstancePage processInstancePage; + + @Test(groups={"functionTests","processInstance"},dependsOnGroups = { "login","workflow" },description = "testRerun") + public void testRerun() throws InterruptedException { + processInstancePage = new ProcessInstancePage(driver); + //rerun workflow + System.out.println("start rerun workflow"); + processInstancePage.rerunWorkflowPage(); + System.out.println("end rerun workflow"); + System.out.println("==================================="); + } +} diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestCreateProject.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestProject.java similarity index 81% rename from e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestCreateProject.java rename to e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestProject.java index 925cf02a8b..75c41be22b 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestCreateProject.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestProject.java @@ -17,21 +17,21 @@ package org.apache.dolphinscheduler.testcase.testProject; import org.apache.dolphinscheduler.base.BaseTest; -import org.apache.dolphinscheduler.page.project.CreateProjectPage; +import org.apache.dolphinscheduler.page.project.ProjectPage; import org.testng.annotations.Test; -public class TestCreateProject extends BaseTest { - private CreateProjectPage createProjectPage; +public class TestProject extends BaseTest { + private ProjectPage projectPage; @Test(groups={"functionTests","project"},dependsOnGroups = { "login" },description = "CreateProjectTest") public void testCreateProject() throws InterruptedException { - createProjectPage = new CreateProjectPage(driver); + projectPage = new ProjectPage(driver); // enter user manage page System.out.println("jump to the projectManage page to create project "); - createProjectPage.jumpProjectManagePage(); + projectPage.jumpProjectManagePage(); //assert user manage page System.out.println("start create project"); - assert createProjectPage.createProject(); + assert projectPage.createProject(); System.out.println("end create project"); System.out.println("==================================="); } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestRunWorkflow.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestRunWorkflow.java index 9d641bd2fb..70bb7774a7 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestRunWorkflow.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestRunWorkflow.java @@ -17,14 +17,14 @@ package org.apache.dolphinscheduler.testcase.testProject; import org.apache.dolphinscheduler.base.BaseTest; -import org.apache.dolphinscheduler.page.project.CreateProjectPage; -import org.apache.dolphinscheduler.page.project.CreateWorkflowPage; +import org.apache.dolphinscheduler.page.project.ProjectPage; +import org.apache.dolphinscheduler.page.project.WorkflowDefinePage; import org.apache.dolphinscheduler.page.project.RunWorkflowPage; import org.testng.annotations.Test; public class TestRunWorkflow extends BaseTest { - private CreateWorkflowPage createWorkflowPage; - private CreateProjectPage createProjectPage; + private WorkflowDefinePage createWorkflowPage; + private ProjectPage projectPage; private RunWorkflowPage runWorkflowPage; @@ -32,7 +32,7 @@ public class TestRunWorkflow extends BaseTest { public void testRunWorkflow() throws InterruptedException { runWorkflowPage = new RunWorkflowPage(driver); - createProjectPage = new CreateProjectPage(driver); + projectPage = new ProjectPage(driver); System.out.println("start run workflow"); assert runWorkflowPage.runWorkflow(); System.out.println("end run workflow"); diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestCreateWorkflow.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestWorkflowDefine.java similarity index 59% rename from e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestCreateWorkflow.java rename to e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestWorkflowDefine.java index bac5b1a43c..31df36d2a1 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestCreateWorkflow.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestWorkflowDefine.java @@ -17,32 +17,40 @@ package org.apache.dolphinscheduler.testcase.testProject; import org.apache.dolphinscheduler.base.BaseTest; -import org.apache.dolphinscheduler.page.project.CreateProjectPage; -import org.apache.dolphinscheduler.page.project.CreateWorkflowPage; +import org.apache.dolphinscheduler.page.project.ProjectPage; +import org.apache.dolphinscheduler.page.project.WorkflowDefinePage; import org.testng.annotations.Test; -public class TestCreateWorkflow extends BaseTest { - private CreateWorkflowPage createWorkflowPage; - private CreateProjectPage createProjectPage; +public class TestWorkflowDefine extends BaseTest { + private WorkflowDefinePage workflowDefinePage; + private ProjectPage projectPage; - - @Test(groups={"functionTests","workflow"},dependsOnGroups = { "login" },description = "TestCreateWorkflow") + /** + * test Create WorkflowDefine + * @throws InterruptedException + */ + @Test(groups={"functionTests","workflow"},dependsOnGroups = { "login" },description = "TestWorkflowDefine") public void testCreateWorkflow() throws InterruptedException { - createProjectPage = new CreateProjectPage(driver); + projectPage = new ProjectPage(driver); System.out.println("jump to the projectManage page to create workflow"); - createProjectPage.jumpProjectManagePage(); + projectPage.jumpProjectManagePage(); - createWorkflowPage = new CreateWorkflowPage(driver); + workflowDefinePage = new WorkflowDefinePage(driver); System.out.println("Click on the project name to jump to the project homepage"); - createWorkflowPage.jumpWorkflowPage(); + workflowDefinePage.jumpWorkflowPage(); System.out.println("start create workflow"); - assert createWorkflowPage.createWorkflow(); - assert createWorkflowPage.saveWorkflow(); + assert workflowDefinePage.createWorkflow(); + assert workflowDefinePage.saveWorkflow(); System.out.println("end create workflow"); + System.out.println("==================================="); + } + @Test(groups={"functionTests","workflow"},dependsOnGroups = { "login" },description = "TestOnlineWorkflow") + public void testOnlineWorkflow() throws InterruptedException { + workflowDefinePage = new WorkflowDefinePage(driver); System.out.println("start online workflow"); - assert createWorkflowPage.onlineWorkflow(); + assert workflowDefinePage.onlineWorkflow(); System.out.println("end online workflow"); System.out.println("==================================="); } diff --git a/e2e/testng.xml b/e2e/testng.xml index ee09e197d9..32bd0c6009 100644 --- a/e2e/testng.xml +++ b/e2e/testng.xml @@ -26,6 +26,7 @@ + @@ -37,8 +38,19 @@ - - + + + + + + + + + + + + + @@ -50,8 +62,19 @@ + + + + + + + + + + + + -