From d01e19321a9d9aed0a2cc83772d8153d77727902 Mon Sep 17 00:00:00 2001 From: dailidong Date: Thu, 5 Mar 2020 23:53:12 +0800 Subject: [PATCH 001/401] fix actions/checkout@v2 problem (#2095) --- .github/workflows/ci_backend.yml | 12 ++++++++++++ .github/workflows/ci_e2e.yml | 6 ++++++ .github/workflows/ci_frontend.yml | 12 ++++++++++++ .github/workflows/ci_ut.yml | 6 ++++++ 4 files changed, 36 insertions(+) diff --git a/.github/workflows/ci_backend.yml b/.github/workflows/ci_backend.yml index 1ca15c2813..0273251e99 100644 --- a/.github/workflows/ci_backend.yml +++ b/.github/workflows/ci_backend.yml @@ -46,6 +46,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: @@ -56,6 +62,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml index 7d9c5e3e72..fe818d0668 100644 --- a/.github/workflows/ci_e2e.yml +++ b/.github/workflows/ci_e2e.yml @@ -30,6 +30,12 @@ jobs: steps: - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - uses: actions/cache@v1 with: path: ~/.m2/repository diff --git a/.github/workflows/ci_frontend.yml b/.github/workflows/ci_frontend.yml index 1f959b40eb..494d12dbae 100644 --- a/.github/workflows/ci_frontend.yml +++ b/.github/workflows/ci_frontend.yml @@ -35,6 +35,12 @@ jobs: os: [ubuntu-latest, macos-latest] steps: - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - name: Set up Node.js uses: actions/setup-node@v1 with: @@ -50,6 +56,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index e7e47437d4..48cd1e16ce 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -30,6 +30,12 @@ jobs: steps: - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - uses: actions/cache@v1 with: path: ~/.m2/repository From fdc9e0090b094e03063b62e476942f166f5e66df Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Sat, 7 Mar 2020 18:07:53 +0800 Subject: [PATCH 002/401] remove incorrect or unnecessary URL tag (#2099) --- dolphinscheduler-common/pom.xml | 2 +- dolphinscheduler-dao/pom.xml | 2 +- dolphinscheduler-remote/pom.xml | 2 -- dolphinscheduler-server/pom.xml | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index e7789f724b..ca75a84a62 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -25,7 +25,7 @@ dolphinscheduler-common dolphinscheduler-common - http://maven.apache.org + jar UTF-8 diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml index fd43458f87..3aea888f94 100644 --- a/dolphinscheduler-dao/pom.xml +++ b/dolphinscheduler-dao/pom.xml @@ -25,7 +25,7 @@ dolphinscheduler-dao ${project.artifactId} - http://maven.apache.org + UTF-8 diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml index b67b033ffa..39c7c6a7c0 100644 --- a/dolphinscheduler-remote/pom.xml +++ b/dolphinscheduler-remote/pom.xml @@ -12,8 +12,6 @@ dolphinscheduler-remote dolphinscheduler-remote - - http://www.example.com UTF-8 diff --git a/dolphinscheduler-server/pom.xml b/dolphinscheduler-server/pom.xml index 86490197b6..e8e84297e2 100644 --- a/dolphinscheduler-server/pom.xml +++ b/dolphinscheduler-server/pom.xml @@ -25,7 +25,7 @@ dolphinscheduler-server dolphinscheduler-server - http://maven.apache.org + jar UTF-8 From 58046b47f676ce0e0e0df69590c22b92bd071532 Mon Sep 17 00:00:00 2001 From: liwenhe1993 <32166572+liwenhe1993@users.noreply.github.com> Date: Sun, 8 Mar 2020 15:56:24 +0800 Subject: [PATCH 003/401] Add docker internal server check script (#2092) * add environment variables add checkpoint.sh file to docker image 1. add `POSTGRESQL_DATABASE` environment variable 2. add `DOLPHINSCHEDULER_DATA_BASEDIR_PATH` environment variable 3. add `DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH` environment variable 4. add `DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH` environment variable 5. add checkpoint.sh file to check server process 6. modify `README.md` and `README_zh_CN.md` * add environment variables add checkpoint.sh file to docker image 1. add `POSTGRESQL_DATABASE` environment variable 2. add `DOLPHINSCHEDULER_DATA_BASEDIR_PATH` environment variable 3. add `DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH` environment variable 4. add `DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH` environment variable 5. add checkpoint.sh file to check server process 6. modify `README.md` and `README_zh_CN.md` * add logger port and modify dolphinscheduler_env to dolphinscheduler_env.sh --- dockerfile/Dockerfile | 13 +++--- dockerfile/README.md | 40 ++++++++++++++----- dockerfile/README_zh_CN.md | 38 ++++++++++++++---- dockerfile/checkpoint.sh | 27 +++++++++++++ .../application.properties.tpl | 2 +- .../dolphinscheduler/common.properties.tpl | 12 +++--- ...nscheduler_env => dolphinscheduler_env.sh} | 0 .../dolphinscheduler/quartz.properties.tpl | 2 +- dockerfile/startup-init-conf.sh | 4 ++ 9 files changed, 108 insertions(+), 30 deletions(-) create mode 100644 dockerfile/checkpoint.sh rename dockerfile/conf/dolphinscheduler/env/{dolphinscheduler_env => dolphinscheduler_env.sh} (100%) diff --git a/dockerfile/Dockerfile b/dockerfile/Dockerfile index 1fc064c489..8fa4886f31 100644 --- a/dockerfile/Dockerfile +++ b/dockerfile/Dockerfile @@ -63,19 +63,22 @@ RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \ ADD ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d #7. add configuration and modify permissions and set soft links +ADD ./checkpoint.sh /root/checkpoint.sh ADD ./startup-init-conf.sh /root/startup-init-conf.sh ADD ./startup.sh /root/startup.sh ADD ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/ -ADD ./conf/dolphinscheduler/env/dolphinscheduler_env /opt/dolphinscheduler/conf/env/ -RUN chmod +x /root/startup-init-conf.sh && \ +ADD conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/ +RUN chmod +x /root/checkpoint.sh && \ + chmod +x /root/startup-init-conf.sh && \ chmod +x /root/startup.sh && \ - chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env && \ + chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \ chmod +x /opt/dolphinscheduler/script/*.sh && \ chmod +x /opt/dolphinscheduler/bin/*.sh && \ chmod +x /opt/zookeeper/bin/*.sh && \ + dos2unix /root/checkpoint.sh && \ dos2unix /root/startup-init-conf.sh && \ dos2unix /root/startup.sh && \ - dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env && \ + dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \ dos2unix /opt/dolphinscheduler/script/*.sh && \ dos2unix /opt/dolphinscheduler/bin/*.sh && \ dos2unix /opt/zookeeper/bin/*.sh && \ @@ -87,6 +90,6 @@ RUN chmod +x /root/startup-init-conf.sh && \ RUN rm -rf /var/cache/apk/* #9. expose port -EXPOSE 2181 2888 3888 5432 12345 8888 +EXPOSE 2181 2888 3888 5432 12345 50051 8888 ENTRYPOINT ["/root/startup.sh"] \ No newline at end of file diff --git a/dockerfile/README.md b/dockerfile/README.md index b5a9d0d3aa..60af7fad8f 100644 --- a/dockerfile/README.md +++ b/dockerfile/README.md @@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org #### You can start a dolphinscheduler instance ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test \ +-e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \ -p 8888:8888 \ dolphinscheduler all ``` @@ -25,13 +25,13 @@ The default postgres user `root`, postgres password `root` and database `dolphin The default zookeeper is created in the `startup.sh`. -#### Or via Environment Variables **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`ZOOKEEPER_QUORUM`** +#### Or via Environment Variables **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`POSTGRESQL_DATABASE`** **`ZOOKEEPER_QUORUM`** You can specify **existing postgres service**. Example: ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -p 8888:8888 \ dolphinscheduler all @@ -42,7 +42,7 @@ You can specify **existing zookeeper service**. Example: ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -p 8888:8888 \ dolphinscheduler all ``` @@ -56,7 +56,7 @@ You can start a standalone dolphinscheduler server. ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler master-server ``` @@ -66,7 +66,7 @@ dolphinscheduler master-server ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler worker-server ``` @@ -75,7 +75,7 @@ dolphinscheduler worker-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -p 12345:12345 \ dolphinscheduler api-server @@ -85,7 +85,7 @@ dolphinscheduler api-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler alert-server ``` @@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \ dolphinscheduler frontend ``` -**Note**: You must be specify `POSTGRESQL_HOST` `POSTGRESQL_PORT` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server. +**Note**: You must be specify `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server. ## How to build a docker image @@ -140,14 +140,36 @@ This environment variable sets the port for PostgreSQL. The default value is `54 This environment variable sets the username for PostgreSQL. The default value is `root`. +**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. + **`POSTGRESQL_PASSWORD`** This environment variable sets the password for PostgreSQL. The default value is `root`. +**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. + +**`POSTGRESQL_DATABASE`** + +This environment variable sets the database for PostgreSQL. The default value is `dolphinscheduler`. + +**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. + **`DOLPHINSCHEDULER_ENV_PATH`** This environment variable sets the runtime environment for task. The default value is `/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh`. +**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`** + +User data directory path, self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler` + +**`DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH`** + +Directory path for user data download. self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler/download` + +**`DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH`** + +Process execute directory. self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler/exec` + **`TASK_QUEUE`** This environment variable sets the task queue for `master-server` and `worker-serverr`. The default value is `zookeeper`. diff --git a/dockerfile/README_zh_CN.md b/dockerfile/README_zh_CN.md index 5830e30b07..900c8b50d9 100644 --- a/dockerfile/README_zh_CN.md +++ b/dockerfile/README_zh_CN.md @@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org #### 你可以运行一个dolphinscheduler实例 ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test \ +-e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \ -p 8888:8888 \ dolphinscheduler all ``` @@ -31,7 +31,7 @@ dolphinscheduler all ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -p 8888:8888 \ dolphinscheduler all @@ -42,7 +42,7 @@ dolphinscheduler all ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -p 8888:8888 \ dolphinscheduler all ``` @@ -56,7 +56,7 @@ dolphinscheduler all ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler master-server ``` @@ -66,7 +66,7 @@ dolphinscheduler master-server ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler worker-server ``` @@ -75,7 +75,7 @@ dolphinscheduler worker-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -p 12345:12345 \ dolphinscheduler api-server @@ -85,7 +85,7 @@ dolphinscheduler api-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler alert-server ``` @@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \ dolphinscheduler frontend ``` -**注意**: 当你运行dolphinscheduler中的部分服务时，你必须指定这些环境变量 `POSTGRESQL_HOST` `POSTGRESQL_PORT` `ZOOKEEPER_QUORUM`。 +**注意**: 当你运行dolphinscheduler中的部分服务时，你必须指定这些环境变量 `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM`。 ## 如何构建一个docker镜像 @@ -140,14 +140,36 @@ Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些 配置`PostgreSQL`的`USERNAME`， 默认值 `root`。 +**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 + **`POSTGRESQL_PASSWORD`** 配置`PostgreSQL`的`PASSWORD`， 默认值 `root`。 +**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 + +**`POSTGRESQL_DATABASE`** + +配置`PostgreSQL`的`DATABASE`， 默认值 `dolphinscheduler`。 + +**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 + **`DOLPHINSCHEDULER_ENV_PATH`** 任务执行时的环境变量配置文件， 默认值 `/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh`。 +**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`** + +用户数据目录, 用户自己配置, 请确保这个目录存在并且用户读写权限， 默认值 `/tmp/dolphinscheduler`。 + +**`DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH`** + +用户数据下载目录, 用户自己配置, 请确保这个目录存在并且用户读写权限， 默认值 `/tmp/dolphinscheduler/download`。 + +**`DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH`** + +任务执行目录, 用户自己配置, 请确保这个目录存在并且用户读写权限， 默认值 `/tmp/dolphinscheduler/exec`。 + **`TASK_QUEUE`** 配置`master-server`和`worker-serverr`的`Zookeeper`任务队列名， 默认值 `zookeeper`。 diff --git a/dockerfile/checkpoint.sh b/dockerfile/checkpoint.sh new file mode 100644 index 0000000000..cd2774f9ce --- /dev/null +++ b/dockerfile/checkpoint.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +if [ "$(ps -ef | grep java | grep -c $1)" -eq 0 ]; then + echo "[ERROR] $1 process not exits." + exit 1 +else + echo "[INFO] $1 process exits." + exit 0 +fi diff --git a/dockerfile/conf/dolphinscheduler/application.properties.tpl b/dockerfile/conf/dolphinscheduler/application.properties.tpl index c643c414cd..6dd8a18e11 100644 --- a/dockerfile/conf/dolphinscheduler/application.properties.tpl +++ b/dockerfile/conf/dolphinscheduler/application.properties.tpl @@ -19,7 +19,7 @@ spring.datasource.type=com.alibaba.druid.pool.DruidDataSource # postgre spring.datasource.driver-class-name=org.postgresql.Driver -spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/dolphinscheduler?characterEncoding=utf8 +spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8 # mysql #spring.datasource.driver-class-name=com.mysql.jdbc.Driver #spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 diff --git a/dockerfile/conf/dolphinscheduler/common.properties.tpl b/dockerfile/conf/dolphinscheduler/common.properties.tpl index ea03e0b78e..8134fc7a9b 100644 --- a/dockerfile/conf/dolphinscheduler/common.properties.tpl +++ b/dockerfile/conf/dolphinscheduler/common.properties.tpl @@ -38,6 +38,12 @@ dolphinscheduler.env.path=${DOLPHINSCHEDULER_ENV_PATH} resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties # is development state? default "false" development.state=true +# user data directory path, self configuration, please make sure the directory exists and have read write permissions +data.basedir.path=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH} +# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions +data.download.basedir.path=${DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH} +# process execute directory. self configuration, please make sure the directory exists and have read write permissions +process.exec.basepath=${DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH} # resource upload startup type : HDFS,S3,NONE res.upload.startup.type=NONE @@ -49,12 +55,6 @@ res.upload.startup.type=NONE hdfs.root.user=hdfs # data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended data.store2hdfs.basepath=/dolphinscheduler -# user data directory path, self configuration, please make sure the directory exists and have read write permissions -data.basedir.path=/tmp/dolphinscheduler -# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions -data.download.basedir.path=/tmp/dolphinscheduler/download -# process execute directory. self configuration, please make sure the directory exists and have read write permissions -process.exec.basepath=/tmp/dolphinscheduler/exec # whether kerberos starts hadoop.security.authentication.startup.state=false # java.security.krb5.conf path diff --git a/dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env b/dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh similarity index 100% rename from dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env rename to dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh diff --git a/dockerfile/conf/dolphinscheduler/quartz.properties.tpl b/dockerfile/conf/dolphinscheduler/quartz.properties.tpl index de5496bc3d..7c7c92e425 100644 --- a/dockerfile/conf/dolphinscheduler/quartz.properties.tpl +++ b/dockerfile/conf/dolphinscheduler/quartz.properties.tpl @@ -22,7 +22,7 @@ org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.PostgreSQLDelegate # postgre org.quartz.dataSource.myDs.driver = org.postgresql.Driver -org.quartz.dataSource.myDs.URL = jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/dolphinscheduler?characterEncoding=utf8 +org.quartz.dataSource.myDs.URL = jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8 org.quartz.dataSource.myDs.user = ${POSTGRESQL_USERNAME} org.quartz.dataSource.myDs.password = ${POSTGRESQL_PASSWORD} org.quartz.scheduler.instanceName = DolphinScheduler diff --git a/dockerfile/startup-init-conf.sh b/dockerfile/startup-init-conf.sh index db37976168..d2b50fa3e7 100644 --- a/dockerfile/startup-init-conf.sh +++ b/dockerfile/startup-init-conf.sh @@ -28,11 +28,15 @@ export POSTGRESQL_HOST=${POSTGRESQL_HOST:-"127.0.0.1"} export POSTGRESQL_PORT=${POSTGRESQL_PORT:-"5432"} export POSTGRESQL_USERNAME=${POSTGRESQL_USERNAME:-"root"} export POSTGRESQL_PASSWORD=${POSTGRESQL_PASSWORD:-"root"} +export POSTGRESQL_DATABASE=${POSTGRESQL_DATABASE:-"dolphinscheduler"} #============================================================================ # System #============================================================================ export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"} +export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"} +export DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH:-"/tmp/dolphinscheduler/download"} +export DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH=${DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH:-"/tmp/dolphinscheduler/exec"} #============================================================================ # Zookeeper From 12841bb4af33e07742c58ed63bc78c993058d4b8 Mon Sep 17 00:00:00 2001 From: tswstarplanet Date: Sun, 8 Mar 2020 15:57:06 +0800 Subject: [PATCH 004/401] use stringbuilder to append string (#2108) --- .../service/zk/AbstractZKClient.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java index 135bfdabc6..fa1a0bfced 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java @@ -65,14 +65,16 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator { if (splits.length != Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH){ return; } - String str = splits[0] + Constants.COMMA - + splits[1] + Constants.COMMA - + OSUtils.cpuUsage() + Constants.COMMA - + OSUtils.memoryUsage() + Constants.COMMA - + OSUtils.loadAverage() + Constants.COMMA - + splits[5] + Constants.COMMA - + DateUtils.dateToString(new Date()); - zkClient.setData().forPath(znode,str.getBytes()); + StringBuilder sb = new StringBuilder(); + sb.append(splits[0]).append(Constants.COMMA) + .append(splits[1]).append(Constants.COMMA) + .append(OSUtils.cpuUsage()).append(Constants.COMMA) + .append(OSUtils.memoryUsage()).append(Constants.COMMA) + .append(OSUtils.loadAverage()).append(Constants.COMMA) + .append(splits[5]).append(Constants.COMMA) + .append(DateUtils.dateToString(new Date())); + + zkClient.setData().forPath(znode, sb.toString().getBytes()); } catch (Exception e) { logger.error("heartbeat for zk failed", e); From bd43bed64d718c3bcd06c409074ea7ec3e37e892 Mon Sep 17 00:00:00 2001 From: Rubik-W <39549317+Rubik-W@users.noreply.github.com> Date: Sun, 8 Mar 2020 16:00:47 +0800 Subject: [PATCH 005/401] fix: remove redundant import class (#2110) --- .../org/apache/dolphinscheduler/remote/command/Command.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java index 86ba79c884..ed46e1ff51 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java @@ -16,8 +16,6 @@ */ package org.apache.dolphinscheduler.remote.command; -import com.sun.org.apache.regexp.internal.RE; - import java.io.Serializable; import java.util.concurrent.atomic.AtomicLong; From 0209cb7431ee029a6205bddc36d10c4de423e956 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Sun, 8 Mar 2020 21:22:07 +0800 Subject: [PATCH 006/401] move UT class to reasonable package (#2116) --- .../dolphinscheduler/common}/utils/PreconditionsTest.java | 3 +-- .../dolphinscheduler/service/quartz}/cron/CronUtilsTest.java | 3 +-- .../dolphinscheduler/service}/queue/BaseTaskQueueTest.java | 2 +- .../dolphinscheduler/service}/queue/TaskQueueZKImplTest.java | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) rename {dolphinscheduler-service/src/test/java => dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common}/utils/PreconditionsTest.java (98%) rename dolphinscheduler-service/src/test/java/{ => org/apache/dolphinscheduler/service/quartz}/cron/CronUtilsTest.java (99%) rename dolphinscheduler-service/src/test/java/{ => org/apache/dolphinscheduler/service}/queue/BaseTaskQueueTest.java (96%) rename dolphinscheduler-service/src/test/java/{ => org/apache/dolphinscheduler/service}/queue/TaskQueueZKImplTest.java (99%) diff --git a/dolphinscheduler-service/src/test/java/utils/PreconditionsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java similarity index 98% rename from dolphinscheduler-service/src/test/java/utils/PreconditionsTest.java rename to dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java index a1b85f1b12..47b24bb93c 100644 --- a/dolphinscheduler-service/src/test/java/utils/PreconditionsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java @@ -14,9 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package utils; +package org.apache.dolphinscheduler.common.utils; -import org.apache.dolphinscheduler.common.utils.Preconditions; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-service/src/test/java/cron/CronUtilsTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtilsTest.java similarity index 99% rename from dolphinscheduler-service/src/test/java/cron/CronUtilsTest.java rename to dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtilsTest.java index 6a402b5e67..b4f864c5b4 100644 --- a/dolphinscheduler-service/src/test/java/cron/CronUtilsTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtilsTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package cron; +package org.apache.dolphinscheduler.service.quartz.cron; import com.cronutils.builder.CronBuilder; import com.cronutils.model.Cron; @@ -25,7 +25,6 @@ import com.cronutils.model.field.CronFieldName; import com.cronutils.model.field.expression.*; import org.apache.dolphinscheduler.common.enums.CycleEnum; import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/BaseTaskQueueTest.java similarity index 96% rename from dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java rename to dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/BaseTaskQueueTest.java index 97ab9969a3..17e2ae4056 100644 --- a/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/BaseTaskQueueTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package queue; +package org.apache.dolphinscheduler.service.queue; import org.apache.dolphinscheduler.service.queue.ITaskQueue; import org.apache.dolphinscheduler.service.queue.TaskQueueFactory; diff --git a/dolphinscheduler-service/src/test/java/queue/TaskQueueZKImplTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskQueueZKImplTest.java similarity index 99% rename from dolphinscheduler-service/src/test/java/queue/TaskQueueZKImplTest.java rename to dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskQueueZKImplTest.java index d29c5aa610..5d464ac3c9 100644 --- a/dolphinscheduler-service/src/test/java/queue/TaskQueueZKImplTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskQueueZKImplTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package queue; +package org.apache.dolphinscheduler.service.queue; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.IpUtils; From 6a8cef08e4873816eb039092f3891f0d943fadeb Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Sun, 8 Mar 2020 21:28:49 +0800 Subject: [PATCH 007/401] Adapting partial code(file name start with E) to the sonar cloud rule (#1999) * Adapting partial code(file name start with E) to the sonar cloud rule * remove isEmpty invoke * resolve conflicts --- .../manager/EnterpriseWeChatManager.java | 4 +- .../alert/utils/EnterpriseWeChatUtils.java | 38 +++++++++---------- .../utils/EnterpriseWeChatUtilsTest.java | 8 ++-- .../api/controller/ExecutorController.java | 2 +- .../api/service/ExecutorService.java | 15 +++----- .../dao/mapper/ErrorCommandMapperTest.java | 4 +- 6 files changed, 34 insertions(+), 37 deletions(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java index 9bcad56c24..bb06be6561 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java @@ -42,8 +42,8 @@ public class EnterpriseWeChatManager { public Map send(Alert alert, String token){ Map retMap = new HashMap<>(); retMap.put(Constants.STATUS, false); - String agentId = EnterpriseWeChatUtils.enterpriseWeChatAgentId; - String users = EnterpriseWeChatUtils.enterpriseWeChatUsers; + String agentId = EnterpriseWeChatUtils.ENTERPRISE_WE_CHAT_AGENT_ID; + String users = EnterpriseWeChatUtils.ENTERPRISE_WE_CHAT_USERS; List userList = Arrays.asList(users.split(",")); logger.info("send message {}",alert); String msg = EnterpriseWeChatUtils.makeUserSendMsg(userList, agentId,EnterpriseWeChatUtils.markdownByAlert(alert)); diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java index ff8822421a..900c120cd4 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java @@ -43,24 +43,24 @@ public class EnterpriseWeChatUtils { public static final Logger logger = LoggerFactory.getLogger(EnterpriseWeChatUtils.class); - private static final String enterpriseWeChatCorpId = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_CORP_ID); + private static final String ENTERPRISE_WE_CHAT_CORP_ID = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_CORP_ID); - private static final String enterpriseWeChatSecret = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_SECRET); + private static final String ENTERPRISE_WE_CHAT_SECRET = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_SECRET); - private static final String enterpriseWeChatTokenUrl = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TOKEN_URL); - private static String enterpriseWeChatTokenUrlReplace = enterpriseWeChatTokenUrl - .replaceAll("\\$corpId", enterpriseWeChatCorpId) - .replaceAll("\\$secret", enterpriseWeChatSecret); + private static final String ENTERPRISE_WE_CHAT_TOKEN_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TOKEN_URL); + private static final String ENTERPRISE_WE_CHAT_TOKEN_URL_REPLACE = ENTERPRISE_WE_CHAT_TOKEN_URL + .replaceAll("\\$corpId", ENTERPRISE_WE_CHAT_CORP_ID) + .replaceAll("\\$secret", ENTERPRISE_WE_CHAT_SECRET); - private static final String enterpriseWeChatPushUrl = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_PUSH_URL); + private static final String ENTERPRISE_WE_CHAT_PUSH_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_PUSH_URL); - private static final String enterpriseWeChatTeamSendMsg = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TEAM_SEND_MSG); + private static final String ENTERPRISE_WE_CHAT_TEAM_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TEAM_SEND_MSG); - private static final String enterpriseWeChatUserSendMsg = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USER_SEND_MSG); + private static final String ENTERPRISE_WE_CHAT_USER_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USER_SEND_MSG); - public static final String enterpriseWeChatAgentId = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_AGENT_ID); + public static final String ENTERPRISE_WE_CHAT_AGENT_ID = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_AGENT_ID); - public static final String enterpriseWeChatUsers = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USERS); + public static final String ENTERPRISE_WE_CHAT_USERS = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USERS); /** * get Enterprise WeChat is enable @@ -87,7 +87,7 @@ public class EnterpriseWeChatUtils { CloseableHttpClient httpClient = HttpClients.createDefault(); try { - HttpGet httpGet = new HttpGet(enterpriseWeChatTokenUrlReplace); + HttpGet httpGet = new HttpGet(ENTERPRISE_WE_CHAT_TOKEN_URL_REPLACE); CloseableHttpResponse response = httpClient.execute(httpGet); try { HttpEntity entity = response.getEntity(); @@ -114,7 +114,7 @@ public class EnterpriseWeChatUtils { * @return Enterprise WeChat send message */ public static String makeTeamSendMsg(String toParty, String agentId, String msg) { - return enterpriseWeChatTeamSendMsg.replaceAll("\\$toParty", toParty) + return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\$toParty", toParty) .replaceAll("\\$agentId", agentId) .replaceAll("\\$msg", msg); } @@ -128,7 +128,7 @@ public class EnterpriseWeChatUtils { */ public static String makeTeamSendMsg(Collection toParty, String agentId, String msg) { String listParty = FuncUtils.mkString(toParty, "|"); - return enterpriseWeChatTeamSendMsg.replaceAll("\\$toParty", listParty) + return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\$toParty", listParty) .replaceAll("\\$agentId", agentId) .replaceAll("\\$msg", msg); } @@ -141,7 +141,7 @@ public class EnterpriseWeChatUtils { * @return Enterprise WeChat send message */ public static String makeUserSendMsg(String toUser, String agentId, String msg) { - return enterpriseWeChatUserSendMsg.replaceAll("\\$toUser", toUser) + return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\$toUser", toUser) .replaceAll("\\$agentId", agentId) .replaceAll("\\$msg", msg); } @@ -155,7 +155,7 @@ public class EnterpriseWeChatUtils { */ public static String makeUserSendMsg(Collection toUser, String agentId, String msg) { String listUser = FuncUtils.mkString(toUser, "|"); - return enterpriseWeChatUserSendMsg.replaceAll("\\$toUser", listUser) + return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\$toUser", listUser) .replaceAll("\\$agentId", agentId) .replaceAll("\\$msg", msg); } @@ -169,7 +169,7 @@ public class EnterpriseWeChatUtils { * @throws IOException the IOException */ public static String sendEnterpriseWeChat(String charset, String data, String token) throws IOException { - String enterpriseWeChatPushUrlReplace = enterpriseWeChatPushUrl.replaceAll("\\$token", token); + String enterpriseWeChatPushUrlReplace = ENTERPRISE_WE_CHAT_PUSH_URL.replaceAll("\\$token", token); CloseableHttpClient httpClient = HttpClients.createDefault(); try { @@ -184,8 +184,8 @@ public class EnterpriseWeChatUtils { } finally { response.close(); } - logger.info("Enterprise WeChat send [{}], param:{}, resp:{}", - enterpriseWeChatPushUrl, data, resp); + logger.info("Enterprise WeChat send [{}], param:{}, resp:{}", + ENTERPRISE_WE_CHAT_PUSH_URL, data, resp); return resp; } finally { httpClient.close(); diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java index 3471f6efdd..15b92a622e 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java @@ -53,7 +53,7 @@ public class EnterpriseWeChatUtilsTest { String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); String errmsg = JSON.parseObject(resp).getString("errmsg"); - Assert.assertEquals(errmsg, "ok"); + Assert.assertEquals("ok",errmsg); } catch (IOException e) { e.printStackTrace(); } @@ -68,7 +68,7 @@ public class EnterpriseWeChatUtilsTest { String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); String errmsg = JSON.parseObject(resp).getString("errmsg"); - Assert.assertEquals(errmsg, "ok"); + Assert.assertEquals("ok",errmsg); } catch (IOException e) { e.printStackTrace(); } @@ -95,7 +95,7 @@ public class EnterpriseWeChatUtilsTest { String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); String errmsg = JSON.parseObject(resp).getString("errmsg"); - Assert.assertEquals(errmsg, "ok"); + Assert.assertEquals("ok",errmsg); } catch (IOException e) { e.printStackTrace(); } @@ -110,7 +110,7 @@ public class EnterpriseWeChatUtilsTest { String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); String errmsg = JSON.parseObject(resp).getString("errmsg"); - Assert.assertEquals(errmsg, "ok"); + Assert.assertEquals("ok",errmsg); } catch (IOException e) { e.printStackTrace(); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java index cae4993942..ffedd5703c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java @@ -149,7 +149,7 @@ public class ExecutorController extends BaseController { ) { try { logger.info("execute command, login user: {}, project:{}, process instance id:{}, execute type:{}", - loginUser.getUserName(), projectName, processInstanceId, executeType.toString()); + loginUser.getUserName(), projectName, processInstanceId, executeType); Map result = execService.execute(loginUser, projectName, processInstanceId, executeType); return returnDataList(result); } catch (Exception e) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index 152292a21b..86b507f0a0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -259,10 +259,7 @@ public class ExecutorService extends BaseService{ // checkTenantExists(); Tenant tenant = processService.getTenantForProcess(processDefinition.getTenantId(), processDefinition.getUserId()); - if(tenant == null){ - return false; - } - return true; + return tenant != null; } /** @@ -298,6 +295,7 @@ public class ExecutorService extends BaseService{ if (executionStatus.typeIsPause()|| executionStatus.typeIsCancel()) { checkResult = true; } + break; default: break; } @@ -369,7 +367,7 @@ public class ExecutorService extends BaseService{ * @return check result code */ public Map startCheckByProcessDefinedId(int processDefineId) { - Map result = new HashMap(); + Map result = new HashMap<>(); if (processDefineId == 0){ logger.error("process definition id is null"); @@ -378,10 +376,9 @@ public class ExecutorService extends BaseService{ List ids = new ArrayList<>(); processService.recurseFindSubProcessId(processDefineId, ids); Integer[] idArray = ids.toArray(new Integer[ids.size()]); - if (ids.size() > 0){ - List processDefinitionList; - processDefinitionList = processDefinitionMapper.queryDefinitionListByIdList(idArray); - if (processDefinitionList != null && processDefinitionList.size() > 0){ + if (!ids.isEmpty()){ + List processDefinitionList = processDefinitionMapper.queryDefinitionListByIdList(idArray); + if (processDefinitionList != null){ for (ProcessDefinition processDefinition : processDefinitionList){ /** * if there is no online process, exit directly diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java index 3653e6a53d..5fb7dfc09f 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java @@ -67,7 +67,7 @@ public class ErrorCommandMapperTest { //update errorCommand.setUpdateTime(new Date()); int update = errorCommandMapper.updateById(errorCommand); - Assert.assertEquals(update, 1); + Assert.assertEquals(1,update); errorCommandMapper.deleteById(errorCommand.getId()); } @@ -79,7 +79,7 @@ public class ErrorCommandMapperTest { ErrorCommand errorCommand = insertOne(); int delete = errorCommandMapper.deleteById(errorCommand.getId()); - Assert.assertEquals(delete, 1); + Assert.assertEquals(1,delete); } /** From 493499f118d4a6e68f8ce346f3824d9f624c7a04 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Tue, 10 Mar 2020 18:59:45 +0800 Subject: [PATCH 008/401] Adapting partial code(file name start with F) to the sonar cloud rule (#2045) * Adapting partial code(file name start with F) to the sonar cloud rule * add more unit test * add License * add includes configuration to maven-surefire-plugin * fix getResourceFilesList incorrect logic --- .../alert/utils/FuncUtils.java | 2 +- .../alert/utils/FuncUtilsTest.java | 2 +- .../common/task/flink/FlinkParameters.java | 14 +++-- .../common/utils/FileUtils.java | 6 +- .../common/task/FlinkParametersTest.java | 55 +++++++++++++++++++ .../common/utils/FileUtilsTest.java | 17 +++--- .../server/utils/FlinkArgsUtils.java | 16 ++---- .../worker/runner/TaskScheduleThread.java | 4 +- .../server/utils/FlinkArgsUtilsTest.java | 24 ++++---- pom.xml | 1 + 10 files changed, 97 insertions(+), 44 deletions(-) create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java index dd6ca4b8a6..d68532a82b 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java @@ -20,7 +20,7 @@ import org.apache.dolphinscheduler.common.utils.StringUtils; public class FuncUtils { - static public String mkString(Iterable list, String split) { + public static String mkString(Iterable list, String split) { if (null == list || StringUtils.isEmpty(split)){ return null; diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java index e2b25d961d..a4aeea9c0c 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java @@ -46,7 +46,7 @@ public class FuncUtilsTest { logger.info(result); //Expected result string - assertEquals(result, "user1|user2|user3"); + assertEquals("user1|user2|user3", result); //Null list expected return null result = FuncUtils.mkString(null, split); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java index 0638b3858e..1fbd9ab354 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -207,12 +208,15 @@ public class FlinkParameters extends AbstractParameters { @Override public List getResourceFilesList() { - if(resourceList !=null ) { - this.resourceList.add(mainJar); - return resourceList.stream() - .map(p -> p.getRes()).collect(Collectors.toList()); + if(resourceList != null ) { + List resourceFiles = resourceList.stream() + .map(ResourceInfo::getRes).collect(Collectors.toList()); + if(mainJar != null) { + resourceFiles.add(mainJar.getRes()); + } + return resourceFiles; } - return null; + return Collections.emptyList(); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java index c84848fbae..dc60b04c59 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java @@ -44,7 +44,7 @@ public class FileUtils { String fileSuffix = ""; if (StringUtils.isNotEmpty(filename)) { - int lastIndex = filename.lastIndexOf("."); + int lastIndex = filename.lastIndexOf('.'); if (lastIndex > 0) { fileSuffix = filename.substring(lastIndex + 1); } @@ -325,10 +325,8 @@ public class FileUtils { } } else { File parent = file.getParentFile(); - if (parent != null) { - if (!parent.mkdirs() && !parent.isDirectory()) { + if (parent != null && !parent.mkdirs() && !parent.isDirectory()) { throw new IOException("Directory '" + parent + "' could not be created"); - } } } return new FileOutputStream(file, append); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java new file mode 100644 index 0000000000..7ce00e875a --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.task; + +import org.apache.dolphinscheduler.common.process.ResourceInfo; +import org.apache.dolphinscheduler.common.task.flink.FlinkParameters; +import org.junit.Assert; +import org.junit.Test; + +import java.util.LinkedList; +import java.util.List; + +public class FlinkParametersTest { + @Test + public void getResourceFilesList() { + FlinkParameters flinkParameters = new FlinkParameters(); + Assert.assertNotNull(flinkParameters.getResourceFilesList()); + Assert.assertTrue(flinkParameters.getResourceFilesList().isEmpty()); + + ResourceInfo mainResource = new ResourceInfo(); + mainResource.setRes("testFlinkMain-1.0.0-SNAPSHOT.jar"); + flinkParameters.setMainJar(mainResource); + + List resourceInfos = new LinkedList<>(); + ResourceInfo resourceInfo1 = new ResourceInfo(); + resourceInfo1.setRes("testFlinkParameters1.jar"); + resourceInfos.add(resourceInfo1); + + flinkParameters.setResourceList(resourceInfos); + Assert.assertNotNull(flinkParameters.getResourceFilesList()); + Assert.assertEquals(2, flinkParameters.getResourceFilesList().size()); + + ResourceInfo resourceInfo2 = new ResourceInfo(); + resourceInfo2.setRes("testFlinkParameters2.jar"); + resourceInfos.add(resourceInfo2); + + flinkParameters.setResourceList(resourceInfos); + Assert.assertNotNull(flinkParameters.getResourceFilesList()); + Assert.assertEquals(3, flinkParameters.getResourceFilesList().size()); + } +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java index 89458f6f1c..96217842bf 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java @@ -30,29 +30,32 @@ public class FileUtilsTest { @Test public void suffix() { - Assert.assertEquals(FileUtils.suffix("ninfor.java"),"java"); + Assert.assertEquals("java", FileUtils.suffix("ninfor.java")); + Assert.assertEquals("", FileUtils.suffix(null)); + Assert.assertEquals("", FileUtils.suffix("")); + Assert.assertEquals("", FileUtils.suffix("ninfor-java")); } @Test public void testGetDownloadFilename() { PowerMockito.mockStatic(DateUtils.class); PowerMockito.when(DateUtils.getCurrentTime(YYYYMMDDHHMMSS)).thenReturn("20190101101059"); - Assert.assertEquals(FileUtils.getDownloadFilename("test"), - "/tmp/dolphinscheduler/download/20190101101059/test"); + Assert.assertEquals("/tmp/dolphinscheduler/download/20190101101059/test", + FileUtils.getDownloadFilename("test")); } @Test public void testGetUploadFilename() { - Assert.assertEquals(FileUtils.getUploadFilename("aaa","bbb"), - "/tmp/dolphinscheduler/aaa/resources/bbb"); + Assert.assertEquals("/tmp/dolphinscheduler/aaa/resources/bbb", + FileUtils.getUploadFilename("aaa","bbb")); } @Test public void testGetProcessExecDir() { String dir = FileUtils.getProcessExecDir(1,2,3, 4); - Assert.assertEquals(dir, "/tmp/dolphinscheduler/exec/process/1/2/3/4"); + Assert.assertEquals("/tmp/dolphinscheduler/exec/process/1/2/3/4", dir); dir = FileUtils.getProcessExecDir(1,2,3); - Assert.assertEquals(dir, "/tmp/dolphinscheduler/exec/process/1/2/3"); + Assert.assertEquals("/tmp/dolphinscheduler/exec/process/1/2/3", dir); } @Test diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java index 4c33ef8db2..12c7eb2d56 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java @@ -17,12 +17,11 @@ package org.apache.dolphinscheduler.server.utils; +import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.flink.FlinkParameters; -import org.apache.commons.lang.StringUtils; -import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; @@ -32,12 +31,7 @@ import java.util.List; * spark args utils */ public class FlinkArgsUtils { - - /** - * logger of FlinkArgsUtils - */ - private static final org.slf4j.Logger logger = LoggerFactory.getLogger(FlinkArgsUtils.class); - + private static final String LOCAL_DEPLOY_MODE = "local"; /** * build args * @param param flink parameters @@ -52,7 +46,7 @@ public class FlinkArgsUtils { deployMode = tmpDeployMode; } - if (!"local".equals(deployMode)) { + if (!LOCAL_DEPLOY_MODE.equals(deployMode)) { args.add(Constants.FLINK_RUN_MODE); //-m args.add(Constants.FLINK_YARN_CLUSTER); //yarn-cluster @@ -113,12 +107,12 @@ public class FlinkArgsUtils { String queue = param.getQueue(); if (StringUtils.isNotEmpty(others)) { - if (!others.contains(Constants.FLINK_QUEUE) && StringUtils.isNotEmpty(queue) && !deployMode.equals("local")) { + if (!others.contains(Constants.FLINK_QUEUE) && StringUtils.isNotEmpty(queue) && !deployMode.equals(LOCAL_DEPLOY_MODE)) { args.add(Constants.FLINK_QUEUE); args.add(param.getQueue()); } args.add(others); - } else if (StringUtils.isNotEmpty(queue) && !deployMode.equals("local")) { + } else if (StringUtils.isNotEmpty(queue) && !deployMode.equals(LOCAL_DEPLOY_MODE)) { args.add(Constants.FLINK_QUEUE); args.add(param.getQueue()); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java index 5e68acf94e..21ee1dfa40 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java @@ -296,9 +296,7 @@ public class TaskScheduleThread implements Runnable { if (baseParam != null) { List projectResourceFiles = baseParam.getResourceFilesList(); - if (projectResourceFiles != null) { - projectFiles.addAll(projectResourceFiles); - } + projectFiles.addAll(projectResourceFiles); } return new ArrayList<>(projectFiles); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java index 710d2c2505..2e4861e2a2 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java @@ -87,35 +87,35 @@ public class FlinkArgsUtilsTest { } //Expected values and order - assertEquals(result.size(),20); + assertEquals(20, result.size()); - assertEquals(result.get(0),"-m"); - assertEquals(result.get(1),"yarn-cluster"); + assertEquals("-m", result.get(0)); + assertEquals("yarn-cluster", result.get(1)); - assertEquals(result.get(2),"-ys"); + assertEquals("-ys", result.get(2)); assertSame(Integer.valueOf(result.get(3)),slot); - assertEquals(result.get(4),"-ynm"); + assertEquals("-ynm",result.get(4)); assertEquals(result.get(5),appName); - assertEquals(result.get(6),"-yn"); + assertEquals("-yn", result.get(6)); assertSame(Integer.valueOf(result.get(7)),taskManager); - assertEquals(result.get(8),"-yjm"); + assertEquals("-yjm", result.get(8)); assertEquals(result.get(9),jobManagerMemory); - assertEquals(result.get(10),"-ytm"); + assertEquals("-ytm", result.get(10)); assertEquals(result.get(11),taskManagerMemory); - assertEquals(result.get(12),"-d"); + assertEquals("-d", result.get(12)); - assertEquals(result.get(13),"-c"); + assertEquals("-c", result.get(13)); assertEquals(result.get(14),mainClass); assertEquals(result.get(15),mainJar.getRes()); assertEquals(result.get(16),mainArgs); - assertEquals(result.get(17),"--qu"); + assertEquals("--qu", result.get(17)); assertEquals(result.get(18),queue); assertEquals(result.get(19),others); @@ -125,7 +125,7 @@ public class FlinkArgsUtilsTest { param1.setQueue(queue); param1.setDeployMode(mode); result = FlinkArgsUtils.buildArgs(param1); - assertEquals(result.size(),5); + assertEquals(5, result.size()); } } \ No newline at end of file diff --git a/pom.xml b/pom.xml index dee1dce8b2..0a01dadadf 100644 --- a/pom.xml +++ b/pom.xml @@ -688,6 +688,7 @@ **/common/threadutils/*.java **/common/graph/*.java **/common/queue/*.java + **/common/task/FlinkParametersTest.java **/common/task/SqoopParameterEntityTest.java **/api/utils/CheckUtilsTest.java **/api/utils/FileUtilsTest.java From c1484386a84094345f0c6a21136947c6ec3c2603 Mon Sep 17 00:00:00 2001 From: AhahaGe Date: Tue, 10 Mar 2020 22:00:40 +0800 Subject: [PATCH 009/401] add .asf.yaml file (#2136) Co-authored-by: guirong.ggr --- .asf.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .asf.yaml diff --git a/.asf.yaml b/.asf.yaml new file mode 100644 index 0000000000..fa71a46552 --- /dev/null +++ b/.asf.yaml @@ -0,0 +1,10 @@ +staging: + profile: ~ + whoami: dev + foo: trigger + +publish: + whoami: dev + +github: + description: “Dolphin Scheduler is a distributed and easy-to-extend visual DAG workflow scheduling system, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.(分布式易扩展的可视化工作流任务调度)” \ No newline at end of file From 0ea9a281b2cd0b246a490361432536b7bdcb8cd7 Mon Sep 17 00:00:00 2001 From: AhahaGe Date: Tue, 10 Mar 2020 22:28:40 +0800 Subject: [PATCH 010/401] edit README.md modify word from expand to extend (#2138) Co-authored-by: guirong.ggr --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0a9a164b85..ebd620efee 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Dolphin Scheduler Official Website ### Design features: -A distributed and easy-to-expand visual DAG workflow scheduling system. Dedicated to solving the complex dependencies in data processing, making the scheduling system `out of the box` for data processing. +A distributed and easy-to-extend visual DAG workflow scheduling system. Dedicated to solving the complex dependencies in data processing, making the scheduling system `out of the box` for data processing. Its main objectives are as follows: - Associate the Tasks according to the dependencies of the tasks in a DAG graph, which can visualize the running state of task in real time. From ec93e70f2f8168390da64f77d3c3737cdfe877bf Mon Sep 17 00:00:00 2001 From: Jave-Chen Date: Thu, 12 Mar 2020 10:09:13 +0800 Subject: [PATCH 011/401] fix wrong sonar analysis with checkout V2. (#2148) --- .github/workflows/ci_ut.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 48cd1e16ce..12f7c04ed6 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -15,7 +15,7 @@ # limitations under the License. # -on: ["pull_request"] +on: ["pull_request", "push"] env: DOCKER_DIR: ./docker LOG_DIR: /tmp/dolphinscheduler @@ -52,7 +52,15 @@ jobs: run: | export MAVEN_OPTS='-Dmaven.repo.local=.m2/repository -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:-UseGCOverheadLimit -Xmx3g' mvn test -B -Dmaven.test.skip=false + - name: Upload coverage report to codecov + if: github.event_name == 'pull_request' + run: | CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash) + - name: Git fetch unshallow + run: | + git fetch --unshallow + git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" + git fetch origin - name: Run SonarCloud Analysis run: > mvn verify --batch-mode From da3f25dd8be019ea7ef5aa73f3e4bc079c841a4f Mon Sep 17 00:00:00 2001 From: liwenhe1993 <32166572+liwenhe1993@users.noreply.github.com> Date: Thu, 12 Mar 2020 10:11:07 +0800 Subject: [PATCH 012/401] docker run failed when docker command was `docker run -d` (#2125) * docker run failed when docker command was `docker run -d` 1. add tini 2. replace `tee` command by `tail` command in startup.sh 3. api-server need to support zookeeper * modify `check.sh` in dockerfile * add init directories and log files --- dockerfile/Dockerfile | 6 +++--- dockerfile/hooks/check | 2 +- dockerfile/startup.sh | 8 ++++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/dockerfile/Dockerfile b/dockerfile/Dockerfile index 8fa4886f31..acd2cb99e5 100644 --- a/dockerfile/Dockerfile +++ b/dockerfile/Dockerfile @@ -23,11 +23,11 @@ ENV TZ Asia/Shanghai ENV LANG C.UTF-8 ENV DEBIAN_FRONTEND noninteractive -#1. install dos2unix shadow bash openrc python sudo vim wget iputils net-tools ssh pip kazoo. +#1. install dos2unix shadow bash openrc python sudo vim wget iputils net-tools ssh pip tini kazoo. #If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example: #RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories RUN apk update && \ - apk add dos2unix shadow bash openrc python sudo vim wget iputils net-tools openssh-server py2-pip && \ + apk add dos2unix shadow bash openrc python sudo vim wget iputils net-tools openssh-server py2-pip tini && \ apk add --update procps && \ openrc boot && \ pip install kazoo @@ -92,4 +92,4 @@ RUN rm -rf /var/cache/apk/* #9. expose port EXPOSE 2181 2888 3888 5432 12345 50051 8888 -ENTRYPOINT ["/root/startup.sh"] \ No newline at end of file +ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"] \ No newline at end of file diff --git a/dockerfile/hooks/check b/dockerfile/hooks/check index fbc3f8bd44..21f3f4db12 100644 --- a/dockerfile/hooks/check +++ b/dockerfile/hooks/check @@ -17,7 +17,7 @@ # echo "------ dolphinscheduler check - server - status -------" sleep 20 -server_num=$(docker top `docker container list | grep startup | awk '{print $1}'`| grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) +server_num=$(docker top `docker container list | grep '/sbin/tini' | awk '{print $1}'`| grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) if [ $server_num -eq 5 ] then echo "Server all start successfully" diff --git a/dockerfile/startup.sh b/dockerfile/startup.sh index 610a86f5d3..30439c2321 100644 --- a/dockerfile/startup.sh +++ b/dockerfile/startup.sh @@ -164,6 +164,7 @@ case "$1" in LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log ;; (api-server) + initZK initPostgreSQL initApiServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log @@ -187,6 +188,9 @@ case "$1" in ;; esac -echo "tee begin" -exec tee ${LOGFILE} +# init directories and log files +mkdir -p ${DOLPHINSCHEDULER_LOGS} && mkdir -p /var/log/nginx/ && cat /dev/null >> ${LOGFILE} + +echo "tail begin" +exec bash -c "tail -n 1 -f ${LOGFILE}" From 6b6584ac8ea6f3841e7f9a87bdbdc32859f6f3c8 Mon Sep 17 00:00:00 2001 From: tswstarplanet Date: Thu, 12 Mar 2020 17:38:38 +0800 Subject: [PATCH 013/401] use logger to print exception (#2151) --- .../org/apache/dolphinscheduler/common/shell/AbstractShell.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java index 101da18b2c..f846b19741 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java @@ -335,7 +335,7 @@ public abstract class AbstractShell { try{ entry.getValue().destroy(); } catch (Exception e) { - e.printStackTrace(); + logger.error("Destroy All Processes error", e); } } From 89c0a796f9d57a9dc2b666fbb5546c03dd110cdd Mon Sep 17 00:00:00 2001 From: tswstarplanet Date: Thu, 12 Mar 2020 17:39:25 +0800 Subject: [PATCH 014/401] fix field access modifier (#2150) --- .../org/apache/dolphinscheduler/api/utils/ZooKeeperState.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java index 5aa6be858a..a33754397d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java @@ -121,7 +121,7 @@ public class ZooKeeperState { private class SendThread extends Thread { private String cmd; - public String ret = ""; + private String ret = ""; public SendThread(String cmd) { this.cmd = cmd; From b827c45c14eb3dbd378cf4ae2fef7787e5e81828 Mon Sep 17 00:00:00 2001 From: tswstarplanet Date: Thu, 12 Mar 2020 17:41:10 +0800 Subject: [PATCH 015/401] fix unequal number of arguments and placeholder (#2149) --- .../java/org/apache/dolphinscheduler/alert/utils/MailUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java index 99efdc8a6a..130ad8ade6 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java @@ -334,7 +334,7 @@ public class MailUtils { * @param e the exception */ private static void handleException(Collection receivers, Map retMap, Exception e) { - logger.error("Send email to {} failed {}", receivers, e); + logger.error("Send email to {} failed", receivers, e); retMap.put(Constants.MESSAGE, "Send email to {" + String.join(",", receivers) + "} failed，" + e.toString()); } From deee1a837bfba70ef142fb2952d5f8ab79767274 Mon Sep 17 00:00:00 2001 From: t1mon <178317391@qq.com> Date: Thu, 12 Mar 2020 17:50:37 +0800 Subject: [PATCH 016/401] Add mail server configuration description information (#2143) * Add mail server configuration description information * Remove mail server configuration note in alert.properties --- dolphinscheduler-alert/src/main/resources/alert.properties | 1 - install.sh | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-alert/src/main/resources/alert.properties b/dolphinscheduler-alert/src/main/resources/alert.properties index 000d0653b7..9f5acea188 100644 --- a/dolphinscheduler-alert/src/main/resources/alert.properties +++ b/dolphinscheduler-alert/src/main/resources/alert.properties @@ -28,7 +28,6 @@ mail.server.port=25 mail.sender=xxx@xxx.com mail.user=xxx@xxx.com mail.passwd=111111 - # TLS mail.smtp.starttls.enable=true # SSL diff --git a/install.sh b/install.sh index 8735543c47..ed66ce5bd9 100644 --- a/install.sh +++ b/install.sh @@ -109,6 +109,7 @@ mailProtocol="SMTP" mailServerHost="smtp.exmail.qq.com" # mail server port +# note: Different protocols and encryption methods correspond to different ports, when SSL/TLS is enabled, make sure the port is correct. mailServerPort="25" # sender @@ -118,6 +119,7 @@ mailSender="xxxxxxxxxx" mailUser="xxxxxxxxxx" # sender password +# note: The mail.passwd is email service authorization code, not the email login password. mailPassword="xxxxxxxxxx" # TLS mail protocol support From f1087ff3a11b3876b6599076cdcd7d13f09cf5c5 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Thu, 12 Mar 2020 17:59:16 +0800 Subject: [PATCH 017/401] Limit customization file content to no more than 3000 lines(#2128) (#2141) * Change DOM label * Change name to lowercase * Limit customization file content to no more than 3000 lines --- .../pages/file/pages/create/index.vue | 5 ++- .../resource/pages/file/pages/edit/index.vue | 38 ++++++++++++------- .../src/js/module/i18n/locale/en_US.js | 1 + .../src/js/module/i18n/locale/zh_CN.js | 1 + 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue index caa31ed4d8..df84f0f292 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue @@ -132,7 +132,10 @@ this.$message.warning(`${i18n.$t('Please enter the resource content')}`) return false } - + if (editor.doc.size>3000) { + this.$message.warning(`${i18n.$t('Resource content cannot exceed 3000 lines')}`) + return false + } return true }, /** diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue index a9aee1c9fd..a0d1d7d187 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue @@ -45,6 +45,7 @@ @@ -257,4 +272,4 @@ } } } - \ No newline at end of file + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sqoop.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sqoop.vue index eead745a06..6594ffcf74 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sqoop.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sqoop.vue @@ -65,7 +65,7 @@ @@ -186,8 +186,8 @@
@@ -385,7 +385,7 @@
@@ -556,7 +556,8 @@ targetType:"HDFS", sourceMysqlParams:{ - srcDatasource:-1, + srcType:"MYSQL", + srcDatasource:"", srcTable:"", srcQueryType:"1", srcQuerySql:'', @@ -588,7 +589,8 @@ }, targetMysqlParams:{ - targetDatasource:-1, + targetType:"MYSQL", + targetDatasource:"", targetTable:"", targetColumns:"", fieldsTerminated:"", @@ -680,6 +682,7 @@ * return data source */ _onSourceDsData (o) { + this.sourceMysqlParams.srcType = o.type this.sourceMysqlParams.srcDatasource = o.datasource }, @@ -687,6 +690,7 @@ * return data source */ _onTargetDsData (o) { + this.targetMysqlParams.targetType = o.type this.targetMysqlParams.targetDatasource = o.datasource }, @@ -697,7 +701,7 @@ var params = null switch(this.sourceType){ case "MYSQL": - this.sourceMysqlParams.srcQuerySql = editor.getValue() + this.sourceMysqlParams.srcQuerySql = editor ? editor.getValue() : this.sourceMysqlParams.srcQuerySql params = JSON.stringify(this.sourceMysqlParams) break; case "ORACLE": @@ -879,7 +883,9 @@ * Processing code highlighting */ _handlerEditor () { - editor = codemirror('code-sql-mirror', { + this._destroyEditor() + + editor = codemirror('code-sqoop-mirror', { mode: 'sql', readOnly: this.isDetails }) @@ -892,9 +898,15 @@ } } + this.changes = () => { + this._cacheParams() + } + // Monitor keyboard editor.on('keypress', this.keypress) + editor.on('changes', this.changes) + editor.setValue(this.sourceMysqlParams.srcQuerySql) return editor @@ -906,6 +918,27 @@ _onLocalParams (a) { this.localParams = a }, + + _cacheParams () { + this.$emit('on-cache-params', { + concurrency:this.concurrency, + modelType:this.modelType, + sourceType:this.sourceType, + targetType:this.targetType, + sourceParams:this._handleSourceParams(), + targetParams:this._handleTargetParams(), + localParams:this.localParams + }); + }, + + _destroyEditor () { + if (editor) { + editor.toTextArea() // Uninstall + editor.off($('.code-sqoop-mirror'), 'keypress', this.keypress) + editor.off($('.code-sqoop-mirror'), 'changes', this.changes) + editor = null + } + }, }, watch: { // Listening to sqlType @@ -927,11 +960,12 @@ }, //Watch the cacheParams cacheParams (val) { - this.$emit('on-cache-params', val); + this._cacheParams() } }, created () { + this._destroyEditor() let o = this.backfillItem // Non-null objects represent backfill @@ -963,11 +997,28 @@ */ if (editor) { editor.toTextArea() // Uninstall - editor.off($('.code-sql-mirror'), 'keypress', this.keypress) + editor.off($('.code-sqoop-mirror'), 'keypress', this.keypress) + editor.off($('.code-sqoop-mirror'), 'changes', this.changes) + editor = null } }, computed: { + cacheParams () { + return { + concurrency:this.concurrency, + modelType:this.modelType, + sourceType:this.sourceType, + targetType:this.targetType, + localParams:this.localParams, + sourceMysqlParams:this.sourceMysqlParams, + sourceHdfsParams:this.sourceHdfsParams, + sourceHiveParams:this.sourceHiveParams, + targetHdfsParams:this.targetHdfsParams, + targetMysqlParams:this.targetMysqlParams, + targetHiveParams:this.targetHiveParams + } + } }, components: { mListBox, mDatasource, mLocalParams} } From e0f4e22cf436e74df4a72031d00297d193967d77 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Tue, 3 Mar 2020 14:11:05 +0800 Subject: [PATCH 021/401] Change DOM label --- .../conf/home/pages/dag/_source/formModel/tasks/conditions.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue index fb3f2c295c..639b2f4ad7 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue @@ -19,7 +19,7 @@
{{$t('Custom Parameters')}}
-
+
From d7614c059b68f6ab8cd358444a3488ab130e7b34 Mon Sep 17 00:00:00 2001 From: Yelli Date: Thu, 12 Mar 2020 21:52:48 +0800 Subject: [PATCH 022/401] fix bug: data too long for column 'app_link' (#2146) * fix bug: data to long for app_link field * change app_link field length in 1.2.2 ddl * change app_link field type to text --- .../mysql/dolphinscheduler_ddl.sql | 22 ++++++++++++++++++- .../postgresql/dolphinscheduler_ddl.sql | 18 +++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql b/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql index e5785d9997..049484ce3a 100644 --- a/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.2.2_schema/mysql/dolphinscheduler_ddl.sql @@ -54,4 +54,24 @@ d// delimiter ; CALL uc_dolphin_T_t_ds_task_instance_A_executor_id; -DROP PROCEDURE uc_dolphin_T_t_ds_task_instance_A_executor_id; \ No newline at end of file +DROP PROCEDURE uc_dolphin_T_t_ds_task_instance_A_executor_id; + +-- uc_dolphin_T_t_ds_task_instance_C_app_link +drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_task_instance_C_app_link; +delimiter d// +CREATE PROCEDURE uc_dolphin_T_t_ds_task_instance_C_app_link() + BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_task_instance' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='app_link') + THEN + ALTER TABLE t_ds_task_instance CHANGE COLUMN app_link app_link text COMMENT 'yarn app id'; + END IF; + END; + +d// + +delimiter ; +CALL uc_dolphin_T_t_ds_task_instance_C_app_link; +DROP PROCEDURE uc_dolphin_T_t_ds_task_instance_C_app_link; \ No newline at end of file diff --git a/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql b/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql index 70471cb26d..b1e0fd941c 100644 --- a/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.2.2_schema/postgresql/dolphinscheduler_ddl.sql @@ -49,3 +49,21 @@ d// delimiter ; SELECT uc_dolphin_T_t_ds_task_instance_A_executor_id(); DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_task_instance_A_executor_id(); + +-- uc_dolphin_T_t_ds_task_instance_C_app_link +delimiter d// +CREATE OR REPLACE FUNCTION uc_dolphin_T_t_ds_task_instance_C_app_link() RETURNS void AS $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_task_instance' + AND COLUMN_NAME ='app_link') + THEN + ALTER TABLE t_ds_task_instance ALTER COLUMN app_link type text; + END IF; +END; +$$ LANGUAGE plpgsql; +d// + +delimiter ; +SELECT uc_dolphin_T_t_ds_task_instance_C_app_link(); +DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_task_instance_C_app_link(); \ No newline at end of file From 956cdf35343125f57e88a96646f48f7dff4ef09f Mon Sep 17 00:00:00 2001 From: dailidong Date: Fri, 13 Mar 2020 11:13:17 +0800 Subject: [PATCH 023/401] Delete .asf.yaml (#2158) --- .asf.yaml | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 .asf.yaml diff --git a/.asf.yaml b/.asf.yaml deleted file mode 100644 index fa71a46552..0000000000 --- a/.asf.yaml +++ /dev/null @@ -1,10 +0,0 @@ -staging: - profile: ~ - whoami: dev - foo: trigger - -publish: - whoami: dev - -github: - description: “Dolphin Scheduler is a distributed and easy-to-extend visual DAG workflow scheduling system, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.(分布式易扩展的可视化工作流任务调度)” \ No newline at end of file From 33e86a97bf0791b88a2f71fad2de7d54d374d9f6 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Fri, 13 Mar 2020 13:02:51 +0800 Subject: [PATCH 024/401] Adapting partial code(file name start with I) to the sonar cloud rule (#2155) --- .../common/utils/IOUtils.java | 19 +++++-------------- .../common/utils/IpUtils.java | 5 ----- .../common/utils/IpUtilsTest.java | 6 +++--- 3 files changed, 8 insertions(+), 22 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java index 73df158aa3..ce551d8405 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java @@ -19,26 +19,17 @@ package org.apache.dolphinscheduler.common.utils; +import java.io.Closeable; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; public class IOUtils { - public static void closeQuietly(InputStream fis){ - if(fis != null){ + public static void closeQuietly(Closeable closeable){ + if(closeable != null){ try { - fis.close(); - } catch (IOException ignore) { - } - } - } - - public static void closeQuietly(InputStreamReader reader){ - if(reader != null){ - try { - reader.close(); + closeable.close(); } catch (IOException ignore) { + // nothing need to do } } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java index e7e0b34bdd..3b068c60d2 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java @@ -17,16 +17,11 @@ package org.apache.dolphinscheduler.common.utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - /** * http utils */ public class IpUtils { - private static final Logger logger = LoggerFactory.getLogger(IpUtils.class); public static final String DOT = "."; /** diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/IpUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/IpUtilsTest.java index e65bcd219b..ec6ffa35a7 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/IpUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/IpUtilsTest.java @@ -29,13 +29,13 @@ public class IpUtilsTest { long longNumber = IpUtils.ipToLong(ip); long longNumber2 = IpUtils.ipToLong(ip2); System.out.println(longNumber); - Assert.assertEquals(longNumber, 3232263681L); - Assert.assertEquals(longNumber2, 0L); + Assert.assertEquals(3232263681L, longNumber); + Assert.assertEquals(0L, longNumber2); String ip3 = "255.255.255.255"; long longNumber3 = IpUtils.ipToLong(ip3); System.out.println(longNumber3); - Assert.assertEquals(longNumber3, 4294967295L); + Assert.assertEquals(4294967295L, longNumber3); } From eebf5fa796b8d019af4b02af1a99d9fc3d714c9f Mon Sep 17 00:00:00 2001 From: gaojun2048 <32193458+gaojun2048@users.noreply.github.com> Date: Fri, 13 Mar 2020 13:04:33 +0800 Subject: [PATCH 025/401] fix rpmbuild and DS Ambari plugin bug (#2144) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update quick-start.md * 更新demo的地址 * add rpm plugin * delete the git ignore files * 测试git ignore * 测试ignore * optimize the .gitignore file * add rpm package, delete unuse copy jar action in assembly * delete docs dir * add rpm plugin support * add rpm plugin support * add rpm plugin support * dolphinscheduler-common have not bin dir * dolphinscheduler-common have not bin dir * delete unuse config about assembly * add defineSatement * add rpm plugin support * add rpm plugin support * add rpm plugin support * add rpm support * update install dirname , make front and backend together * update rpm name * update rpm name * update rpm config * add jars excludes * add jars excludes * add rpm plugion * add rpm plugion * add rpm plugion * add auto create /opt/soft * The install dir can not be fixed to apache-dolphinscheduler-incubating-1.2.1 , its version is 1.2.2 * install from rpm will create a link * the ui install thrown rpm only support jetty not nginx * the ui install thrown rpm only support jetty not nginx * sql目录里加上soft_version文件 * env dir need in conf dir * remove no use module * add module * updaate * rpmbuild add dolphinscheduler-service module --- .../DOLPHIN/1.2.1/package/scripts/params.py | 2 +- dolphinscheduler-dist/pom.xml | 53 +++++++++--------- dolphinscheduler-ui/pom.xml | 55 +++++++++++++++++++ pom.xml | 2 +- 4 files changed, 83 insertions(+), 29 deletions(-) diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py b/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py index 230ad14565..3780f6c27e 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py +++ b/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py @@ -30,7 +30,7 @@ sys.setdefaultencoding('utf-8') config = Script.get_config() # conf_dir = "/etc/" -dolphin_home = "/opt/soft/apache-dolphinscheduler-incubating-1.2.1" +dolphin_home = "/opt/soft/dolphinscheduler" dolphin_conf_dir = dolphin_home + "/conf" dolphin_log_dir = dolphin_home + "/logs" dolphin_bin_dir = dolphin_home + "/bin" diff --git a/dolphinscheduler-dist/pom.xml b/dolphinscheduler-dist/pom.xml index 80ddfec9b0..0ead2b9880 100644 --- a/dolphinscheduler-dist/pom.xml +++ b/dolphinscheduler-dist/pom.xml @@ -220,10 +220,7 @@ ${basedir}/../dolphinscheduler-alert/src/main/resources - **/*.properties - **/*.xml - **/*.json - **/*.ftl + **/*.* @@ -232,9 +229,7 @@ ${basedir}/../dolphinscheduler-common/src/main/resources - **/*.properties - **/*.xml - **/*.json + **/*.* @@ -243,10 +238,7 @@ ${basedir}/../dolphinscheduler-dao/src/main/resources - **/*.properties - **/*.xml - **/*.json - **/*.yml + **/*.* @@ -255,9 +247,7 @@ ${basedir}/../dolphinscheduler-api/src/main/resources - **/*.properties - **/*.xml - **/*.json + **/*.* @@ -266,13 +256,19 @@ ${basedir}/../dolphinscheduler-server/src/main/resources - **/*.properties - **/*.xml - **/*.json config/*.* + + + ${basedir}/../dolphinscheduler-service/src/main/resources + + + *.* + + + ${basedir}/../script @@ -342,14 +338,6 @@ - - - ${basedir}/../dolphinscheduler-ui - - - install-dolphinscheduler-ui.sh - - ${basedir}/release-docs @@ -362,7 +350,7 @@ - /opt/soft/${project.build.finalName}/dist + /opt/soft/${project.build.finalName}/ui 755 root root @@ -391,6 +379,14 @@ **/*.* + + + ${basedir}/../sql + + + soft_version + + @@ -405,7 +401,7 @@ ${basedir}/../script - **/*.* + *.sh @@ -416,6 +412,9 @@ + + + diff --git a/dolphinscheduler-ui/pom.xml b/dolphinscheduler-ui/pom.xml index 3fd9aa6650..78869ffbc4 100644 --- a/dolphinscheduler-ui/pom.xml +++ b/dolphinscheduler-ui/pom.xml @@ -89,6 +89,61 @@ + + rpmbuild + + + + com.github.eirslett + frontend-maven-plugin + ${frontend-maven-plugin.version} + + + install node and npm + + install-node-and-npm + + + ${node.version} + ${npm.version} + + + + npm install node-sass --unsafe-perm + + npm + + generate-resources + + install node-sass --unsafe-perm + + + + npm install + + npm + + generate-resources + + install + + + + npm run build:release + + npm + + + run build:release + + + + + + + + + nginx diff --git a/pom.xml b/pom.xml index 0a01dadadf..6a84bcc07b 100644 --- a/pom.xml +++ b/pom.xml @@ -881,4 +881,4 @@ dolphinscheduler-service - + \ No newline at end of file From 2939b193b878ee12387323d246e7a73c78b2d116 Mon Sep 17 00:00:00 2001 From: liwenhe1993 <32166572+liwenhe1993@users.noreply.github.com> Date: Fri, 13 Mar 2020 23:38:38 +0800 Subject: [PATCH 026/401] Toolbar of the DAG has not prompt (#2161) * Toolbar of the DAG has not prompt * Remove duplication code --- dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue | 2 +- dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js | 1 - dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue index 7d6f95d753..a1ccd39260 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue @@ -25,7 +25,7 @@ :key="v" v-for="(item,v) in tasksTypeList" @mousedown="_getDagId(v)"> -
+
diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js index 4507250aba..64ab298437 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js @@ -567,7 +567,6 @@ export default { 'Data Target': 'Data Target', 'All Columns': 'All Columns', 'Some Columns': 'Some Columns', - 'Modify User': 'Modify User', 'Branch flow': 'Branch flow', 'Cannot select the same node for successful branch flow and failed branch flow': 'Cannot select the same node for successful branch flow and failed branch flow' } diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js index 132b4d8745..3363eea198 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -567,7 +567,6 @@ export default { 'Data Target': '数据目的', 'All Columns': '全表导入', 'Some Columns': '选择列', - 'Modify User': '修改用户', 'Branch flow': '分支流转', 'Cannot select the same node for successful branch flow and failed branch flow': '成功分支流转和失败分支流转不能选择同一个节点' } From 610c95a470e147b69a33ceefe9593833d901fe7c Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Fri, 13 Mar 2020 23:53:22 +0800 Subject: [PATCH 027/401] use static method in static class JSON (#2156) --- .../alert/utils/JSONUtils.java | 6 ++--- .../api/service/DataSourceService.java | 25 ++++++++++--------- .../controller/ResourcesControllerTest.java | 23 +++++++++-------- .../common/model/TaskNode.java | 4 +-- .../common/utils/JSONUtils.java | 15 +++++------ .../common/utils/ParameterUtils.java | 4 +-- .../common/utils/JSONUtilsTest.java | 4 +-- .../common/utils/ParameterUtilsTest.java | 8 +++--- .../dao/entity/ProcessDefinition.java | 8 +++--- .../master/runner/MasterExecThread.java | 4 +-- .../master/runner/MasterTaskExecThread.java | 4 +-- .../worker/runner/TaskScheduleThread.java | 6 ++--- .../task/processdure/ProcedureTask.java | 4 +-- .../server/worker/task/sql/SqlTask.java | 9 ++++--- .../server/master/MasterExecThreadTest.java | 4 +-- .../shell/ShellCommandExecutorTest.java | 4 +-- .../server/worker/sql/SqlExecutorTest.java | 4 +-- .../worker/task/sqoop/SqoopTaskTest.java | 10 ++++---- .../service/process/ProcessService.java | 7 +++--- 19 files changed, 79 insertions(+), 74 deletions(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java index 1cd9f490b2..8d66255fc4 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java @@ -16,8 +16,8 @@ */ package org.apache.dolphinscheduler.alert.utils; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; -import com.alibaba.fastjson.JSONObject; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,7 +38,7 @@ public class JSONUtils { */ public static String toJsonString(Object object) { try{ - return JSONObject.toJSONString(object,false); + return JSON.toJSONString(object,false); } catch (Exception e) { throw new RuntimeException("Json deserialization exception.", e); } @@ -57,7 +57,7 @@ public class JSONUtils { return null; } try { - return JSONArray.parseArray(json, clazz); + return JSON.parseArray(json, clazz); } catch (Exception e) { logger.error("JSONArray.parseArray exception!",e); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java index f6d8903dd8..6a732fed0e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.api.service; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; @@ -303,7 +304,7 @@ public class DataSourceService extends BaseService{ for (DataSource dataSource : dataSourceList) { String connectionParams = dataSource.getConnectionParams(); - JSONObject object = JSONObject.parseObject(connectionParams); + JSONObject object = JSON.parseObject(connectionParams); object.put(Constants.PASSWORD, Constants.XXXXXX); dataSource.setConnectionParams(JSONUtils.toJson(object)); @@ -367,11 +368,11 @@ public class DataSourceService extends BaseService{ try { switch (dbType) { case POSTGRESQL: - datasource = JSONObject.parseObject(parameter, PostgreDataSource.class); + datasource = JSON.parseObject(parameter, PostgreDataSource.class); Class.forName(Constants.ORG_POSTGRESQL_DRIVER); break; case MYSQL: - datasource = JSONObject.parseObject(parameter, MySQLDataSource.class); + datasource = JSON.parseObject(parameter, MySQLDataSource.class); Class.forName(Constants.COM_MYSQL_JDBC_DRIVER); break; case HIVE: @@ -386,26 +387,26 @@ public class DataSourceService extends BaseService{ getString(org.apache.dolphinscheduler.common.Constants.LOGIN_USER_KEY_TAB_PATH)); } if (dbType == DbType.HIVE){ - datasource = JSONObject.parseObject(parameter, HiveDataSource.class); + datasource = JSON.parseObject(parameter, HiveDataSource.class); }else if (dbType == DbType.SPARK){ - datasource = JSONObject.parseObject(parameter, SparkDataSource.class); + datasource = JSON.parseObject(parameter, SparkDataSource.class); } Class.forName(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER); break; case CLICKHOUSE: - datasource = JSONObject.parseObject(parameter, ClickHouseDataSource.class); + datasource = JSON.parseObject(parameter, ClickHouseDataSource.class); Class.forName(Constants.COM_CLICKHOUSE_JDBC_DRIVER); break; case ORACLE: - datasource = JSONObject.parseObject(parameter, OracleDataSource.class); + datasource = JSON.parseObject(parameter, OracleDataSource.class); Class.forName(Constants.COM_ORACLE_JDBC_DRIVER); break; case SQLSERVER: - datasource = JSONObject.parseObject(parameter, SQLServerDataSource.class); + datasource = JSON.parseObject(parameter, SQLServerDataSource.class); Class.forName(Constants.COM_SQLSERVER_JDBC_DRIVER); break; case DB2: - datasource = JSONObject.parseObject(parameter, DB2ServerDataSource.class); + datasource = JSON.parseObject(parameter, DB2ServerDataSource.class); Class.forName(Constants.COM_DB2_JDBC_DRIVER); break; default: @@ -507,7 +508,7 @@ public class DataSourceService extends BaseService{ parameterMap.put(Constants.PRINCIPAL,principal); } if (other != null && !"".equals(other)) { - LinkedHashMap map = JSONObject.parseObject(other, new TypeReference>() { + LinkedHashMap map = JSON.parseObject(other, new TypeReference>() { }); if (map.size() > 0) { StringBuilder otherSb = new StringBuilder(); @@ -523,9 +524,9 @@ public class DataSourceService extends BaseService{ } if(logger.isDebugEnabled()){ - logger.info("parameters map-----" + JSONObject.toJSONString(parameterMap)); + logger.info("parameters map-----" + JSON.toJSONString(parameterMap)); } - return JSONObject.toJSONString(parameterMap); + return JSON.toJSONString(parameterMap); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java index 40bdd5490d..a56e3f83ef 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.api.controller; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.ResourceType; @@ -54,7 +55,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -78,7 +79,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -281,7 +282,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -303,7 +304,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -324,7 +325,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -344,7 +345,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -365,7 +366,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -386,7 +387,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -406,7 +407,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -427,7 +428,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -446,7 +447,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java index b45bd8aeb8..c0ad907dca 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.common.model; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; @@ -23,7 +24,6 @@ import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import com.alibaba.fastjson.JSONObject; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -294,7 +294,7 @@ public class TaskNode { if(StringUtils.isNotEmpty(this.getTimeout())){ String formatStr = String.format("%s,%s", TaskTimeoutStrategy.WARN.name(), TaskTimeoutStrategy.FAILED.name()); String timeout = this.getTimeout().replace(formatStr,TaskTimeoutStrategy.WARNFAILED.name()); - return JSONObject.parseObject(timeout,TaskTimeoutParameter.class); + return JSON.parseObject(timeout,TaskTimeoutParameter.class); } return new TaskTimeoutParameter(false); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java index ec523b1ff2..b6ed71f402 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.TypeReference; @@ -59,7 +60,7 @@ public class JSONUtils { */ public static String toJson(Object object) { try{ - return JSONObject.toJSONString(object,false); + return JSON.toJSONString(object,false); } catch (Exception e) { logger.error("object to json exception!",e); } @@ -89,7 +90,7 @@ public class JSONUtils { } try { - return JSONObject.parseObject(json, clazz); + return JSON.parseObject(json, clazz); } catch (Exception e) { logger.error("parse object exception!",e); } @@ -178,7 +179,7 @@ public class JSONUtils { } try { - return JSONObject.parseObject(json, new TypeReference>(){}); + return JSON.parseObject(json, new TypeReference>(){}); } catch (Exception e) { logger.error("json to map exception!",e); } @@ -203,7 +204,7 @@ public class JSONUtils { } try { - return JSONObject.parseObject(json, new TypeReference>() {}); + return JSON.parseObject(json, new TypeReference>() {}); } catch (Exception e) { logger.error("json to map exception!",e); } @@ -218,7 +219,7 @@ public class JSONUtils { */ public static String toJsonString(Object object) { try{ - return JSONObject.toJSONString(object,false); + return JSON.toJSONString(object,false); } catch (Exception e) { throw new RuntimeException("Json deserialization exception.", e); } @@ -226,7 +227,7 @@ public class JSONUtils { public static JSONObject parseObject(String text) { try{ - return JSONObject.parseObject(text); + return JSON.parseObject(text); } catch (Exception e) { throw new RuntimeException("Json deserialization exception.", e); } @@ -234,7 +235,7 @@ public class JSONUtils { public static JSONArray parseArray(String text) { try{ - return JSONObject.parseArray(text); + return JSON.parseArray(text); } catch (Exception e) { throw new RuntimeException("Json deserialization exception.", e); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java index 9492b49cb1..7a4553aaf5 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DataType; @@ -23,7 +24,6 @@ import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils; import org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils; -import com.alibaba.fastjson.JSONObject; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.time.DateUtils; import org.slf4j.Logger; @@ -157,7 +157,7 @@ public class ParameterUtils { property.setValue(val); } } - return JSONObject.toJSONString(globalParamList); + return JSON.toJSONString(globalParamList); } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java index bd924e4852..d249314819 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java @@ -16,10 +16,10 @@ */ package org.apache.dolphinscheduler.common.utils; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.Direct; import org.apache.dolphinscheduler.common.process.Property; -import com.alibaba.fastjson.JSONObject; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import org.junit.Assert; @@ -53,7 +53,7 @@ public class JSONUtilsTest { property.setType(DataType.VARCHAR); property.setValue("sssssss"); String str = "{\"direct\":\"IN\",\"prop\":\"ds\",\"type\":\"VARCHAR\",\"value\":\"sssssss\"}"; - Property property1 = JSONObject.parseObject(str, Property.class); + Property property1 = JSON.parseObject(str, Property.class); Direct direct = property1.getDirect(); Assert.assertEquals(direct , Direct.IN); } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java index 8bb64b03c8..abdc15cc6e 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import org.apache.commons.lang.time.DateUtils; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DataType; @@ -91,13 +91,13 @@ public class ParameterUtilsTest { globalParamList.add(property); String result2 = ParameterUtils.curingGlobalParams(null,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,scheduleTime); - Assert.assertEquals(result2, JSONObject.toJSONString(globalParamList)); + Assert.assertEquals(result2, JSON.toJSONString(globalParamList)); String result3 = ParameterUtils.curingGlobalParams(globalParamMap,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,null); - Assert.assertEquals(result3, JSONObject.toJSONString(globalParamList)); + Assert.assertEquals(result3, JSON.toJSONString(globalParamList)); String result4 = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime); - Assert.assertEquals(result4, JSONObject.toJSONString(globalParamList)); + Assert.assertEquals(result4, JSON.toJSONString(globalParamList)); //test var $ startsWith globalParamMap.put("bizDate","${system.biz.date}"); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java index dbb880c025..f59d11f3fe 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java @@ -16,10 +16,10 @@ */ package org.apache.dolphinscheduler.dao.entity; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.process.Property; -import com.alibaba.fastjson.JSONObject; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; @@ -266,7 +266,7 @@ public class ProcessDefinition { } public void setGlobalParams(String globalParams) { - this.globalParamList = JSONObject.parseArray(globalParams, Property.class); + this.globalParamList = JSON.parseArray(globalParams, Property.class); this.globalParams = globalParams; } @@ -275,7 +275,7 @@ public class ProcessDefinition { } public void setGlobalParamList(List globalParamList) { - this.globalParams = JSONObject.toJSONString(globalParamList); + this.globalParams = JSON.toJSONString(globalParamList); this.globalParamList = globalParamList; } @@ -283,7 +283,7 @@ public class ProcessDefinition { List propList; if (globalParamMap == null && StringUtils.isNotEmpty(globalParams)) { - propList = JSONObject.parseArray(globalParams, Property.class); + propList = JSON.parseArray(globalParams, Property.class); globalParamMap = propList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 0b81a30191..849bbd5680 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.server.master.runner; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import com.google.common.collect.Lists; import org.apache.commons.io.FileUtils; import org.apache.dolphinscheduler.common.Constants; @@ -454,7 +454,7 @@ public class MasterExecThread implements Runnable { // process instance id taskInstance.setProcessInstanceId(processInstance.getId()); // task instance node json - taskInstance.setTaskJson(JSONObject.toJSONString(taskNode)); + taskInstance.setTaskJson(JSON.toJSONString(taskNode)); // task instance type taskInstance.setTaskType(taskNode.getType()); // task instance whether alert diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java index 37262c05e2..3cdb8a0271 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.server.master.runner; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; @@ -25,7 +26,6 @@ import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import com.alibaba.fastjson.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -176,7 +176,7 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { */ private TaskTimeoutParameter getTaskTimeoutParameter(){ String taskJson = taskInstance.getTaskJson(); - TaskNode taskNode = JSONObject.parseObject(taskJson, TaskNode.class); + TaskNode taskNode = JSON.parseObject(taskJson, TaskNode.class); return taskNode.getTaskTimeoutParameter(); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java index 21ee1dfa40..d36d4de5b4 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.server.worker.runner; import ch.qos.logback.classic.LoggerContext; import ch.qos.logback.classic.sift.SiftingAppender; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; @@ -93,7 +93,7 @@ public class TaskScheduleThread implements Runnable { logger.info("script path : {}", taskInstance.getExecutePath()); // task node - TaskNode taskNode = JSONObject.parseObject(taskInstance.getTaskJson(), TaskNode.class); + TaskNode taskNode = JSON.parseObject(taskInstance.getTaskJson(), TaskNode.class); // get resource files List resourceFiles = createProjectResFiles(taskNode); @@ -176,7 +176,7 @@ public class TaskScheduleThread implements Runnable { String globalParamsStr = taskInstance.getProcessInstance().getGlobalParams(); if (globalParamsStr != null) { - List globalParamsList = JSONObject.parseArray(globalParamsStr, Property.class); + List globalParamsList = JSON.parseArray(globalParamsStr, Property.class); globalParamsMap.putAll(globalParamsList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue))); } return globalParamsMap; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java index fb881453e9..fd00e517b5 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.server.worker.task.processdure; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import com.cronutils.utils.StringUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DataType; @@ -75,7 +75,7 @@ public class ProcedureTask extends AbstractTask { logger.info("procedure task params {}", taskProps.getTaskParams()); - this.procedureParameters = JSONObject.parseObject(taskProps.getTaskParams(), ProcedureParameters.class); + this.procedureParameters = JSON.parseObject(taskProps.getTaskParams(), ProcedureParameters.class); // check parameters if (!procedureParameters.checkParameters()) { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index aae11f5530..82034740fc 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.server.worker.task.sql; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.serializer.SerializerFeature; @@ -92,7 +93,7 @@ public class SqlTask extends AbstractTask { super(taskProps, logger); logger.info("sql task params {}", taskProps.getTaskParams()); - this.sqlParameters = JSONObject.parseObject(taskProps.getTaskParams(), SqlParameters.class); + this.sqlParameters = JSON.parseObject(taskProps.getTaskParams(), SqlParameters.class); if (!sqlParameters.checkParameters()) { throw new RuntimeException("sql task params is not valid"); @@ -308,16 +309,16 @@ public class SqlTask extends AbstractTask { } resultJSONArray.add(mapOfColValues); } - logger.debug("execute sql : {}", JSONObject.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); + logger.debug("execute sql : {}", JSON.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); // if there is a result set if ( !resultJSONArray.isEmpty() ) { if (StringUtils.isNotEmpty(sqlParameters.getTitle())) { sendAttachment(sqlParameters.getTitle(), - JSONObject.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); + JSON.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); }else{ sendAttachment(taskProps.getNodeName() + " query resultsets ", - JSONObject.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); + JSON.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java index d2a0fb2407..af312d9601 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.server.master; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.enums.*; import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.utils.DateUtils; @@ -85,7 +85,7 @@ public class MasterExecThreadTest { Map cmdParam = new HashMap<>(); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, "2020-01-01 00:00:00"); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, "2020-01-31 23:00:00"); - Mockito.when(processInstance.getCommandParam()).thenReturn(JSONObject.toJSONString(cmdParam)); + Mockito.when(processInstance.getCommandParam()).thenReturn(JSON.toJSONString(cmdParam)); ProcessDefinition processDefinition = new ProcessDefinition(); processDefinition.setGlobalParamMap(Collections.EMPTY_MAP); processDefinition.setGlobalParamList(Collections.EMPTY_LIST); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index 5d4263644b..250c8a2680 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.server.worker.shell; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.TaskNode; @@ -68,7 +68,7 @@ public class ShellCommandExecutorTest { TaskInstance taskInstance = processService.findTaskInstanceById(7657); String taskJson = taskInstance.getTaskJson(); - TaskNode taskNode = JSONObject.parseObject(taskJson, TaskNode.class); + TaskNode taskNode = JSON.parseObject(taskJson, TaskNode.class); taskProps.setTaskParams(taskNode.getParams()); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java index c395eabe51..07b700239b 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.server.worker.sql; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; @@ -112,7 +112,7 @@ public class SqlExecutorTest { TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId); String taskJson = taskInstance.getTaskJson(); - TaskNode taskNode = JSONObject.parseObject(taskJson, TaskNode.class); + TaskNode taskNode = JSON.parseObject(taskJson, TaskNode.class); taskProps.setTaskParams(taskNode.getParams()); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java index 511102e4b5..5c7afa8155 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.server.worker.task.sqoop; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; import org.apache.dolphinscheduler.dao.entity.DataSource; @@ -74,7 +74,7 @@ public class SqoopTaskTest { @Test public void testGenerator(){ String data1 = "{\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HDFS\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"0\\\",\\\"srcQuerySql\\\":\\\"\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[]}\",\"targetParams\":\"{\\\"targetPath\\\":\\\"/ods/tmp/test/person7\\\",\\\"deleteTargetDir\\\":true,\\\"fileType\\\":\\\"--as-textfile\\\",\\\"compressionCodec\\\":\\\"\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; - SqoopParameters sqoopParameters1 = JSONObject.parseObject(data1,SqoopParameters.class); + SqoopParameters sqoopParameters1 = JSON.parseObject(data1,SqoopParameters.class); SqoopJobGenerator generator = new SqoopJobGenerator(); String script = generator.generateSqoopJob(sqoopParameters1); @@ -82,21 +82,21 @@ public class SqoopTaskTest { Assert.assertEquals(expected, script); String data2 = "{\"concurrency\":1,\"modelType\":\"export\",\"sourceType\":\"HDFS\",\"targetType\":\"MYSQL\",\"sourceParams\":\"{\\\"exportDir\\\":\\\"/ods/tmp/test/person7\\\"}\",\"targetParams\":\"{\\\"targetDatasource\\\":2,\\\"targetTable\\\":\\\"person_3\\\",\\\"targetColumns\\\":\\\"id,name,age,sex,create_time\\\",\\\"preQuery\\\":\\\"\\\",\\\"isUpdate\\\":true,\\\"targetUpdateKey\\\":\\\"id\\\",\\\"targetUpdateMode\\\":\\\"allowinsert\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; - SqoopParameters sqoopParameters2 = JSONObject.parseObject(data2,SqoopParameters.class); + SqoopParameters sqoopParameters2 = JSON.parseObject(data2,SqoopParameters.class); String script2 = generator.generateSqoopJob(sqoopParameters2); String expected2 = "sqoop export -m 1 --export-dir /ods/tmp/test/person7 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_3 --columns id,name,age,sex,create_time --fields-terminated-by '@' --lines-terminated-by '\\n' --update-key id --update-mode allowinsert"; Assert.assertEquals(expected2, script2); String data3 = "{\"concurrency\":1,\"modelType\":\"export\",\"sourceType\":\"HIVE\",\"targetType\":\"MYSQL\",\"sourceParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-17\\\"}\",\"targetParams\":\"{\\\"targetDatasource\\\":2,\\\"targetTable\\\":\\\"person_3\\\",\\\"targetColumns\\\":\\\"\\\",\\\"preQuery\\\":\\\"\\\",\\\"isUpdate\\\":false,\\\"targetUpdateKey\\\":\\\"\\\",\\\"targetUpdateMode\\\":\\\"allowinsert\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; - SqoopParameters sqoopParameters3 = JSONObject.parseObject(data3,SqoopParameters.class); + SqoopParameters sqoopParameters3 = JSON.parseObject(data3,SqoopParameters.class); String script3 = generator.generateSqoopJob(sqoopParameters3); String expected3 = "sqoop export -m 1 --hcatalog-database stg --hcatalog-table person_internal --hcatalog-partition-keys date --hcatalog-partition-values 2020-02-17 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_3 --fields-terminated-by '@' --lines-terminated-by '\\n'"; Assert.assertEquals(expected3, script3); String data4 = "{\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HIVE\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"1\\\",\\\"srcQuerySql\\\":\\\"SELECT * FROM person_2\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[{\\\"prop\\\":\\\"id\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"Integer\\\"}]}\",\"targetParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal_2\\\",\\\"createHiveTable\\\":true,\\\"dropDelimiter\\\":false,\\\"hiveOverWrite\\\":true,\\\"replaceDelimiter\\\":\\\"\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-16\\\"}\",\"localParams\":[]}"; - SqoopParameters sqoopParameters4 = JSONObject.parseObject(data4,SqoopParameters.class); + SqoopParameters sqoopParameters4 = JSON.parseObject(data4,SqoopParameters.class); String script4 = generator.generateSqoopJob(sqoopParameters4); String expected4 = "sqoop import -m 1 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --query 'SELECT * FROM person_2 WHERE $CONDITIONS' --map-column-java id=Integer --hive-import --hive-table stg.person_internal_2 --create-hive-table --hive-overwrite -delete-target-dir --hive-partition-key date --hive-partition-value 2020-02-16"; diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index 8bcd64f1fd..c848ec5197 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.service.process; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.cronutils.model.Cron; import org.apache.commons.lang.ArrayUtils; @@ -207,7 +208,7 @@ public class ProcessService { CommandType commandType = command.getCommandType(); if(cmdTypeMap.containsKey(commandType)){ - JSONObject cmdParamObj = (JSONObject) JSONObject.parse(command.getCommandParam()); + JSONObject cmdParamObj = (JSONObject) JSON.parse(command.getCommandParam()); JSONObject tempObj; int processInstanceId = cmdParamObj.getInteger(CMDPARAM_RECOVER_PROCESS_ID_STRING); @@ -215,7 +216,7 @@ public class ProcessService { // for all commands for (Command tmpCommand:commands){ if(cmdTypeMap.containsKey(tmpCommand.getCommandType())){ - tempObj = (JSONObject) JSONObject.parse(tmpCommand.getCommandParam()); + tempObj = (JSONObject) JSON.parse(tmpCommand.getCommandParam()); if(tempObj != null && processInstanceId == tempObj.getInteger(CMDPARAM_RECOVER_PROCESS_ID_STRING)){ isNeedCreate = false; break; @@ -309,7 +310,7 @@ public class ProcessService { for (TaskNode taskNode : taskNodeList){ String parameter = taskNode.getParams(); if (parameter.contains(CMDPARAM_SUB_PROCESS_DEFINE_ID)){ - SubProcessParameters subProcessParam = JSONObject.parseObject(parameter, SubProcessParameters.class); + SubProcessParameters subProcessParam = JSON.parseObject(parameter, SubProcessParameters.class); ids.add(subProcessParam.getProcessDefinitionId()); recurseFindSubProcessId(subProcessParam.getProcessDefinitionId(),ids); } From 81f53ba0db1ed828c51ae7c19567dae8b99ebfa6 Mon Sep 17 00:00:00 2001 From: DS Date: Sat, 14 Mar 2020 08:30:41 +0800 Subject: [PATCH 028/401] Create .asf.yaml (#2166) --- .asf.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .asf.yaml diff --git a/.asf.yaml b/.asf.yaml new file mode 100644 index 0000000000..650e3c7e75 --- /dev/null +++ b/.asf.yaml @@ -0,0 +1,4 @@ +# Web site staging services: + +github: + description: "Dolphin Scheduler is a distributed and easy-to-extend visual workflow scheduling platform, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.(分布式易扩展的可视化工作流任务调度)" From c0a0d6937f93cc3346300ebea3d243037e08ae6b Mon Sep 17 00:00:00 2001 From: DS Date: Sat, 14 Mar 2020 08:42:55 +0800 Subject: [PATCH 029/401] Update .asf.yaml (#2167) --- .asf.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.asf.yaml b/.asf.yaml index 650e3c7e75..7e950a0a53 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -1,4 +1,7 @@ -# Web site staging services: +staging: + profile: ~ + whoami: master + foo: trigger github: description: "Dolphin Scheduler is a distributed and easy-to-extend visual workflow scheduling platform, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.(分布式易扩展的可视化工作流任务调度)" From 226e87e3d8e7a60af23aee6663aeb2229631bbe1 Mon Sep 17 00:00:00 2001 From: DS Date: Sat, 14 Mar 2020 08:47:23 +0800 Subject: [PATCH 030/401] Delete .asf.yaml (#2168) --- .asf.yaml | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 .asf.yaml diff --git a/.asf.yaml b/.asf.yaml deleted file mode 100644 index 7e950a0a53..0000000000 --- a/.asf.yaml +++ /dev/null @@ -1,7 +0,0 @@ -staging: - profile: ~ - whoami: master - foo: trigger - -github: - description: "Dolphin Scheduler is a distributed and easy-to-extend visual workflow scheduling platform, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.(分布式易扩展的可视化工作流任务调度)" From 8b61afc9be2988992934c6d00316a8463ec85232 Mon Sep 17 00:00:00 2001 From: Rubik-W <39549317+Rubik-W@users.noreply.github.com> Date: Sat, 14 Mar 2020 18:10:27 +0800 Subject: [PATCH 031/401] The master and worker server exit exception #2163 (#2176) * fix: #2163 * fix: format --- .../common/thread/ThreadUtils.java | 17 +++++-- .../server/master/MasterServer.java | 33 +++++++------- .../server/worker/WorkerServer.java | 45 +++++++------------ 3 files changed, 46 insertions(+), 49 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java index 0a4ed9b5ac..d8ef0bb38d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java @@ -120,12 +120,24 @@ public class ThreadUtils { /** * Wrapper over ScheduledThreadPoolExecutor + * @param threadName * @param corePoolSize * @return */ - public static ScheduledExecutorService newDaemonThreadScheduledExecutor(String threadName,int corePoolSize) { + public static ScheduledExecutorService newDaemonThreadScheduledExecutor(String threadName, int corePoolSize) { + return newThreadScheduledExecutor(threadName, corePoolSize, true); + } + + /** + * Wrapper over ScheduledThreadPoolExecutor + * @param threadName + * @param corePoolSize + * @param isDaemon + * @return + */ + public static ScheduledExecutorService newThreadScheduledExecutor(String threadName, int corePoolSize, boolean isDaemon) { ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setDaemon(true) + .setDaemon(isDaemon) .setNameFormat(threadName) .build(); ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(corePoolSize, threadFactory); @@ -135,7 +147,6 @@ public class ThreadUtils { return executor; } - public static ThreadInfo getThreadInfo(Thread t) { long tid = t.getId(); return threadBean.getThreadInfo(tid, STACK_DEPTH); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index 6b5063cba4..9512b1a1c2 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.runner.MasterSchedulerThread; +import org.apache.dolphinscheduler.server.worker.WorkerServer; import org.apache.dolphinscheduler.server.zk.ZKMasterClient; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; @@ -37,8 +38,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.WebApplicationType; import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.FilterType; import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -46,7 +49,9 @@ import java.util.concurrent.TimeUnit; /** * master server */ -@ComponentScan("org.apache.dolphinscheduler") +@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = { + @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, classes = {WorkerServer.class}) +}) public class MasterServer implements IStoppable { /** @@ -112,7 +117,7 @@ public class MasterServer implements IStoppable { masterSchedulerService = ThreadUtils.newDaemonSingleThreadExecutor("Master-Scheduler-Thread"); - heartbeatMasterService = ThreadUtils.newDaemonThreadScheduledExecutor("Master-Main-Thread",Constants.DEFAULT_MASTER_HEARTBEAT_THREAD_NUM); + heartbeatMasterService = ThreadUtils.newThreadScheduledExecutor("Master-Main-Thread",Constants.DEFAULT_MASTER_HEARTBEAT_THREAD_NUM, false); // heartbeat thread implement Runnable heartBeatThread = heartBeatThread(); @@ -147,23 +152,17 @@ public class MasterServer implements IStoppable { } logger.error("start Quartz failed", e); } - - - /** - * register hooks, which are called before the process exits - */ - Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { - @Override - public void run() { - if (zkMasterClient.getActiveMasterNum() <= 1) { - zkMasterClient.getAlertDao().sendServerStopedAlert( - 1, OSUtils.getHost(), "Master-Server"); - } - stop("shutdownhook"); - } - })); } + @PreDestroy + public void destroy() { + // master server exit alert + if (zkMasterClient.getActiveMasterNum() <= 1) { + zkMasterClient.getAlertDao().sendServerStopedAlert( + 1, OSUtils.getHost(), "Master-Server"); + } + stop("shutdownhook"); + } /** * gracefully stop diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index ace93079ff..86bb7d3e07 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -29,6 +29,7 @@ import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.server.master.MasterServer; import org.apache.dolphinscheduler.server.utils.ProcessUtils; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.runner.FetchTaskThread; @@ -43,10 +44,13 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.FilterType; import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -56,7 +60,10 @@ import java.util.concurrent.TimeUnit; /** * worker server */ -@ComponentScan("org.apache.dolphinscheduler") +@SpringBootApplication +@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = { + @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, classes = {MasterServer.class}) +}) public class WorkerServer implements IStoppable { /** @@ -104,11 +111,6 @@ public class WorkerServer implements IStoppable { */ private ExecutorService fetchTaskExecutorService; - /** - * CountDownLatch latch - */ - private CountDownLatch latch; - @Value("${server.is-combined-server:false}") private Boolean isCombinedServer; @@ -149,7 +151,7 @@ public class WorkerServer implements IStoppable { this.fetchTaskExecutorService = ThreadUtils.newDaemonSingleThreadExecutor("Worker-Fetch-Thread-Executor"); - heartbeatWorkerService = ThreadUtils.newDaemonThreadScheduledExecutor("Worker-Heartbeat-Thread-Executor", Constants.DEFAUL_WORKER_HEARTBEAT_THREAD_NUM); + heartbeatWorkerService = ThreadUtils.newThreadScheduledExecutor("Worker-Heartbeat-Thread-Executor", Constants.DEFAUL_WORKER_HEARTBEAT_THREAD_NUM, false); // heartbeat thread implement Runnable heartBeatThread = heartBeatThread(); @@ -171,29 +173,15 @@ public class WorkerServer implements IStoppable { // submit fetch task thread fetchTaskExecutorService.execute(fetchTaskThread); + } - /** - * register hooks, which are called before the process exits - */ - Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { - @Override - public void run() { - // worker server exit alert - if (zkWorkerClient.getActiveMasterNum() <= 1) { - alertDao.sendServerStopedAlert(1, OSUtils.getHost(), "Worker-Server"); - } - stop("shutdownhook"); - } - })); - - //let the main thread await - latch = new CountDownLatch(1); - if (!isCombinedServer) { - try { - latch.await(); - } catch (InterruptedException ignore) { - } + @PreDestroy + public void destroy() { + // worker server exit alert + if (zkWorkerClient.getActiveMasterNum() <= 1) { + alertDao.sendServerStopedAlert(1, OSUtils.getHost(), "Worker-Server"); } + stop("shutdownhook"); } @Override @@ -251,7 +239,6 @@ public class WorkerServer implements IStoppable { }catch (Exception e){ logger.warn("zookeeper service stopped exception:{}",e.getMessage()); } - latch.countDown(); logger.info("zookeeper service stopped"); } catch (Exception e) { From bdfb9d6a5097cb8ac9a4a66af6e2b3f917f5c7e2 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Sat, 14 Mar 2020 18:12:14 +0800 Subject: [PATCH 032/401] Adapting partial code(file name start with J) to the sonar cloud rule (#2165) * Adapting partial code(file name start with J) to the sonar cloud rule * fix unit failure --- .../alert/utils/JSONUtils.java | 7 ++--- .../alert/utils/JSONUtilsTest.java | 27 ++++++++++--------- .../common/utils/JSONUtils.java | 10 ++----- .../common/utils/JSONUtilsTest.java | 18 ++++++------- 4 files changed, 29 insertions(+), 33 deletions(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java index 8d66255fc4..5f8371de2d 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collections; import java.util.List; /** @@ -50,11 +51,11 @@ public class JSONUtils { * @param json the json * @param clazz c * @param the generic clazz - * @return the result list + * @return the result list or empty list */ public static List toList(String json, Class clazz) { if (StringUtils.isEmpty(json)) { - return null; + return Collections.emptyList(); } try { return JSON.parseArray(json, clazz); @@ -62,7 +63,7 @@ public class JSONUtils { logger.error("JSONArray.parseArray exception!",e); } - return null; + return Collections.emptyList(); } } diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java index 277c42b5bd..a151abc714 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java @@ -26,8 +26,7 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.Assert.*; public class JSONUtilsTest { @@ -73,7 +72,7 @@ public class JSONUtilsTest { result = JSONUtils.toJsonString(null); logger.info(result); - assertEquals(result,"null"); + assertEquals("null", result); } @@ -86,25 +85,27 @@ public class JSONUtilsTest { //Invoke toList List result = JSONUtils.toList(expected ,LinkedHashMap.class); //Equal list size=1 - assertEquals(result.size(),1); + assertEquals(1,result.size()); //Transform entity to LinkedHashMap LinkedHashMap entity = result.get(0); //Equal expected values - assertEquals(entity.get("mysql service name"),"mysql200"); - assertEquals(entity.get("mysql address"),"192.168.xx.xx"); - assertEquals(entity.get("port"),"3306"); - assertEquals(entity.get("no index of number"),"80"); - assertEquals(entity.get("database client connections"),"190"); + assertEquals("mysql200",entity.get("mysql service name")); + assertEquals("192.168.xx.xx", entity.get("mysql address")); + assertEquals("3306", entity.get("port")); + assertEquals("80", entity.get("no index of number")); + assertEquals("190", entity.get("database client connections")); - //If param is null, then return null + //If param is null, then return empty list result = JSONUtils.toList(null ,LinkedHashMap.class); - assertNull(result); + assertNotNull(result); + assertTrue(result.isEmpty()); - //If param is incorrect, then return null and log error message + //If param is incorrect, then return empty list and log error message result = JSONUtils.toList("}{" ,LinkedHashMap.class); - assertNull(result); + assertNotNull(result); + assertTrue(result.isEmpty()); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java index b6ed71f402..f0aed91a0d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java @@ -42,12 +42,6 @@ public class JSONUtils { */ private static final ObjectMapper objectMapper = new ObjectMapper(); - /** - * init - */ - private static final JSONUtils instance = new JSONUtils(); - - private JSONUtils() { //Feature that determines whether encountering of unknown properties, false means not analyzer unknown properties objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false).setTimeZone(TimeZone.getDefault()); @@ -221,7 +215,7 @@ public class JSONUtils { try{ return JSON.toJSONString(object,false); } catch (Exception e) { - throw new RuntimeException("Json deserialization exception.", e); + throw new RuntimeException("Object json deserialization exception.", e); } } @@ -229,7 +223,7 @@ public class JSONUtils { try{ return JSON.parseObject(text); } catch (Exception e) { - throw new RuntimeException("Json deserialization exception.", e); + throw new RuntimeException("String json deserialization exception.", e); } } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java index d249314819..8ce60349ed 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java @@ -40,8 +40,8 @@ public class JSONUtilsTest { String jsonStr = "{\"id\":\"1001\",\"name\":\"Jobs\"}"; Map models = JSONUtils.toMap(jsonStr); - Assert.assertEquals(models.get("id"), "1001"); - Assert.assertEquals(models.get("name"), "Jobs"); + Assert.assertEquals("1001", models.get("id")); + Assert.assertEquals("Jobs", models.get("name")); } @@ -55,7 +55,7 @@ public class JSONUtilsTest { String str = "{\"direct\":\"IN\",\"prop\":\"ds\",\"type\":\"VARCHAR\",\"value\":\"sssssss\"}"; Property property1 = JSON.parseObject(str, Property.class); Direct direct = property1.getDirect(); - Assert.assertEquals(direct , Direct.IN); + Assert.assertEquals(Direct.IN, direct); } @@ -66,12 +66,12 @@ public class JSONUtilsTest { List maps = JSONUtils.toList(str, LinkedHashMap.class); - Assert.assertEquals(maps.size(), 1); - Assert.assertEquals(maps.get(0).get("mysql service name"), "mysql200"); - Assert.assertEquals(maps.get(0).get("mysql address"), "192.168.xx.xx"); - Assert.assertEquals(maps.get(0).get("port"), "3306"); - Assert.assertEquals(maps.get(0).get("no index of number"), "80"); - Assert.assertEquals(maps.get(0).get("database client connections"), "190"); + Assert.assertEquals(1, maps.size()); + Assert.assertEquals("mysql200", maps.get(0).get("mysql service name")); + Assert.assertEquals("192.168.xx.xx", maps.get(0).get("mysql address")); + Assert.assertEquals("3306", maps.get(0).get("port")); + Assert.assertEquals("80", maps.get(0).get("no index of number")); + Assert.assertEquals("190", maps.get(0).get("database client connections")); } public String list2String(){ From e7c7cea4afe108df6411887c0d0a1d2c9e599b27 Mon Sep 17 00:00:00 2001 From: Rubik-W <39549317+Rubik-W@users.noreply.github.com> Date: Sat, 14 Mar 2020 19:20:46 +0800 Subject: [PATCH 033/401] fix: NullPointException in the case of task queuing (#2175) --- .../dolphinscheduler/server/master/runner/MasterExecThread.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 849bbd5680..0c44b7a77e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -1062,7 +1062,7 @@ public class MasterExecThread implements Runnable { TaskInstance taskInstance = taskExecThread.getTaskInstance(); taskInstance = processService.findTaskInstanceById(taskInstance.getId()); - if(taskInstance.getState().typeIsFinished()){ + if(taskInstance != null && taskInstance.getState().typeIsFinished()){ continue; } From 7f7f50a828c37fcaeda5723ff6d501cce4d13e76 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Sat, 14 Mar 2020 19:22:24 +0800 Subject: [PATCH 034/401] Adapting partial code(file name start with L) to the sonar cloud rule (#2171) --- .../interceptor/LoginHandlerInterceptor.java | 11 ---------- .../common/utils/LoggerUtils.java | 2 +- .../server/log/LoggerRequestProcessor.java | 21 +++++-------------- .../service/log/LogClientService.java | 8 +++---- 4 files changed, 10 insertions(+), 32 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java index 380eea5774..98bac42f72 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java @@ -27,7 +27,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.servlet.HandlerInterceptor; -import org.springframework.web.servlet.ModelAndView; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -90,14 +89,4 @@ public class LoginHandlerInterceptor implements HandlerInterceptor { return true; } - @Override - public void postHandle(HttpServletRequest request, HttpServletResponse response, Object handler, ModelAndView modelAndView) throws Exception { - - } - - @Override - public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) throws Exception { - - } - } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java index fc08eb645b..191df335c5 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java @@ -79,7 +79,7 @@ public class LoggerUtils { */ public static List getAppIds(String log, Logger logger) { - List appIds = new ArrayList(); + List appIds = new ArrayList<>(); Matcher matcher = APPLICATION_REGEX.matcher(log); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java index 4e4404ea1c..4b21d5f4a2 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.log; import io.netty.channel.Channel; +import org.apache.dolphinscheduler.common.utils.IOUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.log.*; @@ -116,16 +117,8 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { }catch (IOException e){ logger.error("get file bytes error",e); }finally { - if (bos != null){ - try { - bos.close(); - } catch (IOException ignore) {} - } - if (in != null){ - try { - in.close(); - } catch (IOException ignore) {} - } + IOUtils.closeQuietly(bos); + IOUtils.closeQuietly(in); } return new byte[0]; } @@ -146,7 +139,7 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { } catch (IOException e) { logger.error("read file error",e); } - return Collections.EMPTY_LIST; + return Collections.emptyList(); } /** @@ -168,11 +161,7 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { }catch (IOException e){ logger.error("read file error",e); }finally { - try { - if (br != null){ - br.close(); - } - } catch (IOException ignore) {} + IOUtils.closeQuietly(br); } return ""; } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java index 01a49910df..163cd3696f 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java @@ -40,7 +40,7 @@ public class LogClientService { /** * request time out */ - private final long logRequestTimeout = 10 * 1000; + private static final long LOG_REQUEST_TIMEOUT = 10 * 1000L; /** * construct client @@ -75,7 +75,7 @@ public class LogClientService { final Address address = new Address(host, port); try { Command command = request.convert2Command(); - Command response = this.client.sendSync(address, command, logRequestTimeout); + Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); if(response != null){ RollViewLogResponseCommand rollReviewLog = FastJsonSerializer.deserialize( response.getBody(), RollViewLogResponseCommand.class); @@ -103,7 +103,7 @@ public class LogClientService { final Address address = new Address(host, port); try { Command command = request.convert2Command(); - Command response = this.client.sendSync(address, command, logRequestTimeout); + Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); if(response != null){ ViewLogResponseCommand viewLog = FastJsonSerializer.deserialize( response.getBody(), ViewLogResponseCommand.class); @@ -131,7 +131,7 @@ public class LogClientService { final Address address = new Address(host, port); try { Command command = request.convert2Command(); - Command response = this.client.sendSync(address, command, logRequestTimeout); + Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); if(response != null){ GetLogBytesResponseCommand getLog = FastJsonSerializer.deserialize( response.getBody(), GetLogBytesResponseCommand.class); From 2f4fda7cdbb831191099d3e753aff42196e73e11 Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Sun, 15 Mar 2020 19:05:33 +0800 Subject: [PATCH 035/401] Adapting partial code(file name start with M #1) to the sonar cloud rule (#2173) * Adapting partial code(file name start with M #1) to the sonar cloud rule * remove unused import --- .../alert/utils/MailUtils.java | 52 ++++++++-------- .../common/task/mr/MapreduceParameters.java | 14 +++-- .../dao/entity/ProcessInstance.java | 2 +- .../dao/entity/TaskInstance.java | 4 +- .../master/runner/MasterExecThread.java | 61 +++++++++---------- .../server/worker/task/mr/MapReduceTask.java | 17 +++--- 6 files changed, 72 insertions(+), 78 deletions(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java index 130ad8ade6..66a63dcff4 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java @@ -39,29 +39,29 @@ public class MailUtils { public static final Logger logger = LoggerFactory.getLogger(MailUtils.class); - public static final String mailProtocol = PropertyUtils.getString(Constants.MAIL_PROTOCOL); + public static final String MAIL_PROTOCOL = PropertyUtils.getString(Constants.MAIL_PROTOCOL); - public static final String mailServerHost = PropertyUtils.getString(Constants.MAIL_SERVER_HOST); + public static final String MAIL_SERVER_HOST = PropertyUtils.getString(Constants.MAIL_SERVER_HOST); - public static final Integer mailServerPort = PropertyUtils.getInt(Constants.MAIL_SERVER_PORT); + public static final Integer MAIL_SERVER_PORT = PropertyUtils.getInt(Constants.MAIL_SERVER_PORT); - public static final String mailSender = PropertyUtils.getString(Constants.MAIL_SENDER); + public static final String MAIL_SENDER = PropertyUtils.getString(Constants.MAIL_SENDER); - public static final String mailUser = PropertyUtils.getString(Constants.MAIL_USER); + public static final String MAIL_USER = PropertyUtils.getString(Constants.MAIL_USER); - public static final String mailPasswd = PropertyUtils.getString(Constants.MAIL_PASSWD); + public static final String MAIL_PASSWD = PropertyUtils.getString(Constants.MAIL_PASSWD); - public static final Boolean mailUseStartTLS = PropertyUtils.getBoolean(Constants.MAIL_SMTP_STARTTLS_ENABLE); + public static final Boolean MAIL_USE_START_TLS = PropertyUtils.getBoolean(Constants.MAIL_SMTP_STARTTLS_ENABLE); - public static final Boolean mailUseSSL = PropertyUtils.getBoolean(Constants.MAIL_SMTP_SSL_ENABLE); + public static final Boolean MAIL_USE_SSL = PropertyUtils.getBoolean(Constants.MAIL_SMTP_SSL_ENABLE); - public static final String xlsFilePath = PropertyUtils.getString(Constants.XLS_FILE_PATH); + public static final String XLS_FILE_PATH = PropertyUtils.getString(Constants.XLS_FILE_PATH); - public static final String starttlsEnable = PropertyUtils.getString(Constants.MAIL_SMTP_STARTTLS_ENABLE); + public static final String STARTTLS_ENABLE = PropertyUtils.getString(Constants.MAIL_SMTP_STARTTLS_ENABLE); - public static final String sslEnable = PropertyUtils.getString(Constants.MAIL_SMTP_SSL_ENABLE); + public static final String SSL_ENABLE = PropertyUtils.getString(Constants.MAIL_SMTP_SSL_ENABLE); - public static final String sslTrust = PropertyUtils.getString(Constants.MAIL_SMTP_SSL_TRUST); + public static final String SSL_TRUST = PropertyUtils.getString(Constants.MAIL_SMTP_SSL_TRUST); public static final AlertTemplate alertTemplate = AlertTemplateFactory.getMessageTemplate(); @@ -105,7 +105,7 @@ public class MailUtils { try { Session session = getSession(); email.setMailSession(session); - email.setFrom(mailSender); + email.setFrom(MAIL_SENDER); email.setCharset(Constants.UTF_8); if (CollectionUtils.isNotEmpty(receivers)){ // receivers mail @@ -199,10 +199,10 @@ public class MailUtils { // 2. creating mail: Creating a MimeMessage MimeMessage msg = new MimeMessage(session); // 3. set sender - msg.setFrom(new InternetAddress(mailSender)); + msg.setFrom(new InternetAddress(MAIL_SENDER)); // 4. set receivers for (String receiver : receivers) { - msg.addRecipients(MimeMessage.RecipientType.TO, InternetAddress.parse(receiver)); + msg.addRecipients(Message.RecipientType.TO, InternetAddress.parse(receiver)); } return msg; } @@ -213,19 +213,19 @@ public class MailUtils { */ private static Session getSession() { Properties props = new Properties(); - props.setProperty(Constants.MAIL_HOST, mailServerHost); - props.setProperty(Constants.MAIL_PORT, String.valueOf(mailServerPort)); + props.setProperty(Constants.MAIL_HOST, MAIL_SERVER_HOST); + props.setProperty(Constants.MAIL_PORT, String.valueOf(MAIL_SERVER_PORT)); props.setProperty(Constants.MAIL_SMTP_AUTH, Constants.STRING_TRUE); - props.setProperty(Constants.MAIL_TRANSPORT_PROTOCOL, mailProtocol); - props.setProperty(Constants.MAIL_SMTP_STARTTLS_ENABLE, starttlsEnable); - props.setProperty(Constants.MAIL_SMTP_SSL_ENABLE, sslEnable); - props.setProperty(Constants.MAIL_SMTP_SSL_TRUST, sslTrust); + props.setProperty(Constants.MAIL_TRANSPORT_PROTOCOL, MAIL_PROTOCOL); + props.setProperty(Constants.MAIL_SMTP_STARTTLS_ENABLE, STARTTLS_ENABLE); + props.setProperty(Constants.MAIL_SMTP_SSL_ENABLE, SSL_ENABLE); + props.setProperty(Constants.MAIL_SMTP_SSL_TRUST, SSL_TRUST); Authenticator auth = new Authenticator() { @Override protected PasswordAuthentication getPasswordAuthentication() { // mail username and password - return new PasswordAuthentication(mailUser, mailPasswd); + return new PasswordAuthentication(MAIL_USER, MAIL_PASSWD); } }; @@ -248,12 +248,10 @@ public class MailUtils { */ if(CollectionUtils.isNotEmpty(receiversCc)){ for (String receiverCc : receiversCc){ - msg.addRecipients(MimeMessage.RecipientType.CC, InternetAddress.parse(receiverCc)); + msg.addRecipients(Message.RecipientType.CC, InternetAddress.parse(receiverCc)); } } - // set receivers type to cc - // msg.addRecipients(MimeMessage.RecipientType.CC, InternetAddress.parse(propMap.get("${CC}"))); // set subject msg.setSubject(title); MimeMultipart partList = new MimeMultipart(); @@ -263,8 +261,8 @@ public class MailUtils { // set attach file MimeBodyPart part2 = new MimeBodyPart(); // make excel file - ExcelUtils.genExcelFile(content,title,xlsFilePath); - File file = new File(xlsFilePath + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS); + ExcelUtils.genExcelFile(content,title, XLS_FILE_PATH); + File file = new File(XLS_FILE_PATH + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS); part2.attachFile(file); part2.setFileName(MimeUtility.encodeText(title + Constants.EXCEL_SUFFIX_XLS,Constants.UTF_8,"B")); // add components to collection diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/mr/MapreduceParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/mr/MapreduceParameters.java index b8fd6ebcbf..31c9c7292f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/mr/MapreduceParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/mr/MapreduceParameters.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -125,12 +126,15 @@ public class MapreduceParameters extends AbstractParameters { @Override public List getResourceFilesList() { - if (resourceList != null) { - this.resourceList.add(mainJar); - return resourceList.stream() - .map(p -> p.getRes()).collect(Collectors.toList()); + if(resourceList != null ) { + List resourceFiles = resourceList.stream() + .map(ResourceInfo::getRes).collect(Collectors.toList()); + if(mainJar != null) { + resourceFiles.add(mainJar.getRes()); + } + return resourceFiles; } - return null; + return Collections.emptyList(); } @Override diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java index fb5d0cda42..fd9653a36c 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java @@ -506,7 +506,7 @@ public class ProcessInstance { * check this process is start complement data * @return whether complement data */ - public Boolean isComplementData(){ + public boolean isComplementData(){ if(!StringUtils.isNotEmpty(this.historyCmd)){ return false; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java index c692575e3a..702897fba9 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java @@ -373,7 +373,7 @@ public class TaskInstance { } - public Boolean isSubProcess(){ + public boolean isSubProcess(){ return TaskType.SUB_PROCESS.getDescp().equals(this.taskType); } @@ -442,7 +442,7 @@ public class TaskInstance { this.executorName = executorName; } - public Boolean isTaskComplete() { + public boolean isTaskComplete() { return this.getState().typeIsPause() || this.getState().typeIsSuccess() diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 0c44b7a77e..26457a386a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -68,7 +68,7 @@ public class MasterExecThread implements Runnable { /** * runing TaskNode */ - private final Map> activeTaskNode = new ConcurrentHashMap>(); + private final Map> activeTaskNode = new ConcurrentHashMap<>(); /** * task exec service @@ -78,7 +78,7 @@ public class MasterExecThread implements Runnable { /** * submit failure nodes */ - private Boolean taskFailedSubmit = false; + private boolean taskFailedSubmit = false; /** * recover node id list @@ -652,7 +652,7 @@ public class MasterExecThread implements Runnable { continue; } if(task.getState().typeIsPause() || task.getState().typeIsCancel()){ - logger.info("task {} stopped, the state is {}", task.getName(), task.getState().toString()); + logger.info("task {} stopped, the state is {}", task.getName(), task.getState()); }else{ addTaskToStandByList(task); } @@ -685,11 +685,12 @@ public class MasterExecThread implements Runnable { } ExecutionStatus depTaskState = completeTaskList.get(depsNode).getState(); // conditions task would not return failed. - if(depTaskState.typeIsFailure()){ - if(!haveConditionsAfterNode(depsNode) && !dag.getNode(depsNode).isConditionsTask()){ - return DependResult.FAILED; - } + if(depTaskState.typeIsFailure() + && !haveConditionsAfterNode(depsNode) + && !dag.getNode(depsNode).isConditionsTask()){ + return DependResult.FAILED; } + if(depTaskState.typeIsPause() || depTaskState.typeIsCancel()){ return DependResult.WAITING; } @@ -737,7 +738,7 @@ public class MasterExecThread implements Runnable { * * @return Boolean whether has failed task */ - private Boolean hasFailedTask(){ + private boolean hasFailedTask(){ if(this.taskFailedSubmit){ return true; @@ -753,7 +754,7 @@ public class MasterExecThread implements Runnable { * * @return Boolean whether process instance failed */ - private Boolean processFailed(){ + private boolean processFailed(){ if(hasFailedTask()) { if(processInstance.getFailureStrategy() == FailureStrategy.END){ return true; @@ -769,9 +770,9 @@ public class MasterExecThread implements Runnable { * whether task for waiting thread * @return Boolean whether has waiting thread task */ - private Boolean hasWaitingThreadTask(){ + private boolean hasWaitingThreadTask(){ List waitingList = getCompleteTaskByState(ExecutionStatus.WAITTING_THREAD); - return waitingList.size() > 0; + return CollectionUtils.isNotEmpty(waitingList); } /** @@ -787,7 +788,7 @@ public class MasterExecThread implements Runnable { } List pauseList = getCompleteTaskByState(ExecutionStatus.PAUSE); - if(pauseList.size() > 0 + if(CollectionUtils.isNotEmpty(pauseList) || !isComplementEnd() || readyToSubmitTaskList.size() > 0){ return ExecutionStatus.PAUSE; @@ -827,7 +828,8 @@ public class MasterExecThread implements Runnable { if(state == ExecutionStatus.READY_STOP){ List stopList = getCompleteTaskByState(ExecutionStatus.STOP); List killList = getCompleteTaskByState(ExecutionStatus.KILL); - if(stopList.size() > 0 || killList.size() > 0 || !isComplementEnd()){ + if(CollectionUtils.isNotEmpty(stopList) + || CollectionUtils.isNotEmpty(killList) || !isComplementEnd()){ return ExecutionStatus.STOP; }else{ return ExecutionStatus.SUCCESS; @@ -852,7 +854,7 @@ public class MasterExecThread implements Runnable { * whether complement end * @return Boolean whether is complement end */ - private Boolean isComplementEnd() { + private boolean isComplementEnd() { if(!processInstance.isComplementData()){ return true; } @@ -877,8 +879,8 @@ public class MasterExecThread implements Runnable { logger.info( "work flow process instance [id: {}, name:{}], state change from {} to {}, cmd type: {}", processInstance.getId(), processInstance.getName(), - processInstance.getState().toString(), state.toString(), - processInstance.getCommandType().toString()); + processInstance.getState(), state, + processInstance.getCommandType()); processInstance.setState(state); ProcessInstance instance = processService.findProcessInstanceById(processInstance.getId()); instance.setState(state); @@ -894,8 +896,7 @@ public class MasterExecThread implements Runnable { * @return DependResult */ private DependResult getDependResultForTask(TaskInstance taskInstance){ - DependResult inner = isTaskDepsComplete(taskInstance.getName()); - return inner; + return isTaskDepsComplete(taskInstance.getName()); } /** @@ -920,7 +921,7 @@ public class MasterExecThread implements Runnable { * has retry task in standby * @return Boolean whether has retry task in standby */ - private Boolean hasRetryTaskInStandBy(){ + private boolean hasRetryTaskInStandBy(){ for (Map.Entry entry: readyToSubmitTaskList.entrySet()) { if(entry.getValue().getState().typeIsFailure()){ return true; @@ -958,7 +959,7 @@ public class MasterExecThread implements Runnable { continue; } logger.info("task :{}, id:{} complete, state is {} ", - task.getName(), task.getId(), task.getState().toString()); + task.getName(), task.getId(), task.getState()); // node success , post node submit if(task.getState() == ExecutionStatus.SUCCESS){ completeTaskList.put(task.getName(), task); @@ -990,7 +991,7 @@ public class MasterExecThread implements Runnable { completeTaskList.put(task.getName(), task); } // send alert - if(this.recoverToleranceFaultTaskList.size() > 0){ + if(CollectionUtils.isNotEmpty(this.recoverToleranceFaultTaskList)){ alertManager.sendAlertWorkerToleranceFault(processInstance, recoverToleranceFaultTaskList); this.recoverToleranceFaultTaskList.clear(); } @@ -1034,10 +1035,7 @@ public class MasterExecThread implements Runnable { Date now = new Date(); long runningTime = DateUtils.diffMin(now, processInstance.getStartTime()); - if(runningTime > processInstance.getTimeout()){ - return true; - } - return false; + return runningTime > processInstance.getTimeout(); } /** @@ -1081,22 +1079,19 @@ public class MasterExecThread implements Runnable { * @param taskInstance task instance * @return Boolean */ - private Boolean retryTaskIntervalOverTime(TaskInstance taskInstance){ + private boolean retryTaskIntervalOverTime(TaskInstance taskInstance){ if(taskInstance.getState() != ExecutionStatus.FAILURE){ - return Boolean.TRUE; + return true; } if(taskInstance.getId() == 0 || taskInstance.getMaxRetryTimes() ==0 || taskInstance.getRetryInterval() == 0 ){ - return Boolean.TRUE; + return true; } Date now = new Date(); long failedTimeInterval = DateUtils.differSec(now, taskInstance.getEndTime()); // task retry does not over time, return false - if(taskInstance.getRetryInterval() * SEC_2_MINUTES_TIME_UNIT >= failedTimeInterval){ - return Boolean.FALSE; - } - return Boolean.TRUE; + return taskInstance.getRetryInterval() * SEC_2_MINUTES_TIME_UNIT < failedTimeInterval; } /** @@ -1189,7 +1184,7 @@ public class MasterExecThread implements Runnable { */ private List getRecoveryNodeNameList(){ List recoveryNodeNameList = new ArrayList<>(); - if(recoverNodeIdList.size() > 0) { + if(CollectionUtils.isNotEmpty(recoverNodeIdList)) { for (TaskInstance task : recoverNodeIdList) { recoveryNodeNameList.add(task.getName()); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java index b86ff9952e..7f6baad427 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java @@ -122,22 +122,19 @@ public class MapReduceTask extends AbstractYarnTask { } // main class - if(mapreduceParameters.getProgramType() !=null ){ - if(mapreduceParameters.getProgramType()!= ProgramType.PYTHON){ - if(StringUtils.isNotEmpty(mapreduceParameters.getMainClass())){ - result.add(mapreduceParameters.getMainClass()); - } - } + if(!ProgramType.PYTHON.equals(mapreduceParameters.getProgramType()) + && StringUtils.isNotEmpty(mapreduceParameters.getMainClass())){ + result.add(mapreduceParameters.getMainClass()); } // others if (StringUtils.isNotEmpty(mapreduceParameters.getOthers())) { String others = mapreduceParameters.getOthers(); - if(!others.contains(Constants.MR_QUEUE)){ - if (StringUtils.isNotEmpty(mapreduceParameters.getQueue())) { - result.add(String.format("%s %s=%s", Constants.D, Constants.MR_QUEUE, mapreduceParameters.getQueue())); - } + if (!others.contains(Constants.MR_QUEUE) + && StringUtils.isNotEmpty(mapreduceParameters.getQueue())) { + result.add(String.format("%s %s=%s", Constants.D, Constants.MR_QUEUE, mapreduceParameters.getQueue())); } + result.add(mapreduceParameters.getOthers()); }else if (StringUtils.isNotEmpty(mapreduceParameters.getQueue())) { result.add(String.format("%s %s=%s", Constants.D, Constants.MR_QUEUE, mapreduceParameters.getQueue())); From 77c4bcb96e3a0dc55e2e7e3b2c495d86b235d877 Mon Sep 17 00:00:00 2001 From: tswstarplanet Date: Sun, 15 Mar 2020 21:14:43 +0800 Subject: [PATCH 036/401] Cache HadoopUtils instance with specific days expire time (#2181) * Cache HadoopUtils instance with 7 days expire time * solve sonar issue * add kerberos expire time config * move KERBEROS_EXPIRE_TIME to Constants.java --- .../dolphinscheduler/common/Constants.java | 5 + .../common/utils/HadoopUtils.java | 234 +++++++++--------- .../src/main/resources/common.properties | 2 +- 3 files changed, 124 insertions(+), 117 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 73125f4926..9d6ce6d9c7 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -813,6 +813,11 @@ public final class Constants { */ public static final String KERBEROS = "kerberos"; + /** + * kerberos expire time + */ + public static final String KERBEROS_EXPIRE_TIME = "kerberos.expire.time"; + /** * java.security.krb5.conf */ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 6cb58a4324..e767911580 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -16,6 +16,9 @@ */ package org.apache.dolphinscheduler.common.utils; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ResUploadType; @@ -37,6 +40,7 @@ import java.security.PrivilegedExceptionAction; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -48,30 +52,37 @@ public class HadoopUtils implements Closeable { private static final Logger logger = LoggerFactory.getLogger(HadoopUtils.class); - private static HadoopUtils instance = new HadoopUtils(); + private static final String HADOOP_UTILS_KEY = "HADOOP_UTILS_KEY"; + + private static final LoadingCache cache = CacheBuilder + .newBuilder() + .expireAfterWrite(PropertyUtils.getInt(Constants.KERBEROS_EXPIRE_TIME, 7), TimeUnit.DAYS) + .build(new CacheLoader() { + @Override + public HadoopUtils load(String key) throws Exception { + return new HadoopUtils(); + } + }); + private static Configuration configuration; private static FileSystem fs; - private String hdfsUser; + private static String hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER); - private HadoopUtils(){ - hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER); + private HadoopUtils() { init(); initHdfsPath(); } - public static HadoopUtils getInstance(){ - // if kerberos startup , renew HadoopUtils - if (CommonUtils.getKerberosStartupState()){ - return new HadoopUtils(); - } - return instance; + public static HadoopUtils getInstance() { + + return cache.getUnchecked(HADOOP_UTILS_KEY); } /** * init dolphinscheduler root path in hdfs */ - private void initHdfsPath(){ + private void initHdfsPath() { String hdfsPath = PropertyUtils.getString(Constants.DATA_STORE_2_HDFS_BASEPATH); Path path = new Path(hdfsPath); @@ -80,7 +91,7 @@ public class HadoopUtils implements Closeable { fs.mkdirs(path); } } catch (Exception e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); } } @@ -88,82 +99,75 @@ public class HadoopUtils implements Closeable { /** * init hadoop configuration */ - private void init() { - if (configuration == null) { - synchronized (HadoopUtils.class) { - if (configuration == null) { - try { - configuration = new Configuration(); - - String resUploadStartupType = PropertyUtils.getString(Constants.RES_UPLOAD_STARTUP_TYPE); - ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); - - if (resUploadType == ResUploadType.HDFS){ - if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE)){ - System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, - PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)); - configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION,"kerberos"); - UserGroupInformation.setConfiguration(configuration); - UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), - PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)); - } + private static void init() { + try { + configuration = new Configuration(); + + String resUploadStartupType = PropertyUtils.getString(Constants.RES_UPLOAD_STARTUP_TYPE); + ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); + + if (resUploadType == ResUploadType.HDFS) { + if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE)) { + System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, + PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)); + configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + UserGroupInformation.setConfiguration(configuration); + UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), + PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)); + } - String defaultFS = configuration.get(Constants.FS_DEFAULTFS); - //first get key from core-site.xml hdfs-site.xml ,if null ,then try to get from properties file - // the default is the local file system - if(defaultFS.startsWith("file")){ - String defaultFSProp = PropertyUtils.getString(Constants.FS_DEFAULTFS); - if(StringUtils.isNotBlank(defaultFSProp)){ - Map fsRelatedProps = PropertyUtils.getPrefixedProperties("fs."); - configuration.set(Constants.FS_DEFAULTFS,defaultFSProp); - fsRelatedProps.forEach((key, value) -> configuration.set(key, value)); - }else{ - logger.error("property:{} can not to be empty, please set!", Constants.FS_DEFAULTFS ); - throw new RuntimeException( - String.format("property: %s can not to be empty, please set!", Constants.FS_DEFAULTFS) - ); - } - }else{ - logger.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", Constants.FS_DEFAULTFS, defaultFS); - } + String defaultFS = configuration.get(Constants.FS_DEFAULTFS); + //first get key from core-site.xml hdfs-site.xml ,if null ,then try to get from properties file + // the default is the local file system + if (defaultFS.startsWith("file")) { + String defaultFSProp = PropertyUtils.getString(Constants.FS_DEFAULTFS); + if (StringUtils.isNotBlank(defaultFSProp)) { + Map fsRelatedProps = PropertyUtils.getPrefixedProperties("fs."); + configuration.set(Constants.FS_DEFAULTFS, defaultFSProp); + fsRelatedProps.forEach((key, value) -> configuration.set(key, value)); + } else { + logger.error("property:{} can not to be empty, please set!", Constants.FS_DEFAULTFS); + throw new RuntimeException( + String.format("property: %s can not to be empty, please set!", Constants.FS_DEFAULTFS) + ); + } + } else { + logger.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", Constants.FS_DEFAULTFS, defaultFS); + } - if (fs == null) { - if(StringUtils.isNotEmpty(hdfsUser)){ - UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser); - ugi.doAs(new PrivilegedExceptionAction() { - @Override - public Boolean run() throws Exception { - fs = FileSystem.get(configuration); - return true; - } - }); - }else{ - logger.warn("hdfs.root.user is not set value!"); - fs = FileSystem.get(configuration); - } + if (fs == null) { + if (StringUtils.isNotEmpty(hdfsUser)) { + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser); + ugi.doAs(new PrivilegedExceptionAction() { + @Override + public Boolean run() throws Exception { + fs = FileSystem.get(configuration); + return true; } - }else if (resUploadType == ResUploadType.S3){ - configuration.set(Constants.FS_DEFAULTFS, PropertyUtils.getString(Constants.FS_DEFAULTFS)); - configuration.set(Constants.FS_S3A_ENDPOINT, PropertyUtils.getString(Constants.FS_S3A_ENDPOINT)); - configuration.set(Constants.FS_S3A_ACCESS_KEY, PropertyUtils.getString(Constants.FS_S3A_ACCESS_KEY)); - configuration.set(Constants.FS_S3A_SECRET_KEY, PropertyUtils.getString(Constants.FS_S3A_SECRET_KEY)); - fs = FileSystem.get(configuration); - } - - - String rmHaIds = PropertyUtils.getString(Constants.YARN_RESOURCEMANAGER_HA_RM_IDS); - String appAddress = PropertyUtils.getString(Constants.YARN_APPLICATION_STATUS_ADDRESS); - if (!StringUtils.isEmpty(rmHaIds)) { - appAddress = getAppAddress(appAddress, rmHaIds); - logger.info("appAddress : {}", appAddress); - } - configuration.set(Constants.YARN_APPLICATION_STATUS_ADDRESS, appAddress); - } catch (Exception e) { - logger.error(e.getMessage(), e); + }); + } else { + logger.warn("hdfs.root.user is not set value!"); + fs = FileSystem.get(configuration); } - } + } else if (resUploadType == ResUploadType.S3) { + configuration.set(Constants.FS_DEFAULTFS, PropertyUtils.getString(Constants.FS_DEFAULTFS)); + configuration.set(Constants.FS_S3A_ENDPOINT, PropertyUtils.getString(Constants.FS_S3A_ENDPOINT)); + configuration.set(Constants.FS_S3A_ACCESS_KEY, PropertyUtils.getString(Constants.FS_S3A_ACCESS_KEY)); + configuration.set(Constants.FS_S3A_SECRET_KEY, PropertyUtils.getString(Constants.FS_S3A_SECRET_KEY)); + fs = FileSystem.get(configuration); + } + + + String rmHaIds = PropertyUtils.getString(Constants.YARN_RESOURCEMANAGER_HA_RM_IDS); + String appAddress = PropertyUtils.getString(Constants.YARN_APPLICATION_STATUS_ADDRESS); + if (!StringUtils.isEmpty(rmHaIds)) { + appAddress = getAppAddress(appAddress, rmHaIds); + logger.info("appAddress : {}", appAddress); } + configuration.set(Constants.YARN_APPLICATION_STATUS_ADDRESS, appAddress); + } catch (Exception e) { + logger.error(e.getMessage(), e); } } @@ -187,14 +191,14 @@ public class HadoopUtils implements Closeable { /** * cat file on hdfs * - * @param hdfsFilePath hdfs file path + * @param hdfsFilePath hdfs file path * @return byte[] byte array * @throws IOException errors */ public byte[] catFile(String hdfsFilePath) throws IOException { - if(StringUtils.isBlank(hdfsFilePath)){ - logger.error("hdfs file path:{} is blank",hdfsFilePath); + if (StringUtils.isBlank(hdfsFilePath)) { + logger.error("hdfs file path:{} is blank", hdfsFilePath); return new byte[0]; } @@ -203,29 +207,28 @@ public class HadoopUtils implements Closeable { } - /** * cat file on hdfs * - * @param hdfsFilePath hdfs file path - * @param skipLineNums skip line numbers - * @param limit read how many lines + * @param hdfsFilePath hdfs file path + * @param skipLineNums skip line numbers + * @param limit read how many lines * @return content of file * @throws IOException errors */ public List catFile(String hdfsFilePath, int skipLineNums, int limit) throws IOException { - if (StringUtils.isBlank(hdfsFilePath)){ - logger.error("hdfs file path:{} is blank",hdfsFilePath); + if (StringUtils.isBlank(hdfsFilePath)) { + logger.error("hdfs file path:{} is blank", hdfsFilePath); return Collections.emptyList(); } - try (FSDataInputStream in = fs.open(new Path(hdfsFilePath))){ + try (FSDataInputStream in = fs.open(new Path(hdfsFilePath))) { BufferedReader br = new BufferedReader(new InputStreamReader(in)); Stream stream = br.lines().skip(skipLineNums).limit(limit); return stream.collect(Collectors.toList()); } - + } /** @@ -258,17 +261,17 @@ public class HadoopUtils implements Closeable { /** * the src file is on the local disk. Add it to FS at * the given dst name. - - * @param srcFile local file - * @param dstHdfsPath destination hdfs path - * @param deleteSource whether to delete the src - * @param overwrite whether to overwrite an existing file + * + * @param srcFile local file + * @param dstHdfsPath destination hdfs path + * @param deleteSource whether to delete the src + * @param overwrite whether to overwrite an existing file * @return if success or not * @throws IOException errors */ public boolean copyLocalToHdfs(String srcFile, String dstHdfsPath, boolean deleteSource, boolean overwrite) throws IOException { Path srcPath = new Path(srcFile); - Path dstPath= new Path(dstHdfsPath); + Path dstPath = new Path(dstHdfsPath); fs.copyFromLocalFile(deleteSource, overwrite, srcPath, dstPath); @@ -278,10 +281,10 @@ public class HadoopUtils implements Closeable { /** * copy hdfs file to local * - * @param srcHdfsFilePath source hdfs file path - * @param dstFile destination file - * @param deleteSource delete source - * @param overwrite overwrite + * @param srcHdfsFilePath source hdfs file path + * @param dstFile destination file + * @param deleteSource delete source + * @param overwrite overwrite * @return result of copy hdfs file to local * @throws IOException errors */ @@ -299,7 +302,7 @@ public class HadoopUtils implements Closeable { } } - if(!dstPath.getParentFile().exists()){ + if (!dstPath.getParentFile().exists()) { dstPath.getParentFile().mkdirs(); } @@ -307,14 +310,13 @@ public class HadoopUtils implements Closeable { } /** - * * delete a file * * @param hdfsFilePath the path to delete. - * @param recursive if path is a directory and set to - * true, the directory is deleted else throws an exception. In - * case of a file the recursive can be set to either true or false. - * @return true if delete is successful else false. + * @param recursive if path is a directory and set to + * true, the directory is deleted else throws an exception. In + * case of a file the recursive can be set to either true or false. + * @return true if delete is successful else false. * @throws IOException errors */ public boolean delete(String hdfsFilePath, boolean recursive) throws IOException { @@ -339,7 +341,7 @@ public class HadoopUtils implements Closeable { * @return {@link FileStatus} file status * @throws Exception errors */ - public FileStatus[] listFileStatus(String filePath)throws Exception{ + public FileStatus[] listFileStatus(String filePath) throws Exception { try { return fs.listStatus(new Path(filePath)); } catch (IOException e) { @@ -351,10 +353,11 @@ public class HadoopUtils implements Closeable { /** * Renames Path src to Path dst. Can take place on local fs * or remote DFS. + * * @param src path to be renamed * @param dst new path after rename - * @throws IOException on failure * @return true if rename is successful + * @throws IOException on failure */ public boolean rename(String src, String dst) throws IOException { return fs.rename(new Path(src), new Path(dst)); @@ -400,7 +403,6 @@ public class HadoopUtils implements Closeable { } /** - * * @return data hdfs path */ public static String getHdfsDataBasePath() { @@ -427,11 +429,11 @@ public class HadoopUtils implements Closeable { * hdfs user dir * * @param tenantCode tenant code - * @param userId user id + * @param userId user id * @return hdfs resource dir */ - public static String getHdfsUserDir(String tenantCode,int userId) { - return String.format("%s/home/%d", getHdfsTenantDir(tenantCode),userId); + public static String getHdfsUserDir(String tenantCode, int userId) { + return String.format("%s/home/%d", getHdfsTenantDir(tenantCode), userId); } /** @@ -479,7 +481,7 @@ public class HadoopUtils implements Closeable { * getAppAddress * * @param appAddress app address - * @param rmHa resource manager ha + * @param rmHa resource manager ha * @return app address */ public static String getAppAddress(String appAddress, String rmHa) { diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index 5a4aa1441f..8391a9e2d6 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -91,4 +91,4 @@ yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx # If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s - +kerberos.expire.time=7 From 55cc1c7c5b03219ce46b799a39636ab93358688f Mon Sep 17 00:00:00 2001 From: t1mon <178317391@qq.com> Date: Mon, 16 Mar 2020 17:10:55 +0800 Subject: [PATCH 037/401] Replace Charset.forName("UTF-8") with StandardCharsets.UTF_8. (#2193) --- .../dolphinscheduler/api/service/ProcessInstanceService.java | 5 +++-- .../org/apache/dolphinscheduler/remote/utils/Constants.java | 3 ++- .../apache/dolphinscheduler/server/utils/ProcessUtils.java | 4 ++-- .../dolphinscheduler/server/worker/task/datax/DataxTask.java | 4 ++-- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java index 4f81d89505..3530c66ff6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.api.service; +import java.nio.charset.StandardCharsets; import org.apache.dolphinscheduler.api.dto.gantt.GanttDto; import org.apache.dolphinscheduler.api.dto.gantt.Task; import org.apache.dolphinscheduler.api.enums.Status; @@ -49,7 +50,6 @@ import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStreamReader; -import java.nio.charset.Charset; import java.text.ParseException; import java.util.*; import java.util.stream.Collectors; @@ -273,7 +273,8 @@ public class ProcessInstanceService extends BaseDAGService { return resultMap; } - BufferedReader br = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(log.getBytes(Charset.forName("utf8"))), Charset.forName("utf8"))); + BufferedReader br = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(log.getBytes( + StandardCharsets.UTF_8)), StandardCharsets.UTF_8)); String line; while ((line = br.readLine()) != null) { if(line.contains(DEPENDENT_SPLIT)){ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java index 5733b17790..f4791715b8 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.remote.utils; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; /** @@ -31,7 +32,7 @@ public class Constants { /** * charset */ - public static final Charset UTF8 = Charset.forName("UTF-8"); + public static final Charset UTF8 = StandardCharsets.UTF_8; /** * cpus diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java index e0c00c55d9..f3f8a79489 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.server.utils; +import java.nio.charset.StandardCharsets; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.LoggerUtils; @@ -29,7 +30,6 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; -import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; @@ -297,7 +297,7 @@ public class ProcessUtils { File f = new File(commandFile); if (!f.exists()) { - FileUtils.writeStringToFile(new File(commandFile), sb.toString(), Charset.forName("UTF-8")); + FileUtils.writeStringToFile(new File(commandFile), sb.toString(), StandardCharsets.UTF_8); } String runCmd = "sh " + commandFile; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java index ef941cd062..4b6fdefbed 100755 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.server.worker.task.datax; import java.io.File; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; @@ -208,7 +208,7 @@ public class DataxTask extends AbstractTask { logger.debug("datax job json : {}", root.toString()); // create datax json file - FileUtils.writeStringToFile(new File(fileName), root.toString(), Charset.forName("UTF-8")); + FileUtils.writeStringToFile(new File(fileName), root.toString(), StandardCharsets.UTF_8); return fileName; } From 585f833549a4659f8d2e2f9b34f19f9f63152a19 Mon Sep 17 00:00:00 2001 From: tswstarplanet Date: Mon, 16 Mar 2020 18:06:13 +0800 Subject: [PATCH 038/401] change static field to instance field (#2190) --- .../apache/dolphinscheduler/common/utils/HadoopUtils.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index e767911580..6c42704b47 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -64,8 +64,8 @@ public class HadoopUtils implements Closeable { } }); - private static Configuration configuration; - private static FileSystem fs; + private Configuration configuration; + private FileSystem fs; private static String hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER); @@ -99,7 +99,7 @@ public class HadoopUtils implements Closeable { /** * init hadoop configuration */ - private static void init() { + private void init() { try { configuration = new Configuration(); From 1d4cad79132377c2a34007e845fa61179a7a1d57 Mon Sep 17 00:00:00 2001 From: tswstarplanet Date: Mon, 16 Mar 2020 18:56:36 +0800 Subject: [PATCH 039/401] fix ResultSet not close and reformat code (#2183) * fix ResultSet not close * fix ResultSet not close * extract code of closing resource to a method * remove redundant if condition * modify e2e timeout --- .../dolphinscheduler/dao/TaskRecordDao.java | 134 ++++++++++-------- .../test/resources/config/config.properties | 8 +- 2 files changed, 79 insertions(+), 63 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java index f7ba1054a5..f89002ebdc 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java @@ -50,25 +50,28 @@ public class TaskRecordDao { static { try { conf = new PropertiesConfiguration(Constants.APPLICATION_PROPERTIES); - }catch (ConfigurationException e){ - logger.error("load configuration exception",e); + } catch (ConfigurationException e) { + logger.error("load configuration exception", e); System.exit(1); } } /** - * get task record flag + * get task record flag + * * @return whether startup taskrecord */ - public static boolean getTaskRecordFlag(){ - return conf.getBoolean(Constants.TASK_RECORD_FLAG); + public static boolean getTaskRecordFlag() { + return conf.getBoolean(Constants.TASK_RECORD_FLAG); } + /** * create connection + * * @return connection */ private static Connection getConn() { - if(!getTaskRecordFlag()){ + if (!getTaskRecordFlag()) { return null; } String driver = "com.mysql.jdbc.Driver"; @@ -90,101 +93,96 @@ public class TaskRecordDao { /** * generate where sql string + * * @param filterMap filterMap * @return sql string */ private static String getWhereString(Map filterMap) { - if(filterMap.size() ==0){ + if (filterMap.size() == 0) { return ""; } String result = " where 1=1 "; Object taskName = filterMap.get("taskName"); - if(taskName != null && StringUtils.isNotEmpty(taskName.toString())){ + if (taskName != null && StringUtils.isNotEmpty(taskName.toString())) { result += " and PROC_NAME like concat('%', '" + taskName.toString() + "', '%') "; } Object taskDate = filterMap.get("taskDate"); - if(taskDate != null && StringUtils.isNotEmpty(taskDate.toString())){ + if (taskDate != null && StringUtils.isNotEmpty(taskDate.toString())) { result += " and PROC_DATE='" + taskDate.toString() + "'"; } Object state = filterMap.get("state"); - if(state != null && StringUtils.isNotEmpty(state.toString())){ + if (state != null && StringUtils.isNotEmpty(state.toString())) { result += " and NOTE='" + state.toString() + "'"; } Object sourceTable = filterMap.get("sourceTable"); - if(sourceTable!= null && StringUtils.isNotEmpty(sourceTable.toString())){ - result += " and SOURCE_TAB like concat('%', '" + sourceTable.toString()+ "', '%')"; + if (sourceTable != null && StringUtils.isNotEmpty(sourceTable.toString())) { + result += " and SOURCE_TAB like concat('%', '" + sourceTable.toString() + "', '%')"; } Object targetTable = filterMap.get("targetTable"); - if(sourceTable!= null && StringUtils.isNotEmpty(targetTable.toString())){ - result += " and TARGET_TAB like concat('%', '"+ targetTable.toString()+"', '%') " ; + if (sourceTable != null && StringUtils.isNotEmpty(targetTable.toString())) { + result += " and TARGET_TAB like concat('%', '" + targetTable.toString() + "', '%') "; } Object start = filterMap.get("startTime"); - if(start != null && StringUtils.isNotEmpty(start.toString())){ + if (start != null && StringUtils.isNotEmpty(start.toString())) { result += " and STARTDATE>='" + start.toString() + "'"; } Object end = filterMap.get("endTime"); - if(end != null && StringUtils.isNotEmpty(end.toString())){ - result += " and ENDDATE>='" + end.toString()+ "'"; + if (end != null && StringUtils.isNotEmpty(end.toString())) { + result += " and ENDDATE>='" + end.toString() + "'"; } return result; } /** * count task record + * * @param filterMap filterMap - * @param table table + * @param table table * @return task record count */ - public static int countTaskRecord(Map filterMap, String table){ + public static int countTaskRecord(Map filterMap, String table) { int count = 0; Connection conn = null; PreparedStatement pstmt = null; + ResultSet rs = null; try { conn = getConn(); - if(conn == null){ + if (conn == null) { return count; } String sql = String.format("select count(1) as count from %s", table); sql += getWhereString(filterMap); pstmt = conn.prepareStatement(sql); - ResultSet rs = pstmt.executeQuery(); - while(rs.next()){ + rs = pstmt.executeQuery(); + while (rs.next()) { count = rs.getInt("count"); break; } } catch (SQLException e) { logger.error("Exception ", e); - }finally { - try { - if(pstmt != null) { - pstmt.close(); - } - if(conn != null){ - conn.close(); - } - } catch (SQLException e) { - logger.error("Exception ", e); - } + } finally { + closeResource(rs, pstmt, conn); } return count; } /** * query task record by filter map paging + * * @param filterMap filterMap - * @param table table + * @param table table * @return task record list */ - public static List queryAllTaskRecord(Map filterMap , String table) { + public static List queryAllTaskRecord(Map filterMap, String table) { String sql = String.format("select * from %s", table); sql += getWhereString(filterMap); @@ -194,9 +192,9 @@ public class TaskRecordDao { sql += String.format(" order by STARTDATE desc limit %d,%d", offset, pageSize); List recordList = new ArrayList<>(); - try{ + try { recordList = getQueryResult(sql); - }catch (Exception e){ + } catch (Exception e) { logger.error("Exception ", e); } return recordList; @@ -204,6 +202,7 @@ public class TaskRecordDao { /** * convert result set to task record + * * @param resultSet resultSet * @return task record * @throws SQLException if error throws SQLException @@ -232,6 +231,7 @@ public class TaskRecordDao { /** * query task list by select sql + * * @param selectSql select sql * @return task record list */ @@ -239,65 +239,81 @@ public class TaskRecordDao { List recordList = new ArrayList<>(); Connection conn = null; PreparedStatement pstmt = null; + ResultSet rs = null; try { conn = getConn(); - if(conn == null){ + if (conn == null) { return recordList; } pstmt = conn.prepareStatement(selectSql); - ResultSet rs = pstmt.executeQuery(); + rs = pstmt.executeQuery(); - while(rs.next()){ + while (rs.next()) { TaskRecord taskRecord = convertToTaskRecord(rs); recordList.add(taskRecord); } } catch (SQLException e) { logger.error("Exception ", e); - }finally { - try { - if(pstmt != null) { - pstmt.close(); - } - if(conn != null){ - conn.close(); - } - } catch (SQLException e) { - logger.error("Exception ", e); - } + } finally { + closeResource(rs, pstmt, conn); } return recordList; } /** * according to procname and procdate query task record + * * @param procName procName * @param procDate procDate * @return task record status */ - public static TaskRecordStatus getTaskRecordState(String procName,String procDate){ + public static TaskRecordStatus getTaskRecordState(String procName, String procDate) { String sql = String.format("SELECT * FROM eamp_hive_log_hd WHERE PROC_NAME='%s' and PROC_DATE like '%s'" - ,procName,procDate + "%"); + , procName, procDate + "%"); List taskRecordList = getQueryResult(sql); // contains no record and sql exception - if (CollectionUtils.isEmpty(taskRecordList)){ + if (CollectionUtils.isEmpty(taskRecordList)) { // exception return TaskRecordStatus.EXCEPTION; - }else if (taskRecordList.size() > 1){ + } else if (taskRecordList.size() > 1) { return TaskRecordStatus.EXCEPTION; - }else { + } else { TaskRecord taskRecord = taskRecordList.get(0); - if (taskRecord == null){ + if (taskRecord == null) { return TaskRecordStatus.EXCEPTION; } Long targetRowCount = taskRecord.getTargetRowCount(); - if (targetRowCount <= 0){ + if (targetRowCount <= 0) { return TaskRecordStatus.FAILURE; - }else { + } else { return TaskRecordStatus.SUCCESS; } } } + private static void closeResource(ResultSet rs, PreparedStatement pstmt, Connection conn) { + if (rs != null) { + try { + rs.close(); + } catch (SQLException e) { + logger.error("Exception ", e); + } + } + if (pstmt != null) { + try { + pstmt.close(); + } catch (SQLException e) { + logger.error("Exception ", e); + } + } + if (conn != null) { + try { + conn.close(); + } catch (SQLException e) { + logger.error("Exception ", e); + } + } + } } diff --git a/e2e/src/test/resources/config/config.properties b/e2e/src/test/resources/config/config.properties index 6a01234d01..352d9375be 100644 --- a/e2e/src/test/resources/config/config.properties +++ b/e2e/src/test/resources/config/config.properties @@ -27,13 +27,13 @@ PASSWORD=dolphinscheduler123 # driver path driver.chromeDriver=/usr/local/bin/chromedriver # implicitly wait(s) -driver.timeouts.implicitlyWait=10 +driver.timeouts.implicitlyWait=30 # show wait(s) -driver.timeouts.webDriverWait=10 +driver.timeouts.webDriverWait=30 # page load timeout(s) -driver.timeouts.pageLoadTimeout=10 +driver.timeouts.pageLoadTimeout=30 # JS wait timeouts(s) -driver.timeouts.setScriptTimeout=10 +driver.timeouts.setScriptTimeout=30 ############### redis ############## From 9fe04524ae0108a3d1b1a9c534231d1ee383b06d Mon Sep 17 00:00:00 2001 From: Rubik-W <39549317+Rubik-W@users.noreply.github.com> Date: Tue, 17 Mar 2020 11:10:55 +0800 Subject: [PATCH 040/401] fix: Api server startup fail (#2198) --- .../org/apache/dolphinscheduler/api/ApiApplicationServer.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java index 8376c2876d..5998ec5a4d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java @@ -25,7 +25,9 @@ import springfox.documentation.swagger2.annotations.EnableSwagger2; @SpringBootApplication @ServletComponentScan -@ComponentScan("org.apache.dolphinscheduler") +@ComponentScan({"org.apache.dolphinscheduler.api", + "org.apache.dolphinscheduler.dao", + "org.apache.dolphinscheduler.service"}) public class ApiApplicationServer extends SpringBootServletInitializer { public static void main(String[] args) { From 015a8ca2694952bcb6c023756a38eda7379fc18f Mon Sep 17 00:00:00 2001 From: "gabry.wu" Date: Tue, 17 Mar 2020 14:58:16 +0800 Subject: [PATCH 041/401] Adapting partial code(file name start with M #2) to the sonar cloud rule (#2179) * Adapting partial code(file name start with M #2) to the sonar cloud rule * remove unused import --- .../dolphinscheduler/dao/MonitorDBDao.java | 2 +- .../dao/datasource/MySQLDataSource.java | 2 +- .../dao/upgrade/MysqlUpgradeDao.java | 23 +++---------------- .../dao/utils/MysqlPerformance.java | 3 +-- .../server/master/MasterServer.java | 3 +-- .../master/runner/MasterTaskExecThread.java | 7 +++--- .../server/monitor/MonitorServer.java | 2 +- .../sources/MysqlSourceGenerator.java | 19 ++++++++------- .../targets/MysqlTargetGenerator.java | 11 ++++----- 9 files changed, 25 insertions(+), 47 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java index 51f60666d1..5ea5966238 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java @@ -61,7 +61,7 @@ public class MonitorDBDao { return new PostgrePerformance().getMonitorRecord(conn); } }catch (Exception e) { - logger.error("SQLException " + e); + logger.error("SQLException: {}", e.getMessage(), e); }finally { try { if (conn != null) { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java index 6e2fbe3dd8..969c17b54d 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java @@ -39,7 +39,7 @@ public class MySQLDataSource extends BaseDataSource { @Override public String getJdbcUrl() { String address = getAddress(); - if (address.lastIndexOf("/") != (address.length() - 1)) { + if (address.lastIndexOf('/') != (address.length() - 1)) { address += "/"; } String jdbcUrl = address + getDatabase(); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java index 58bd673fc5..a20a3acb95 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java @@ -29,15 +29,7 @@ import java.sql.SQLException; */ public class MysqlUpgradeDao extends UpgradeDao { - public static final Logger logger = LoggerFactory.getLogger(UpgradeDao.class); - - /** - * init - */ - @Override - protected void init() { - - } + public static final Logger logger = LoggerFactory.getLogger(MysqlUpgradeDao.class); /** * mysql upgrade dao holder @@ -69,12 +61,7 @@ public class MysqlUpgradeDao extends UpgradeDao { try { conn = dataSource.getConnection(); rs = conn.getMetaData().getTables(null, null, tableName, null); - if (rs.next()) { - return true; - } else { - return false; - } - + return rs.next(); } catch (SQLException e) { logger.error(e.getMessage(),e); throw new RuntimeException(e.getMessage(),e); @@ -96,11 +83,7 @@ public class MysqlUpgradeDao extends UpgradeDao { try { conn = dataSource.getConnection(); ResultSet rs = conn.getMetaData().getColumns(null,null,tableName,columnName); - if (rs.next()) { - return true; - } else { - return false; - } + return rs.next(); } catch (SQLException e) { logger.error(e.getMessage(),e); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java index 40d12ab36f..1e1ee78036 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java @@ -27,7 +27,6 @@ import java.util.Date; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.Flag; -import org.apache.dolphinscheduler.dao.MonitorDBDao; import org.apache.dolphinscheduler.dao.entity.MonitorRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +36,7 @@ import org.slf4j.LoggerFactory; */ public class MysqlPerformance extends BaseDBPerformance{ - private static Logger logger = LoggerFactory.getLogger(MonitorDBDao.class); + private static Logger logger = LoggerFactory.getLogger(MysqlPerformance.class); /** diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index 9512b1a1c2..bf58607262 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -243,7 +243,7 @@ public class MasterServer implements IStoppable { */ private Runnable heartBeatThread(){ logger.info("start master heart beat thread..."); - Runnable heartBeatThread = new Runnable() { + return new Runnable() { @Override public void run() { if(Stopper.isRunning()) { @@ -257,7 +257,6 @@ public class MasterServer implements IStoppable { } } }; - return heartBeatThread; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java index 3cdb8a0271..38b5419a92 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java @@ -64,7 +64,7 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { /** * whether already Killed,default false */ - private Boolean alreadyKilled = false; + private boolean alreadyKilled = false; /** * submit task instance and wait complete @@ -98,7 +98,7 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { logger.info("wait task: process id: {}, task id:{}, task name:{} complete", this.taskInstance.getProcessInstanceId(), this.taskInstance.getId(), this.taskInstance.getName()); // task time out - Boolean checkTimeout = false; + boolean checkTimeout = false; TaskTimeoutParameter taskTimeoutParameter = getTaskTimeoutParameter(); if(taskTimeoutParameter.getEnable()){ TaskTimeoutStrategy strategy = taskTimeoutParameter.getStrategy(); @@ -189,7 +189,6 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { private long getRemaintime(long timeoutSeconds) { Date startTime = taskInstance.getStartTime(); long usedTime = (System.currentTimeMillis() - startTime.getTime()) / 1000; - long remainTime = timeoutSeconds - usedTime; - return remainTime; + return timeoutSeconds - usedTime; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/MonitorServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/MonitorServer.java index ac549bc386..a1f43add6e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/MonitorServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/MonitorServer.java @@ -30,7 +30,7 @@ import org.springframework.context.annotation.ComponentScan; @ComponentScan("org.apache.dolphinscheduler") public class MonitorServer implements CommandLineRunner { - private static Integer ARGS_LENGTH = 4; + private static final Integer ARGS_LENGTH = 4; private static final Logger logger = LoggerFactory.getLogger(MonitorServer.class); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java index 050fef7cc7..404f10bf39 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java @@ -69,17 +69,16 @@ public class MysqlSourceGenerator implements ISourceGenerator { result.append(" --columns ").append(sourceMysqlParameter.getSrcColumns()); } - }else if(sourceMysqlParameter.getSrcQueryType() == QueryType.SQL.ordinal()){ - if(StringUtils.isNotEmpty(sourceMysqlParameter.getSrcQuerySql())){ - - String srcQuery = sourceMysqlParameter.getSrcQuerySql(); - if(srcQuery.toLowerCase().contains("where")){ - srcQuery += " AND "+"$CONDITIONS"; - }else{ - srcQuery += " WHERE $CONDITIONS"; - } - result.append(" --query \'"+srcQuery+"\'"); + }else if(sourceMysqlParameter.getSrcQueryType() == QueryType.SQL.ordinal() + && StringUtils.isNotEmpty(sourceMysqlParameter.getSrcQuerySql())){ + String srcQuery = sourceMysqlParameter.getSrcQuerySql(); + if(srcQuery.toLowerCase().contains("where")){ + srcQuery += " AND "+"$CONDITIONS"; + }else{ + srcQuery += " WHERE $CONDITIONS"; } + result.append(" --query \'"+srcQuery+"\'"); + } List mapColumnHive = sourceMysqlParameter.getMapColumnHive(); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java index 0733338812..0e33b176e5 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java @@ -75,12 +75,11 @@ public class MysqlTargetGenerator implements ITargetGenerator { result.append(" --lines-terminated-by '").append(targetMysqlParameter.getLinesTerminated()).append("'"); } - if(targetMysqlParameter.isUpdate()){ - if(StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateKey())&& - StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateMode())){ - result.append(" --update-key ").append(targetMysqlParameter.getTargetUpdateKey()) - .append(" --update-mode ").append(targetMysqlParameter.getTargetUpdateMode()); - } + if(targetMysqlParameter.isUpdate() + && StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateKey()) + && StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateMode())){ + result.append(" --update-key ").append(targetMysqlParameter.getTargetUpdateKey()) + .append(" --update-mode ").append(targetMysqlParameter.getTargetUpdateMode()); } } } From 9c754378f3f2d22ad9b67e90db68897e50bba9d9 Mon Sep 17 00:00:00 2001 From: t1mon <178317391@qq.com> Date: Tue, 17 Mar 2020 15:01:42 +0800 Subject: [PATCH 042/401] Optimize PropertyUtils instantiation. (#2199) * Optimize PropertyUtils instantiation. * Fix info error. --- .../dolphinscheduler/common/utils/PropertyUtils.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index 5c1011cd8b..ca928c94d5 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -43,13 +43,11 @@ public class PropertyUtils { private static final Properties properties = new Properties(); - private static final PropertyUtils propertyUtils = new PropertyUtils(); - - private PropertyUtils(){ - init(); + private PropertyUtils() { + throw new IllegalStateException("PropertyUtils class"); } - private void init(){ + static { String[] propertyFiles = new String[]{COMMON_PROPERTIES_PATH}; for (String fileName : propertyFiles) { InputStream fis = null; From 7c54496f5aaaa55f69bf6f1156683bbe3182a654 Mon Sep 17 00:00:00 2001 From: xingchun-chen <55787491+xingchun-chen@users.noreply.github.com> Date: Tue, 17 Mar 2020 17:57:12 +0800 Subject: [PATCH 043/401] =?UTF-8?q?e2e=20add=20the=20case=20of=20deleting?= =?UTF-8?q?=20user=20and=20tenant=20=C2=A0=20(#2210)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add e2e --- .../dolphinscheduler/base/BaseDriver.java | 8 +- .../dolphinscheduler/base/BaseTest.java | 17 ++-- .../dolphinscheduler/data/LoginData.java | 2 +- .../data/project/CreatWorkflowData.java | 2 +- ...rojectData.java => CreateProjectData.java} | 4 +- .../data/project/CreateWorkflowData.java | 73 +++++++++++++++ .../data/security/TenantManageData.java | 10 +-- .../data/security/UserManageData.java | 5 +- .../project/CreateWorkflowLocator.java | 69 ++++++++++++-- .../locator/security/TenantManageLocator.java | 8 +- .../locator/security/UserManageLocator.java | 3 + .../dolphinscheduler/page/LoginPage.java | 14 ++- .../page/project/CreateProjectPage.java | 11 +-- .../page/project/CreateWorkflowPage.java | 89 +++++++++++++++---- .../page/security/TenantManagePage.java | 26 ++++-- .../page/security/UserManagePage.java | 25 ++++-- .../dolphinscheduler/testcase/LoginTest.java | 35 +++----- .../testcase/deleteData/DeleteTenantTest.java | 37 ++++++++ .../testcase/deleteData/DeleteUserTest.java | 36 ++++++++ .../testcase/project/CreateProjectTest.java | 7 +- .../testcase/project/CreateWorkflowTest.java | 10 ++- .../testcase/security/TenantManageTest.java | 11 ++- .../testcase/security/UserManageTest.java | 8 +- e2e/suite.xml | 18 ---- e2e/testng.xml | 28 +++--- 25 files changed, 415 insertions(+), 141 deletions(-) rename e2e/src/test/java/org/apache/dolphinscheduler/data/project/{CreatProjectData.java => CreateProjectData.java} (91%) create mode 100644 e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreateWorkflowData.java create mode 100644 e2e/src/test/java/org/apache/dolphinscheduler/testcase/deleteData/DeleteTenantTest.java create mode 100644 e2e/src/test/java/org/apache/dolphinscheduler/testcase/deleteData/DeleteUserTest.java delete mode 100644 e2e/suite.xml diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseDriver.java b/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseDriver.java index 7d3ab9b837..044e28f082 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseDriver.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseDriver.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.base; import org.apache.dolphinscheduler.constant.TestConstant; import org.apache.dolphinscheduler.util.PropertiesReader; +import org.openqa.selenium.Cookie; import org.openqa.selenium.JavascriptExecutor; import org.openqa.selenium.WebDriver; import org.openqa.selenium.chrome.ChromeDriver; @@ -35,7 +36,7 @@ public class BaseDriver { /** * driver */ - private WebDriver driver; + private static WebDriver driver; /** * chrome driver path @@ -88,6 +89,7 @@ public class BaseDriver { chromeOptions.setPageLoadStrategy(PageLoadStrategy.NONE); chromeOptions.addArguments("--no-sandbox"); chromeOptions.addArguments("--disable-dev-shm-usage"); + //Browser client running requires annotation --headless chromeOptions.addArguments("--headless"); chromeOptions.addArguments("--disable-gpu"); chromeOptions.addArguments("--whitelisted-ips"); @@ -120,7 +122,7 @@ public class BaseDriver { * * @return driver */ - public WebDriver getDriver() { + public static WebDriver getDriver() { return driver; } @@ -141,7 +143,7 @@ public class BaseDriver { public void closeBrowser() throws InterruptedException { // JS Show a pop-up box to indicate the end of the test Thread.sleep(TestConstant.ONE_THOUSANG); - ((JavascriptExecutor) driver).executeScript("alert('Test completed, browser closes after 3s')"); +// ((JavascriptExecutor) driver).executeScript("alert('Test completed, browser closes after 3s')"); Thread.sleep(TestConstant.THREE_THOUSANG); if (driver != null) { driver.quit(); diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseTest.java index c12c19fc1d..88012e9f42 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseTest.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/base/BaseTest.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.base; -import org.apache.dolphinscheduler.page.LoginPage; import org.apache.dolphinscheduler.util.PropertiesReader; import org.openqa.selenium.WebDriver; import org.testng.annotations.*; @@ -28,6 +27,7 @@ import java.util.Properties; /** * base test class */ +@Test(groups={"functionTests"}) public class BaseTest { /** * properties @@ -43,7 +43,7 @@ public class BaseTest { /** * driver */ - public WebDriver driver; + public static WebDriver driver; /** * Executed before executing a test suite  @@ -54,7 +54,7 @@ public class BaseTest { */ @BeforeSuite(alwaysRun = true) @Parameters({"propertiesPath"}) - public void beforeSuite(@Optional("src/test/resources/config/config.properties") String propertiesPath) throws IOException { + public void beforeSuite(@Optional("src/test/resources/config/config.properties") String propertiesPath) throws Exception { // read properties properties = PropertiesReader.readProperties(propertiesPath); } @@ -70,14 +70,13 @@ public class BaseTest { driver = baseDriver.getDriver(); } + + /** * Executed before executing a class method in a test case */ @BeforeClass(alwaysRun = true) - public void setUp() throws IOException, InterruptedException { - LoginPage loginPage = new LoginPage(driver); - loginPage.jumpPage(); - loginPage.login(); + public void setUp() throws Exception { } @@ -85,7 +84,7 @@ public class BaseTest { * Execute after executing a class method in a test case */ @AfterClass(alwaysRun = true) - public void afterClass() { + public void afterClass() throws InterruptedException { // logout } @@ -102,6 +101,6 @@ public class BaseTest { * Execute after executing a testsuite */ @AfterSuite(alwaysRun = true) - public void afterSuite() { + public void afterSuite() throws InterruptedException { } } \ No newline at end of file diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/LoginData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/LoginData.java index 532849565c..e56df5a448 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/data/LoginData.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/LoginData.java @@ -39,5 +39,5 @@ public class LoginData { */ public static final String PASSWORD = PropertiesReader.getKey("PASSWORD"); - public static final String TENANT = "Tenant Manage - DolphinScheduler"; + public static final String TENANT = "租户管理 - DolphinScheduler"; } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreatWorkflowData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreatWorkflowData.java index 765a54f406..70f4238610 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreatWorkflowData.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreatWorkflowData.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.data.project; public class CreatWorkflowData { //input shell task name - public static final String SHELL_TASK_NAME = "shell task description test"; + public static final String SHELL_TASK_NAME = "shell task description test1"; //input shell task description public static final String SHELL_TASK_DESCRIPTION = "shell task description test"; diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreatProjectData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreateProjectData.java similarity index 91% rename from e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreatProjectData.java rename to e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreateProjectData.java index 8f6c9c8e97..57d79e87d7 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreatProjectData.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreateProjectData.java @@ -16,9 +16,9 @@ */ package org.apache.dolphinscheduler.data.project; -public class CreatProjectData { +public class CreateProjectData { // create project name - public static final String PROJECT_NAME = "selenium_project"; + public static final String PROJECT_NAME = "selenium_project_3"; // create project description public static final String DESCRIPTION = "test create project description"; // project page title diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreateWorkflowData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreateWorkflowData.java new file mode 100644 index 0000000000..31b5bdd1e0 --- /dev/null +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/project/CreateWorkflowData.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.data.project; + +public class CreateWorkflowData { + /** + * create workflow data + */ + //input shell task name + public static final String SHELL_TASK_NAME = "shell_task_selenium_5"; + + //input shell task description + public static final String SHELL_TASK_DESCRIPTION = "shell task description test"; + + //input timeout + public static final String INPUT_TIMEOUT = "60"; + + //input shell script + public static final String SHELL_SCRIPT = "echo 1111111"; + + //input custom parameters + public static final String INPUT_CUSTOM_PARAMETERS = "selenium_parameter"; + + //input custom parameters value + public static final String INPUT_CUSTOM_PARAMETERS_VALUE = "selenium_parameter_123"; + + //input add custom parameters + public static final String INPUT_ADD_CUSTOM_PARAMETERS = "selenium_parameter_delete"; + + //input add custom parameters value + public static final String INPUT_ADD_CUSTOM_PARAMETERS_VALUE = "selenium_parameter_delete_456"; + + //create workflow title + public static final String WORKFLOW_TITLE = "创建流程定义 - DolphinScheduler"; + + /** + * save workflow data + */ + //input workflow name + public static final String INPUT_WORKFLOW_NAME = "selenium_shell_1"; + + //input workflow description + public static final String INPUT_WORKFLOW_DESCRIPTION = "test selenium_shell_1 description"; + + //input workflow timeout + public static final String INPUT_WORKFLOW_TIMEOUT = "30"; + + //input workflow global parameters + public static final String INPUT_WORKFLOW_GLOBAL_PARAMETERS = "selenium_global_parameters_1"; + + //input workflow global parameters value + public static final String INPUT_WORKFLOW_GLOBAL_PARAMETERS_VALUES = "selenium_global_parameters_value_1"; + + //input to add workflow global parameters + public static final String INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS = "selenium_global_parameters_2"; + + //input to add workflow global parameters value + public static final String INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS_VALUES = "selenium_global_parameters_value_2"; +} diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/security/TenantManageData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/security/TenantManageData.java index e6f6ee6b86..100351b600 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/data/security/TenantManageData.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/security/TenantManageData.java @@ -20,18 +20,12 @@ */ package org.apache.dolphinscheduler.data.security; -import org.apache.dolphinscheduler.data.LoginData; public class TenantManageData { - /** - * Tenant URL - */ - public static final String TENANAT_URL = LoginData.URL + "/ui/#/security/tenant"; - /** * Tenant Code */ - public static final String TENANAT_CODE = "dolphinscheduler_tenant_code2"; + public static final String TENANAT_CODE = "dolphinscheduler_tenant_code15"; /** * Tenant Name @@ -48,7 +42,7 @@ public class TenantManageData { */ public static final String DESCRIPTION = "creat tenant test"; - public static final String TENANAT_MANAGE = "Tenant Manage - DolphinScheduler"; + public static final String TENANAT_MANAGE = "租户管理 - DolphinScheduler"; diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/data/security/UserManageData.java b/e2e/src/test/java/org/apache/dolphinscheduler/data/security/UserManageData.java index 03c985fd81..6fcd923c53 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/data/security/UserManageData.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/data/security/UserManageData.java @@ -16,12 +16,9 @@ */ package org.apache.dolphinscheduler.data.security; -import org.apache.dolphinscheduler.data.LoginData; public class UserManageData { - public static final String USER_URL = LoginData.URL + "/ui/#/security/users"; - - public static final String USERNAME = "selenium111"; + public static final String USERNAME = "selenium_5"; public static final String PASSWORD = "123456qwe"; diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateWorkflowLocator.java b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateWorkflowLocator.java index f063d6ef61..25defcf53f 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateWorkflowLocator.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateWorkflowLocator.java @@ -20,12 +20,20 @@ package org.apache.dolphinscheduler.locator.project; import org.openqa.selenium.By; public class CreateWorkflowLocator { + + /** + * jump workflow define page + */ + // click project name public static final By CLICK_PROJECT_NAME = By.xpath("//span/a"); // click workflow define public static final By CLICK_WORKFLOW_DEFINE = By.xpath("//li/span"); + /** + * create workflow + */ // click create workflow button public static final By CLICK_CREATE_WORKFLOW_BUTTON = By.xpath("//button/span"); @@ -96,15 +104,15 @@ public class CreateWorkflowLocator { public static final By INPUT_CUSTOM_PARAMETERS = By.xpath("//div[2]/div/div/div/div/div/input"); //input custom parameters value - public static final By INPUT_CUSTOM_PARAMETERS_VALUE = By.xpath("//div[2]/input"); + public static final By INPUT_CUSTOM_PARAMETERS_VALUE = By.xpath("//div[2]/div/div/div/div[1]/div[2]/input"); - //click add custom parameters - public static final By CLICK_ADD_CUSTOM_PARAMETERS = By.xpath("//span[2]/a/em"); + //click to add custom parameters + public static final By CLICK_ADD_CUSTOM_PARAMETERS = By.xpath("//div[2]/div/div/div/div/span[2]/a/em"); - //input add custom parameters - public static final By INPUT_ADD_CUSTOM_PARAMETERS = By.xpath("//div[2]/div/div/div/div[2]/div/input"); + //input to add custom parameters + public static final By INPUT_ADD_CUSTOM_PARAMETERS = By.xpath("//div[2]/div/div/div/div[2]/div[1]/input"); - //input add custom parameters value + //input to add custom parameters value public static final By INPUT_ADD_CUSTOM_PARAMETERS_VALUE = By.xpath("//div[2]/div[2]/input"); //delete custom parameters @@ -112,4 +120,53 @@ public class CreateWorkflowLocator { //click submit button public static final By CLICK_SUBMIT_BUTTON = By.xpath("//button[2]/span"); + + + /** + * save workflow + */ + //click save workflow button + public static final By CLICK_SAVE_WORKFLOW_BUTTON = By.xpath("//button/span"); + + //input workflow name + public static final By INPUT_WORKFLOW_NAME = By.xpath("//input"); + + //input workflow description + public static final By INPUT_WORKFLOW_DESCRIPTION = By.xpath("//textarea"); + + //select tenant + public static final By CLICK_TENANT = By.xpath("//div[4]/div/div/div/input"); + public static final By SELECT_TENANT = By.xpath("//div[2]/div/div/div/ul/li/span"); + + + //click workflow timeout alarm + public static final By CLICK_WORKFLOW_TIMEOUT_ALARM = By.xpath("//span[2]/span/span"); + + //input workflow timeout + public static final By INPUT_WORKFLOW_TIMEOUT = By.xpath("//span/div/input"); + + //click workflow global parameters + public static final By CLICK_WORKFLOW_GLOBAL_PARAMETERS = By.xpath("//span/a/em"); + + //input workflow global parameters + public static final By INPUT_WORKFLOW_GLOBAL_PARAMETERS = By.xpath("//div[8]/div/div/div/div/input"); + + //input workflow global parameters value + public static final By INPUT_WORKFLOW_GLOBAL_PARAMETERS_VALUES = By.xpath("//div[2]/input"); + + //click to add workflow global parameters + public static final By CLICK_ADD_WORKFLOW_GLOBAL_PARAMETERS = By.xpath("//span[2]/a/em"); + + //input to add workflow global parameters + public static final By INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS = By.xpath("//div[8]/div/div/div[2]/div/input"); + + //input to add workflow global parameters value + public static final By INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS_VALUES = By.xpath("//div[2]/div[2]/input"); + + //delete workflow global parameters value + public static final By CLICK_DELETE_WORKFLOW_GLOBAL_PARAMETERS = By.xpath("//div[2]/span/a/em"); + + //click add button + public static final By CLICK_ADD_BUTTON = By.xpath("//button[2]/span"); + } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/TenantManageLocator.java b/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/TenantManageLocator.java index 7d9c8a57ce..d95265cf0b 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/TenantManageLocator.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/TenantManageLocator.java @@ -19,7 +19,9 @@ package org.apache.dolphinscheduler.locator.security; import org.openqa.selenium.By; public class TenantManageLocator{ - public static final By CREATE_TENANT_BUTTON = By.xpath("//button[@class='ans-btn ans-btn-ghost ans-btn-small']"); + public static final By TENANT_MANAGE = By.xpath("//div[2]/div/a/div/a/span"); + + public static final By CREATE_TENANT_BUTTON = By.xpath("//button/span"); public static final By TENANT_INPUT_CODE = By.xpath("//div[2]/div/div/div[2]/div/input"); @@ -30,4 +32,8 @@ public class TenantManageLocator{ public static final By DESCRIPTION = By.xpath("//textarea"); public static final By SUBMIT_BUTTON = By.xpath("//div[3]/button[2]/span"); + + public static final By DELETE_TENANT_BUTTON = By.xpath("//div[3]/div[1]/div/table/tr[2]/td[8]/span/button"); + + public static final By CONFIRM_DELETE_TENANT_BUTTON = By.xpath("//div[2]/div/button[2]/span"); } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/UserManageLocator.java b/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/UserManageLocator.java index 0d84692cb2..b5ecccc67d 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/UserManageLocator.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/locator/security/UserManageLocator.java @@ -42,4 +42,7 @@ public class UserManageLocator { public static final By SUBMIT = By.xpath("//div[3]/button[2]/span"); + public static final By DELETE_USER_BUTTON = By.xpath("//span[2]/button/i"); + + public static final By CONFIRM_DELETE_USER_BUTTON = By.xpath("//div[2]/div/button[2]/span"); } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/LoginPage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/LoginPage.java index cd6b318651..14970bead5 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/LoginPage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/LoginPage.java @@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.constant.TestConstant; import org.apache.dolphinscheduler.data.LoginData; import org.apache.dolphinscheduler.locator.LoginLocator; import org.apache.dolphinscheduler.util.RedisUtil; +import org.openqa.selenium.Cookie; import org.openqa.selenium.WebDriver; @@ -38,18 +39,25 @@ public class LoginPage extends PageCommon { /** * jump page */ - public void jumpPage() { - System.out.println("jump login page"); + public void jumpPageEnlish() { + System.out.println("jump to English login page"); super.jumpPage(LoginData.URL); + Cookie cookie = new Cookie("language", "en_US", "/", null); + driver.manage().addCookie(cookie); } + public void jumpPageChinese() { + super.jumpPage(LoginData.URL); + Cookie cookie = new Cookie("language", "zh_CN", "/", null); + + driver.manage().addCookie(cookie); + } /** * login * * @return Whether to enter the specified page after searching */ public boolean login() throws InterruptedException { - System.out.println("LoginPage"); // login data sendInput(LoginLocator.LOGIN_INPUT_USER, LoginData.USER); sendInput(LoginLocator.LOGIN_INPUT_PASSWORD, LoginData.PASSWORD); diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateProjectPage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateProjectPage.java index 8dd1010a82..d8e1ef655d 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateProjectPage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateProjectPage.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.page.project; import org.apache.dolphinscheduler.common.PageCommon; import org.apache.dolphinscheduler.constant.TestConstant; -import org.apache.dolphinscheduler.data.project.CreatProjectData; +import org.apache.dolphinscheduler.data.project.CreateProjectData; import org.apache.dolphinscheduler.locator.project.CreateProjectLocator; import org.openqa.selenium.WebDriver; @@ -26,6 +26,7 @@ public class CreateProjectPage extends PageCommon { public CreateProjectPage(WebDriver driver) { super(driver); } + /** * jump page */ @@ -36,7 +37,7 @@ public class CreateProjectPage extends PageCommon { } /** - * creatTenant + * createTenant * * @return Whether to enter the specified page after creat tenant */ @@ -46,13 +47,13 @@ public class CreateProjectPage extends PageCommon { Thread.sleep(TestConstant.ONE_THOUSANG); // input create project data - sendInput(CreateProjectLocator.PROJECT_NAME, CreatProjectData.PROJECT_NAME); - sendInput(CreateProjectLocator.PROJECT_DESCRIPTION, CreatProjectData.DESCRIPTION); + sendInput(CreateProjectLocator.PROJECT_NAME, CreateProjectData.PROJECT_NAME); + sendInput(CreateProjectLocator.PROJECT_DESCRIPTION, CreateProjectData.DESCRIPTION); // click submit button clickButton(CreateProjectLocator.SUBMIT_BUTTON); // Whether to enter the specified page after submit - return ifTitleContains(CreatProjectData.PROJECT_TITLE); + return ifTitleContains(CreateProjectData.PROJECT_TITLE); } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateWorkflowPage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateWorkflowPage.java index aeec83cd19..d115124d8f 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateWorkflowPage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateWorkflowPage.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.page.project; import org.apache.dolphinscheduler.common.PageCommon; import org.apache.dolphinscheduler.constant.TestConstant; -import org.apache.dolphinscheduler.data.project.CreatWorkflowData; +import org.apache.dolphinscheduler.data.project.CreateWorkflowData; import org.apache.dolphinscheduler.locator.project.CreateWorkflowLocator; import org.openqa.selenium.WebDriver; @@ -26,29 +26,33 @@ public class CreateWorkflowPage extends PageCommon { public CreateWorkflowPage(WebDriver driver) { super(driver); } + /** - * jump page + * jump create workflow page */ - public boolean createWorkflow() throws InterruptedException { + System.out.println("Click on the project name to jump to the project homepage"); // click project name clickElement(CreateWorkflowLocator.CLICK_PROJECT_NAME); Thread.sleep(TestConstant.ONE_THOUSANG); + System.out.println("Click on workflow define"); // click workflow define clickElement(CreateWorkflowLocator.CLICK_WORKFLOW_DEFINE); Thread.sleep(TestConstant.ONE_THOUSANG); + System.out.println("Click create workflow button"); // click create workflow button clickElement(CreateWorkflowLocator.CLICK_CREATE_WORKFLOW_BUTTON); Thread.sleep(TestConstant.ONE_THOUSANG); + System.out.println("drag shell task"); //drag shell_task dragAndDrop(CreateWorkflowLocator.MOUSE_DOWN_AT_SHELL,CreateWorkflowLocator.MOUSE_MOVE_SHELL_AT_DAG); //input shell task _name - sendInput(CreateWorkflowLocator.INPUT_SHELL_TASK_NAME , CreatWorkflowData.SHELL_TASK_NAME); + sendInput(CreateWorkflowLocator.INPUT_SHELL_TASK_NAME , CreateWorkflowData.SHELL_TASK_NAME); //click stop run type clickElement(CreateWorkflowLocator.CLICK_STOP_RUN_TYPE); @@ -57,7 +61,7 @@ public class CreateWorkflowPage extends PageCommon { clickElement(CreateWorkflowLocator.CLICK_NORMAL_RUN_TYPE); //input shell task description - sendInput(CreateWorkflowLocator.INPUT_SHELL_TASK_DESCRIPTION , CreatWorkflowData.SHELL_TASK_DESCRIPTION); + sendInput(CreateWorkflowLocator.INPUT_SHELL_TASK_DESCRIPTION , CreateWorkflowData.SHELL_TASK_DESCRIPTION); //select task priority clickElement(CreateWorkflowLocator.CLICK_TASK_PRIORITY); @@ -73,19 +77,15 @@ public class CreateWorkflowPage extends PageCommon { //select failed retry interval clickElement(CreateWorkflowLocator.SELECT_FAIL_RETRIES_INTERVAL); - //click timeout alarm clickElement(CreateWorkflowLocator.CLICK_TIMEOUT_ALARM); - //select timeout fail clickElement(CreateWorkflowLocator.SELECT_TIMEOUT_FAIL); - //cancel timeout alarm clickElement(CreateWorkflowLocator.CANCEL_TIMEOUT_ALARM); - //select timeout alarm clickElement(CreateWorkflowLocator.SELECT_TIMEOUT_ALARM); @@ -94,10 +94,10 @@ public class CreateWorkflowPage extends PageCommon { clearInput(CreateWorkflowLocator.SELECT_TIMEOUT); //input timeout - sendInput(CreateWorkflowLocator.SELECT_TIMEOUT,CreatWorkflowData.INPUT_TIMEOUT); + sendInput(CreateWorkflowLocator.SELECT_TIMEOUT, CreateWorkflowData.INPUT_TIMEOUT); //click codeMirror and input script - inputCodeMirror(CreateWorkflowLocator.CLICK_CODE_MIRROR, CreateWorkflowLocator.INPUT_SCRIPT,CreatWorkflowData.SHELL_SCRIPT); + inputCodeMirror(CreateWorkflowLocator.CLICK_CODE_MIRROR, CreateWorkflowLocator.INPUT_SCRIPT, CreateWorkflowData.SHELL_SCRIPT); scrollToElementBottom(); Thread.sleep(TestConstant.ONE_THOUSANG); @@ -105,10 +105,10 @@ public class CreateWorkflowPage extends PageCommon { clickElement(CreateWorkflowLocator.CLICK_CUSTOM_PARAMETERS); //input custom parameters - sendInput(CreateWorkflowLocator.INPUT_CUSTOM_PARAMETERS, CreatWorkflowData.INPUT_CUSTOM_PARAMETERS); + sendInput(CreateWorkflowLocator.INPUT_CUSTOM_PARAMETERS, CreateWorkflowData.INPUT_CUSTOM_PARAMETERS); //input custom parameters value - sendInput(CreateWorkflowLocator.INPUT_CUSTOM_PARAMETERS_VALUE, CreatWorkflowData.INPUT_CUSTOM_PARAMETERS_VALUE); + sendInput(CreateWorkflowLocator.INPUT_CUSTOM_PARAMETERS_VALUE, CreateWorkflowData.INPUT_CUSTOM_PARAMETERS_VALUE); //click add custom parameters clickElement(CreateWorkflowLocator.CLICK_ADD_CUSTOM_PARAMETERS); @@ -117,10 +117,10 @@ public class CreateWorkflowPage extends PageCommon { Thread.sleep(TestConstant.ONE_THOUSANG); //input add custom parameters - sendInput(CreateWorkflowLocator.INPUT_ADD_CUSTOM_PARAMETERS,CreatWorkflowData.INPUT_ADD_CUSTOM_PARAMETERS); + sendInput(CreateWorkflowLocator.INPUT_ADD_CUSTOM_PARAMETERS, CreateWorkflowData.INPUT_ADD_CUSTOM_PARAMETERS); //input add custom parameters value - sendInput(CreateWorkflowLocator.INPUT_ADD_CUSTOM_PARAMETERS_VALUE,CreatWorkflowData.INPUT_ADD_CUSTOM_PARAMETERS_VALUE); + sendInput(CreateWorkflowLocator.INPUT_ADD_CUSTOM_PARAMETERS_VALUE, CreateWorkflowData.INPUT_ADD_CUSTOM_PARAMETERS_VALUE); //click delete custom parameters clickElement(CreateWorkflowLocator.CLICK_DELETE_CUSTOM_PARAMETERS); @@ -129,9 +129,64 @@ public class CreateWorkflowPage extends PageCommon { //click submit button clickElement(CreateWorkflowLocator.CLICK_SUBMIT_BUTTON); Thread.sleep(TestConstant.ONE_THOUSANG); - + System.out.println("Task node set up successfully"); + System.out.println("move to Dag Element "); moveToDragElement(CreateWorkflowLocator.MOUSE_MOVE_SHELL_AT_DAG,-300,-100); - return ifTitleContains(CreatWorkflowData.WORKFLOW_TITLE); + return ifTitleContains(CreateWorkflowData.WORKFLOW_TITLE); + } + + /** + * save workflow + */ + public boolean saveWorkflow() throws InterruptedException { + System.out.println("start to save workflow "); + + //click save workflow button + clickElement(CreateWorkflowLocator.CLICK_SAVE_WORKFLOW_BUTTON); + + //input workflow name + sendInput(CreateWorkflowLocator.INPUT_WORKFLOW_NAME, CreateWorkflowData.INPUT_WORKFLOW_NAME); + + //input workflow description + sendInput(CreateWorkflowLocator.INPUT_WORKFLOW_DESCRIPTION, CreateWorkflowData.INPUT_WORKFLOW_DESCRIPTION); + + //select tenant + clickElement(CreateWorkflowLocator.CLICK_TENANT); + clickElement(CreateWorkflowLocator.SELECT_TENANT); + + //click workflow timeout alarm + clickElement(CreateWorkflowLocator.CLICK_WORKFLOW_TIMEOUT_ALARM); + clearInput(CreateWorkflowLocator.INPUT_WORKFLOW_TIMEOUT); + + //input workflow timeout + sendInput(CreateWorkflowLocator.INPUT_WORKFLOW_TIMEOUT, CreateWorkflowData.INPUT_WORKFLOW_TIMEOUT); + + //click workflow global parameters + clickElement(CreateWorkflowLocator.CLICK_WORKFLOW_GLOBAL_PARAMETERS); + + //input workflow global parameters + sendInput(CreateWorkflowLocator.INPUT_WORKFLOW_GLOBAL_PARAMETERS, CreateWorkflowData.INPUT_WORKFLOW_GLOBAL_PARAMETERS); + + //input workflow global parameters value + sendInput(CreateWorkflowLocator.INPUT_WORKFLOW_GLOBAL_PARAMETERS_VALUES, CreateWorkflowData.INPUT_WORKFLOW_GLOBAL_PARAMETERS_VALUES); + + //click to add workflow global parameters + clickElement(CreateWorkflowLocator.CLICK_ADD_WORKFLOW_GLOBAL_PARAMETERS); + + //input to add workflow global parameters + sendInput(CreateWorkflowLocator.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS, CreateWorkflowData.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS); + + //input to add workflow global parameters value + sendInput(CreateWorkflowLocator.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS_VALUES, CreateWorkflowData.INPUT_ADD_WORKFLOW_GLOBAL_PARAMETERS_VALUES); + + //delete workflow global parameters value + clickElement(CreateWorkflowLocator.CLICK_DELETE_WORKFLOW_GLOBAL_PARAMETERS); + Thread.sleep(TestConstant.ONE_THOUSANG); + + //click add button + clickElement(CreateWorkflowLocator.CLICK_ADD_BUTTON); + System.out.println("submit workflow"); + return ifTitleContains(CreateWorkflowData.WORKFLOW_TITLE); } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/security/TenantManagePage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/security/TenantManagePage.java index 4c88f6575e..618e1bc213 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/security/TenantManagePage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/security/TenantManagePage.java @@ -34,15 +34,6 @@ public class TenantManagePage extends PageCommon { super(driver); } - - /** - * jump page - */ - public void jumpPage() { - System.out.println("jump tenant page"); - super.jumpPage(TenantManageData.TENANAT_URL); - } - /** * createTenant * @@ -50,6 +41,7 @@ public class TenantManagePage extends PageCommon { */ public boolean createTenant() throws InterruptedException { Thread.sleep(TestConstant.ONE_THOUSANG); + clickButton(TenantManageLocator.TENANT_MANAGE); //create tenant clickButton(TenantManageLocator.CREATE_TENANT_BUTTON); @@ -66,4 +58,20 @@ public class TenantManagePage extends PageCommon { // Whether to enter the specified page after submit return ifTitleContains(TenantManageData.TENANAT_MANAGE); } + + public boolean deleteTenant() throws InterruptedException { + Thread.sleep(TestConstant.ONE_THOUSANG); + clickButton(TenantManageLocator.TENANT_MANAGE); + Thread.sleep(TestConstant.ONE_THOUSANG); + + // click delete button + clickButton(TenantManageLocator.DELETE_TENANT_BUTTON); + Thread.sleep(TestConstant.ONE_THOUSANG); + + //click confirm delete button + clickButton(TenantManageLocator.CONFIRM_DELETE_TENANT_BUTTON); + + // Whether to enter the specified page after submit + return ifTitleContains(TenantManageData.TENANAT_MANAGE); + } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/security/UserManagePage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/security/UserManagePage.java index 3e2be2d126..70311f129e 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/security/UserManagePage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/security/UserManagePage.java @@ -26,16 +26,9 @@ public class UserManagePage extends PageCommon { public UserManagePage(WebDriver driver) { super(driver); } - /** - * jump page - */ - public void jumpPage() { - System.out.println("jump tenant page"); - super.jumpPage(UserManageData.USER_URL); - } /** - * creatTenant + * createTenant * * @return Whether to enter the specified page after creat tenant */ @@ -64,4 +57,20 @@ public class UserManagePage extends PageCommon { // Whether to enter the specified page after submit return ifTitleContains(UserManageData.USER_MANAGE); } + + public boolean deleteUser() throws InterruptedException { + Thread.sleep(TestConstant.ONE_THOUSANG); + // click user manage + clickElement(UserManageLocator.CLICK_USER_MANAGE); + Thread.sleep(TestConstant.ONE_THOUSANG); + + // click delete user button + clickButton(UserManageLocator.DELETE_USER_BUTTON ); + + // click confirm delete button + clickButton(UserManageLocator.CONFIRM_DELETE_USER_BUTTON); + + // Whether to enter the specified page after submit + return ifTitleContains(UserManageData.USER_MANAGE); + } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/LoginTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/LoginTest.java index bd3c31d38b..47ce11287d 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/LoginTest.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/LoginTest.java @@ -1,37 +1,26 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package org.apache.dolphinscheduler.testcase; -import org.apache.dolphinscheduler.base.BaseTest; import org.apache.dolphinscheduler.page.LoginPage; +import org.testng.Assert; import org.testng.annotations.Test; +import static org.apache.dolphinscheduler.base.BaseTest.driver; -public class LoginTest extends BaseTest { +@Test(groups={"functionTests","login"}) +public class LoginTest { private LoginPage loginPage; + @Test(description = "LoginTest", priority = 1) public void testLogin() throws InterruptedException { - // init login page loginPage = new LoginPage(driver); + System.out.println("==================================="); + System.out.println("jump to Chinese login page"); + loginPage.jumpPageChinese(); - // enter login page - loginPage.jumpPage(); + System.out.println("start login"); + assert loginPage.login(); + System.out.println("end login"); + System.out.println("==================================="); - //assert login page - assert loginPage.login(); } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/deleteData/DeleteTenantTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/deleteData/DeleteTenantTest.java new file mode 100644 index 0000000000..ddbc9d603b --- /dev/null +++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/deleteData/DeleteTenantTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.testcase.deleteData; + +import org.apache.dolphinscheduler.base.BaseTest; +import org.apache.dolphinscheduler.page.security.TenantManagePage; +import org.testng.annotations.Test; + +public class DeleteTenantTest extends BaseTest { + private TenantManagePage tenantManagePage; + + @Test(groups={"functionTests"},dependsOnGroups = { "login","createTenant"},description = "DeleteTenantTest") + public void testDeleteTenant() throws InterruptedException { + tenantManagePage = new TenantManagePage(driver); + //assert tenant manage page + System.out.println("start delete tenant"); + assert tenantManagePage.deleteTenant(); + System.out.println("end delete tenant"); + System.out.println("==================================="); + } + + +} diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/deleteData/DeleteUserTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/deleteData/DeleteUserTest.java new file mode 100644 index 0000000000..770fae077d --- /dev/null +++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/deleteData/DeleteUserTest.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.testcase.deleteData; + +import org.apache.dolphinscheduler.base.BaseTest; +import org.apache.dolphinscheduler.page.security.UserManagePage; +import org.testng.annotations.Test; + +public class DeleteUserTest extends BaseTest { + private UserManagePage userManagePage; + + @Test(groups={"functionTests"},dependsOnGroups = { "login","user" },description = "DeleteUserTest") + public void testDeleteUser() throws InterruptedException { + userManagePage = new UserManagePage(driver); + //assert user manage page + System.out.println("start delete user"); + assert userManagePage.deleteUser(); + System.out.println("end delete user"); + System.out.println("==================================="); + } +} + diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/project/CreateProjectTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/project/CreateProjectTest.java index 8abd09d37c..546792f0d8 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/project/CreateProjectTest.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/project/CreateProjectTest.java @@ -23,13 +23,16 @@ import org.testng.annotations.Test; public class CreateProjectTest extends BaseTest { private CreateProjectPage createProjectPage; - @Test(description = "TenantTest", priority = 1) - public void testUserManage() throws InterruptedException { + @Test(groups={"functionTests"},dependsOnGroups = { "login" },description = "CreateProjectTest",priority=4) + public void testCreateProject() throws InterruptedException { createProjectPage = new CreateProjectPage(driver); // enter user manage page createProjectPage.jumpProjectManagePage(); //assert user manage page + System.out.println("start create project"); assert createProjectPage.createProject(); + System.out.println("end create project"); + System.out.println("==================================="); } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/project/CreateWorkflowTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/project/CreateWorkflowTest.java index 6ac13f8124..ea87f48529 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/project/CreateWorkflowTest.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/project/CreateWorkflowTest.java @@ -26,12 +26,16 @@ public class CreateWorkflowTest extends BaseTest { private CreateProjectPage createProjectPage; - @Test(description = "CreateWorkflowTest", priority = 1) - public void CreateWorkflowTest() throws InterruptedException { + @Test(groups={"functionTests"},dependsOnGroups = { "login" },description = "CreateWorkflowTest",priority=5) + public void testCreateWorkflow() throws InterruptedException { createProjectPage = new CreateProjectPage(driver); createProjectPage.jumpProjectManagePage(); + createWorkflowPage = new CreateWorkflowPage(driver); - //assert create workflow + System.out.println("start create workflow"); assert createWorkflowPage.createWorkflow(); + assert createWorkflowPage.saveWorkflow(); + System.out.println("end create workflow"); + System.out.println("==================================="); } } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/security/TenantManageTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/security/TenantManageTest.java index 7124b4e094..8b21d3bb61 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/security/TenantManageTest.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/security/TenantManageTest.java @@ -20,15 +20,20 @@ import org.apache.dolphinscheduler.base.BaseTest; import org.apache.dolphinscheduler.page.security.TenantManagePage; import org.testng.annotations.Test; + public class TenantManageTest extends BaseTest { private TenantManagePage tenantManagePage; - @Test(description = "TenantTest", priority = 1) + @Test(groups={"functionTests","createTenant"},dependsOnGroups = { "login" },description = "TenantManageTest") public void testTenantManage() throws InterruptedException { tenantManagePage = new TenantManagePage(driver); - // enter tenant manage page - tenantManagePage.jumpPage(); //assert tenant manage page + System.out.println("start create tenant"); assert tenantManagePage.createTenant(); + System.out.println("end create tenant"); + System.out.println("==================================="); + } + + } diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/security/UserManageTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/security/UserManageTest.java index 834ebdbbf0..9381b68c75 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/security/UserManageTest.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/security/UserManageTest.java @@ -23,12 +23,14 @@ import org.testng.annotations.Test; public class UserManageTest extends BaseTest { private UserManagePage userManagePage; - @Test(description = "TenantTest", priority = 1) + @Test(groups={"functionTests","user"},dependsOnGroups = { "login" },description = "UserManageTest") public void testUserManage() throws InterruptedException { userManagePage = new UserManagePage(driver); - // enter user manage page - userManagePage.jumpPage(); //assert user manage page + System.out.println("start create user"); assert userManagePage.createUser(); + System.out.println("end create user"); + System.out.println("==================================="); + } } diff --git a/e2e/suite.xml b/e2e/suite.xml deleted file mode 100644 index d9d7ae6845..0000000000 --- a/e2e/suite.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/e2e/testng.xml b/e2e/testng.xml index 757ffab248..56c9649135 100644 --- a/e2e/testng.xml +++ b/e2e/testng.xml @@ -1,19 +1,23 @@ - - - - - - + + + + + + + + - + + + + + + + - - - - - + From 98f3201a4d8a52bd054bc50b9edf9a628f5781ce Mon Sep 17 00:00:00 2001 From: xingchun-chen <55787491+xingchun-chen@users.noreply.github.com> Date: Wed, 18 Mar 2020 10:03:03 +0800 Subject: [PATCH 044/401] add LoginTest license (#2217) Co-authored-by: chenxingchun <438044805@qq.com> --- .../dolphinscheduler/testcase/LoginTest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/LoginTest.java b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/LoginTest.java index 47ce11287d..024ffcdc8a 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/testcase/LoginTest.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/testcase/LoginTest.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.dolphinscheduler.testcase; import org.apache.dolphinscheduler.page.LoginPage; From 55e232db844fc80d22a7e46f2e18b5b150e8c57c Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Wed, 18 Mar 2020 11:26:11 +0800 Subject: [PATCH 045/401] dd branch flow node verification --- .../js/conf/home/pages/dag/_source/dag.scss | 2 +- .../js/conf/home/pages/dag/_source/dag.vue | 91 +++++++++++-------- .../pages/dag/_source/formModel/formModel.vue | 2 +- .../formModel/tasks/_source/nodeStatus.vue | 4 - .../src/js/module/i18n/locale/en_US.js | 3 +- .../src/js/module/i18n/locale/zh_CN.js | 3 +- 6 files changed, 60 insertions(+), 45 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss index 6d97856960..9973750d98 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss @@ -135,7 +135,7 @@ width: 36px; height: 36px; float: left; - margin-bottom: 11px; + margin-bottom: 8px; border-radius: 3px; .disabled { .icos { diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue index a1ccd39260..8628fdb8ef 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue @@ -326,45 +326,62 @@ * Storage interface */ _save (sourceType) { - return new Promise((resolve, reject) => { - this.spinnerLoading = true - // Storage store - Dag.saveStore().then(res => { - if (this.urlParam.id) { - /** - * Edit - * @param saveInstanceEditDAGChart => Process instance editing - * @param saveEditDAGChart => Process definition editing - */ - this[this.type === 'instance' ? 'updateInstance' : 'updateDefinition'](this.urlParam.id).then(res => { - this.$message.success(res.msg) - this.spinnerLoading = false - resolve() - }).catch(e => { - this.$message.error(e.msg || '') - this.spinnerLoading = false - reject(e) - }) - } else { - // New - this.saveDAGchart().then(res => { - this.$message.success(res.msg) - this.spinnerLoading = false - // source @/conf/home/pages/dag/_source/editAffirmModel/index.js - if (sourceType !== 'affirm') { - // Jump process definition - this.$router.push({ name: 'projects-definition-list' }) - } - resolve() - }).catch(e => { - this.$message.error(e.msg || '') - this.setName('') - this.spinnerLoading = false - reject(e) - }) - } + if(this._verifConditions()) { + return new Promise((resolve, reject) => { + this.spinnerLoading = true + // Storage store + Dag.saveStore().then(res => { + if (this.urlParam.id) { + /** + * Edit + * @param saveInstanceEditDAGChart => Process instance editing + * @param saveEditDAGChart => Process definition editing + */ + this[this.type === 'instance' ? 'updateInstance' : 'updateDefinition'](this.urlParam.id).then(res => { + this.$message.success(res.msg) + this.spinnerLoading = false + resolve() + }).catch(e => { + this.$message.error(e.msg || '') + this.spinnerLoading = false + reject(e) + }) + } else { + // New + this.saveDAGchart().then(res => { + this.$message.success(res.msg) + this.spinnerLoading = false + // source @/conf/home/pages/dag/_source/editAffirmModel/index.js + if (sourceType !== 'affirm') { + // Jump process definition + this.$router.push({ name: 'projects-definition-list' }) + } + resolve() + }).catch(e => { + this.$message.error(e.msg || '') + this.setName('') + this.spinnerLoading = false + reject(e) + }) + } + }) }) + } + }, + _verifConditions () { + let tasks = this.$store.state.dag.tasks + let bool = true + tasks.map(v=>{ + if(v.type == 'CONDITIONS' && (v.conditionResult.successNode[0] =='' || v.conditionResult.successNode[0] == null || v.conditionResult.failedNode[0] =='' || v.conditionResult.failedNode[0] == null)) { + bool = false + return false + } }) + if(!bool) { + this.$message.warning(`${i18n.$t('Successful branch flow and failed branch flow are required')}`) + return false + } + return true }, /** * Global parameter diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index cc1c8b6d6c..a49e700e00 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -470,7 +470,7 @@ this.$message.warning(`${i18n.$t('Please enter name (required)')}`) return false } - if (this.successBranch !='' && this.successBranch == this.failedBranch) { + if (this.successBranch !='' && this.successBranch !=null && this.successBranch == this.failedBranch) { this.$message.warning(`${i18n.$t('Cannot select the same node for successful branch flow and failed branch flow')}`) return false } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue index fa7ee89e98..0c3f7433a3 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue @@ -128,10 +128,6 @@ this.store.dispatch('dag/getProcessTasksList', { processDefinitionId: ids }).then(res => { resolve(['ALL'].concat(_.map(res, v => v.name))) }) - } else { - this.store.dispatch('dag/getTaskListDefIdAll', { processDefinitionIdList: ids }).then(res => { - resolve(res) - }) } }) }, diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js index 64ab298437..d2c096e503 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js @@ -568,5 +568,6 @@ export default { 'All Columns': 'All Columns', 'Some Columns': 'Some Columns', 'Branch flow': 'Branch flow', - 'Cannot select the same node for successful branch flow and failed branch flow': 'Cannot select the same node for successful branch flow and failed branch flow' + 'Cannot select the same node for successful branch flow and failed branch flow': 'Cannot select the same node for successful branch flow and failed branch flow', + 'Successful branch flow and failed branch flow are required': 'Successful branch flow and failed branch flow are required' } diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js index 3363eea198..1003d6eeba 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -568,5 +568,6 @@ export default { 'All Columns': '全表导入', 'Some Columns': '选择列', 'Branch flow': '分支流转', - 'Cannot select the same node for successful branch flow and failed branch flow': '成功分支流转和失败分支流转不能选择同一个节点' + 'Cannot select the same node for successful branch flow and failed branch flow': '成功分支流转和失败分支流转不能选择同一个节点', + 'Successful branch flow and failed branch flow are required': '成功分支流转和失败分支流转必填', } From 280a240ff53cc7af8e9b84c890c4e837841db286 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Thu, 19 Mar 2020 14:00:18 +0800 Subject: [PATCH 046/401] datax --- .../pages/dag/_source/formModel/formModel.vue | 3 +- .../dag/_source/formModel/tasks/datax.vue | 345 ++++++++++++------ .../module/components/transfer/resource.vue | 4 +- .../src/js/module/i18n/locale/en_US.js | 1 + .../src/js/module/i18n/locale/zh_CN.js | 1 + 5 files changed, 233 insertions(+), 121 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index a49e700e00..56cefd5b31 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -430,7 +430,8 @@ * return params */ _onParams (o) { - this.params = Object.assign(this.params, {}, o) + console.log(o) + this.params = Object.assign({}, o) }, _onCacheParams (o) { diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue index 959610f95a..0f82903844 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue @@ -17,90 +17,127 @@ diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue index 706a35f4fe..3691985d7d 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue @@ -44,19 +44,9 @@
{{$t('Main jar package')}}
- - - - + +
{{ node.raw.fullName }}
+
@@ -88,12 +78,9 @@
{{$t('Resources')}}
- - + +
{{ node.raw.fullName }}
+
@@ -115,6 +102,8 @@ import mListBox from './_source/listBox' import mResources from './_source/resources' import mLocalParams from './_source/localParams' + import Treeselect from '@riophae/vue-treeselect' + import '@riophae/vue-treeselect/dist/vue-treeselect.css' import disabledState from '@/module/mixin/disabledState' export default { name: 'mr', @@ -125,6 +114,7 @@ // Master jar package mainJar: null, // Main jar package (List) + mainJarLists: [], mainJarList: [], // Resource(list) resourceList: [], @@ -139,7 +129,12 @@ // Program type programType: 'JAVA', // Program type(List) - programTypeList: [{ code: 'JAVA' }, { code: 'PYTHON' }] + programTypeList: [{ code: 'JAVA' }, { code: 'PYTHON' }], + normalizer(node) { + return { + label: node.name + } + } } }, props: { @@ -165,6 +160,12 @@ _onCacheResourcesData (a) { this.cacheResourceList = a }, + diGuiTree(item) { // Recursive convenience tree structure + item.forEach(item => { + item.children === '' || item.children === undefined || item.children === null || item.children.length === 0?　　　　　　　　 + delete item.children : this.diGuiTree(item.children); + }) + }, /** * verification */ @@ -179,22 +180,19 @@ return false } - if (!this.$refs.refResources._verifResources()) { - return false - } - // localParams Subcomponent verification if (!this.$refs.refLocalParams._verifProp()) { return false } - // storage this.$emit('on-params', { mainClass: this.mainClass, mainJar: { - res: this.mainJar + id: this.mainJar }, - resourceList: this.resourceList, + resourceList: _.map(this.resourceList, v => { + return {id: v} + }), localParams: this.localParams, mainArgs: this.mainArgs, others: this.others, @@ -202,24 +200,7 @@ }) return true }, - /** - * Get resource data - */ - _getResourcesList () { - return new Promise((resolve, reject) => { - let isJar = (alias) => { - return alias.substring(alias.lastIndexOf('.') + 1, alias.length) !== 'jar' - } - this.mainJarList = _.map(_.cloneDeep(this.store.state.dag.resourcesListS), v => { - return { - id: v.id, - code: v.alias, - disabled: isJar(v.alias) - } - }) - resolve() - }) - } + }, watch: { /** @@ -240,9 +221,11 @@ return { mainClass: this.mainClass, mainJar: { - res: this.mainJar + id: this.mainJar }, - resourceList: this.cacheResourceList, + resourceList: _.map(this.resourceList, v => { + return {id: v} + }), localParams: this.localParams, mainArgs: this.mainArgs, others: this.others, @@ -251,13 +234,18 @@ } }, created () { - this._getResourcesList().then(() => { + let item = this.store.state.dag.resourcesListS + let items = this.store.state.dag.resourcesListJar + this.diGuiTree(item) + this.diGuiTree(items) + this.mainJarList = item + this.mainJarLists = items let o = this.backfillItem // Non-null objects represent backfill if (!_.isEmpty(o)) { this.mainClass = o.params.mainClass || '' - this.mainJar = o.params.mainJar.res || '' + this.mainJar = o.params.mainJar.id || '' this.mainArgs = o.params.mainArgs || '' this.others = o.params.others this.programType = o.params.programType || 'JAVA' @@ -265,7 +253,9 @@ // backfill resourceList let resourceList = o.params.resourceList || [] if (resourceList.length) { - this.resourceList = resourceList + this.resourceList = _.map(resourceList, v => { + return v.id + }) this.cacheResourceList = resourceList } @@ -275,12 +265,11 @@ this.localParams = localParams } } - }) }, mounted () { }, - components: { mLocalParams, mListBox, mResources } + components: { mLocalParams, mListBox, mResources, Treeselect } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue index 6f495d22a0..e8e7e58771 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue @@ -108,10 +108,6 @@ return false } - if (!this.$refs.refResources._verifResources()) { - return false - } - // localParams Subcomponent verification if (!this.$refs.refLocalParams._verifProp()) { return false @@ -119,7 +115,9 @@ // storage this.$emit('on-params', { - resourceList: this.resourceList, + resourceList: _.map(this.resourceList, v => { + return {id: v} + }), localParams: this.localParams, rawScript: editor.getValue() }) @@ -129,8 +127,6 @@ * Processing code highlighting */ _handlerEditor () { - this._destroyEditor() - // editor editor = codemirror('code-python-mirror', { mode: 'python', @@ -145,45 +141,28 @@ } } - this.changes = () => { - this._cacheParams() - } - // Monitor keyboard editor.on('keypress', this.keypress) - editor.on('changes', this.changes) - editor.setValue(this.rawScript) return editor - }, - _cacheParams () { - this.$emit('on-cache-params', { - resourceList: this.cacheResourceList, - localParams: this.localParams, - rawScript: editor ? editor.getValue() : '' - }); - }, - _destroyEditor () { - if (editor) { - editor.toTextArea() // Uninstall - editor.off($('.code-python-mirror'), 'keypress', this.keypress) - editor.off($('.code-python-mirror'), 'changes', this.changes) - } } }, watch: { //Watch the cacheParams cacheParams (val) { - this._cacheParams() + this.$emit('on-cache-params', val); } }, computed: { cacheParams () { return { - resourceList: this.cacheResourceList, - localParams: this.localParams + resourceList: _.map(this.resourceList, v => { + return {id: v} + }), + localParams: this.localParams, + rawScript: editor ? editor.getValue() : '' } } }, @@ -197,7 +176,9 @@ // backfill resourceList let resourceList = o.params.resourceList || [] if (resourceList.length) { - this.resourceList = resourceList + this.resourceList = _.map(resourceList, v => { + return v.id + }) this.cacheResourceList = resourceList } @@ -214,11 +195,8 @@ }, 200) }, destroyed () { - if (editor) { - editor.toTextArea() // Uninstall - editor.off($('.code-python-mirror'), 'keypress', this.keypress) - editor.off($('.code-python-mirror'), 'changes', this.changes) - } + editor.toTextArea() // Uninstall + editor.off($('.code-python-mirror'), 'keypress', this.keypress) }, components: { mLocalParams, mListBox, mResources } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue index a4b20f3310..952fd3a38d 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue @@ -32,6 +32,14 @@
+
{{$t('Resources')}}
+
+ +
{{ node.raw.fullName }}
+ +
+ +
{{$t('Custom Parameters')}}
@@ -63,6 +71,8 @@ import mResources from './_source/resources' import mLocalParams from './_source/localParams' import disabledState from '@/module/mixin/disabledState' + import Treeselect from '@riophae/vue-treeselect' + import '@riophae/vue-treeselect/dist/vue-treeselect.css' import codemirror from '@/conf/home/pages/resource/pages/file/pages/_source/codemirror' let editor @@ -78,7 +88,14 @@ // resource(list) resourceList: [], // Cache ResourceList - cacheResourceList: [] + cacheResourceList: [], + // define options + options: [], + normalizer(node) { + return { + label: node.name + } + }, } }, mixins: [disabledState], @@ -143,17 +160,19 @@ return false } - if (!this.$refs.refResources._verifResources()) { - return false - } - // localParams Subcomponent verification if (!this.$refs.refLocalParams._verifProp()) { return false } + // Process resourcelist + let dataProcessing= _.map(this.resourceList, v => { + return { + id: v + } + }) // storage this.$emit('on-params', { - resourceList: this.resourceList, + resourceList: dataProcessing, localParams: this.localParams, rawScript: editor.getValue() }) @@ -163,8 +182,6 @@ * Processing code highlighting */ _handlerEditor () { - this._destroyEditor() - // editor editor = codemirror('code-shell-mirror', { mode: 'shell', @@ -179,51 +196,41 @@ } } - this.changes = () => { - this._cacheParams() - } - // Monitor keyboard editor.on('keypress', this.keypress) - - editor.on('changes', this.changes) - editor.setValue(this.rawScript) return editor }, - _cacheParams () { - this.$emit('on-cache-params', { - resourceList: this.cacheResourceList, - localParams: this.localParams, - rawScript: editor ? editor.getValue() : '' - }); - }, - _destroyEditor () { - if (editor) { - editor.toTextArea() // Uninstall - editor.off($('.code-sql-mirror'), 'keypress', this.keypress) - editor.off($('.code-sql-mirror'), 'changes', this.changes) - } + diGuiTree(item) { // Recursive convenience tree structure + item.forEach(item => { + item.children === '' || item.children === undefined || item.children === null || item.children.length === 0?　　　　　　　　 + delete item.children : this.diGuiTree(item.children); + }) } }, watch: { //Watch the cacheParams cacheParams (val) { - this._cacheParams() + this.$emit('on-cache-params', val); } }, computed: { cacheParams () { return { - resourceList: this.cacheResourceList, - localParams: this.localParams + resourceList: _.map(this.resourceList, v => { + return {id: v} + }), + localParams: this.localParams, + rawScript: editor ? editor.getValue() : '' } } }, created () { + let item = this.store.state.dag.resourcesListS + this.diGuiTree(item) + this.options = item let o = this.backfillItem - // Non-null objects represent backfill if (!_.isEmpty(o)) { this.rawScript = o.params.rawScript || '' @@ -231,7 +238,9 @@ // backfill resourceList let resourceList = o.params.resourceList || [] if (resourceList.length) { - this.resourceList = resourceList + this.resourceList = _.map(resourceList, v => { + return v.id + }) this.cacheResourceList = resourceList } @@ -251,10 +260,9 @@ if (editor) { editor.toTextArea() // Uninstall editor.off($('.code-shell-mirror'), 'keypress', this.keypress) - editor.off($('.code-shell-mirror'), 'changes', this.changes) } }, - components: { mLocalParams, mListBox, mResources, mScriptBox } + components: { mLocalParams, mListBox, mResources, mScriptBox, Treeselect } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/createUdfFolder/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/createUdfFolder/index.vue new file mode 100755 index 0000000000..2511452269 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/createUdfFolder/index.vue @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/details/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/details/index.vue index e961d8b1ee..6875cd4b2e 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/details/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/details/index.vue @@ -21,7 +21,7 @@

{{name}}
- + {{size}}

diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue index a0d1d7d187..0290af0988 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue @@ -44,8 +44,8 @@ + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subFileFolder/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subFileFolder/index.vue new file mode 100755 index 0000000000..9f903a127b --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subFileFolder/index.vue @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/_source/list.vue new file mode 100755 index 0000000000..f5e801a205 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/_source/list.vue @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/_source/rename.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/_source/rename.vue new file mode 100755 index 0000000000..6f7dacae89 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/_source/rename.vue @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + \ No newline at end of file diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/index.vue new file mode 100755 index 0000000000..12be6b0bc8 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/index.vue @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/createUdfFolder/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/createUdfFolder/index.vue new file mode 100755 index 0000000000..c707ce8c90 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/createUdfFolder/index.vue @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/createUdf.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/createUdf.vue index 01d8d22650..1408c552db 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/createUdf.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/createUdf.vue @@ -15,7 +15,7 @@ * limitations under the License. */ @@ -64,7 +64,8 @@ pageNo: 1, searchVal: '', userId: '' - } + }, + isLeft: true } }, mixins: [listUrlParamHandle], @@ -98,6 +99,11 @@ * get data list */ _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getProcessListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -133,6 +139,9 @@ mounted() { this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mList, mConditions, mSpin, mListConstruction, mSecondaryMenu, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/index.vue index b95d4ed720..891dc2e281 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/index.vue @@ -30,7 +30,7 @@ - + @@ -74,7 +74,8 @@ endDate: '', // Exectuor Name executorName: '' - } + }, + isLeft: true } }, props: {}, @@ -136,6 +137,11 @@ * @desc Prevent functions from being called multiple times */ _debounceGET: _.debounce(function (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this._getProcessInstanceListP(flag) }, 100, { 'leading': false, @@ -183,6 +189,7 @@ beforeDestroy () { // Destruction wheel clearInterval(this.setIntervalP) + sessionStorage.setItem('isLeft',1) }, components: { mList, mInstanceConditions, mSpin, mListConstruction, mSecondaryMenu, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/index.vue index 4cb166647e..594ffad14e 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/index.vue @@ -30,7 +30,7 @@ - + @@ -71,7 +71,8 @@ endDate: '', // Exectuor Name executorName: '' - } + }, + isLeft: true } }, mixins: [listUrlParamHandle], @@ -118,6 +119,11 @@ * @desc Prevent functions from being called multiple times */ _debounceGET: _.debounce(function (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this._getList(flag) }, 100, { 'leading': false, @@ -146,6 +152,7 @@ beforeDestroy () { // Destruction wheel clearInterval(this.setIntervalP) + sessionStorage.setItem('isLeft',1) }, components: { mList, mInstanceConditions, mSpin, mListConstruction, mSecondaryMenu, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/index.vue index 73ce023ee7..5cf343ddda 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/list/index.vue @@ -38,7 +38,7 @@ - + @@ -67,7 +67,8 @@ pageNo: 1, searchVal: '', type: 'FILE' - } + }, + isLeft: true } }, mixins: [listUrlParamHandle], @@ -91,6 +92,11 @@ this.searchParams.pageSize = val }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getResourcesListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -125,6 +131,9 @@ mounted () { this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mListConstruction, mConditions, mList, mSpin, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/index.vue index 12be6b0bc8..dac5cc0a86 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/index.vue @@ -42,7 +42,7 @@ - +
@@ -73,6 +73,7 @@ searchVal: '', type: 'FILE' }, + isLeft: true, breadList: [] } }, @@ -97,6 +98,11 @@ this.searchParams.pageSize = val }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getResourcesListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -159,6 +165,9 @@ this.breadList = dir this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mListConstruction, mConditions, mList, mSpin, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/index.vue index d6c79bd258..74b789fe93 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/index.vue @@ -36,8 +36,7 @@ - - + @@ -64,7 +63,8 @@ pageSize: 10, pageNo: 1, searchVal: '' - } + }, + isLeft: true } }, mixins: [listUrlParamHandle], @@ -110,6 +110,11 @@ this._debounceGET() }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getUdfFuncListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -137,6 +142,9 @@ mounted () { this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mListConstruction, mConditions, mList, mSpin, mCreateUdf, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/resource/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/resource/index.vue index b87b17800a..4058f267b7 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/resource/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/resource/index.vue @@ -37,8 +37,7 @@ - - + @@ -66,7 +65,8 @@ pageNo: 1, searchVal: '', type: 'UDF' - } + }, + isLeft: true } }, mixins: [listUrlParamHandle], @@ -98,6 +98,11 @@ this._debounceGET() }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getResourcesListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -125,6 +130,9 @@ mounted () { this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mListConstruction, mConditions, mList, mSpin, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/index.vue index dd39716f9a..ea8c4d838f 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/index.vue @@ -41,7 +41,7 @@ - +
@@ -72,6 +72,7 @@ searchVal: '', type: 'UDF' }, + isLeft: true, breadList: [] } }, @@ -106,6 +107,11 @@ this._debounceGET() }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getResourcesListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -160,6 +166,9 @@ this.breadList = dir this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mListConstruction, mConditions, mList, mSpin, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/queue/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/queue/index.vue index 47ce14abd7..8e559e45a7 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/queue/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/queue/index.vue @@ -38,7 +38,7 @@ - + @@ -66,6 +66,7 @@ pageNo: 1, searchVal: '' }, + isLeft: true, isADMIN: store.state.user.userInfo.userType === 'ADMIN_USER' } }, @@ -116,6 +117,11 @@ }) }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getQueueListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -143,6 +149,9 @@ mounted () { this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mList, mListConstruction, mConditions, mSpin, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/tenement/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/tenement/index.vue index 0c38f0a911..ca180b1718 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/tenement/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/tenement/index.vue @@ -39,7 +39,7 @@ - + @@ -67,6 +67,7 @@ pageNo: 1, searchVal: '' }, + isLeft: true, isADMIN: store.state.user.userInfo.userType === 'ADMIN_USER' } }, @@ -120,6 +121,11 @@ }) }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getTenantListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -147,6 +153,9 @@ mounted () { this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mList, mListConstruction, mConditions, mSpin, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/index.vue index f8ad40df85..b09b08bcf6 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/index.vue @@ -39,7 +39,7 @@ - + @@ -65,7 +65,8 @@ pageSize: 10, pageNo: 1, searchVal: '' - } + }, + isLeft: true } }, mixins: [listUrlParamHandle], @@ -118,6 +119,11 @@ }) }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getUsersListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -145,6 +151,9 @@ mounted () { this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mList, mListConstruction, mConditions, mSpin, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/warningGroups/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/warningGroups/index.vue index e70ead46ce..ad604ba3a6 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/warningGroups/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/warningGroups/index.vue @@ -39,7 +39,7 @@ - + @@ -67,6 +67,7 @@ pageNo: 1, searchVal: '' }, + isLeft: true, isADMIN: store.state.user.userInfo.userType === 'ADMIN_USER' } }, @@ -120,6 +121,11 @@ }) }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getAlertgroupP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -147,6 +153,9 @@ mounted () { this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mList, mListConstruction, mConditions, mSpin, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/token/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/token/index.vue index 21d2becd32..3398acca13 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/token/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/token/index.vue @@ -40,7 +40,7 @@ - + @@ -67,7 +67,8 @@ pageSize: 10, pageNo: 1, searchVal: '' - } + }, + isLeft: true } }, mixins: [listUrlParamHandle], @@ -120,6 +121,11 @@ }) }, _getList (flag) { + if(sessionStorage.getItem('isLeft')==0) { + this.isLeft = false + } else { + this.isLeft = true + } this.isLoading = !flag this.getTokenListP(this.searchParams).then(res => { if(this.searchParams.pageNo>1 && res.totalList.length == 0) { @@ -147,6 +153,9 @@ mounted () { this.$modal.destroy() }, + beforeDestroy () { + sessionStorage.setItem('isLeft',1) + }, components: { mSecondaryMenu, mList, mListConstruction, mConditions, mSpin, mNoData } } diff --git a/dolphinscheduler-ui/src/js/conf/login/App.vue b/dolphinscheduler-ui/src/js/conf/login/App.vue index a79743a262..8d065920a8 100644 --- a/dolphinscheduler-ui/src/js/conf/login/App.vue +++ b/dolphinscheduler-ui/src/js/conf/login/App.vue @@ -84,6 +84,7 @@ this._gLogin().then(res => { setTimeout(() => { this.spinnerLoading = false + sessionStorage.setItem('isLeft',1); if (res.data.hasOwnProperty("sessionId")) { let sessionId=res.data.sessionId sessionStorage.setItem("sessionId", sessionId) diff --git a/dolphinscheduler-ui/src/js/module/components/secondaryMenu/secondaryMenu.vue b/dolphinscheduler-ui/src/js/module/components/secondaryMenu/secondaryMenu.vue index 102df8f910..5336555c21 100644 --- a/dolphinscheduler-ui/src/js/module/components/secondaryMenu/secondaryMenu.vue +++ b/dolphinscheduler-ui/src/js/module/components/secondaryMenu/secondaryMenu.vue @@ -63,7 +63,8 @@ menuList: menu(this.type), index: 0, id: this.$route.params.id, - isTogHide: false + isTogHide: false, + isLeft: true } }, props: { @@ -82,6 +83,11 @@ }, _toggleMenu () { this.isTogHide = !this.isTogHide + if(this.isTogHide) { + sessionStorage.setItem('isLeft',0) + } else { + sessionStorage.setItem('isLeft',1) + } } }, mounted () { diff --git a/dolphinscheduler-ui/src/js/module/components/spin/spin.vue b/dolphinscheduler-ui/src/js/module/components/spin/spin.vue index 95fc8e924e..7c6a9c3acf 100644 --- a/dolphinscheduler-ui/src/js/module/components/spin/spin.vue +++ b/dolphinscheduler-ui/src/js/module/components/spin/spin.vue @@ -47,7 +47,7 @@ #spin-model { position: fixed; left: 20px; - top: 80px; + top: 60px; background: #fff; z-index: 99; border-radius: 3px; @@ -69,11 +69,11 @@ } &.spin-sp1 { width: calc(100% - 40px); - height: calc(100% - 100px); + height: calc(100% - 60px); } &.spin-sp2 { - width: calc(100% - 240px); - height: calc(100% - 100px); + width: calc(100% - 220px); + height: calc(100% - 60px); left: 220px; } } diff --git a/dolphinscheduler-ui/src/sass/conf/home/index.scss b/dolphinscheduler-ui/src/sass/conf/home/index.scss index 76b9f6da01..16f588363d 100644 --- a/dolphinscheduler-ui/src/sass/conf/home/index.scss +++ b/dolphinscheduler-ui/src/sass/conf/home/index.scss @@ -24,8 +24,13 @@ body { .ans-message-box,.ans-message-wrapper { z-index: 121 !important; } + &::-webkit-scrollbar { + width: 0; + height: 0; + } } + @media screen and (max-width: 960px){ .nav-model { .logo-box, From 1b419f38294cb06f4d524c7dc77b74eb8724d8b8 Mon Sep 17 00:00:00 2001 From: samz406 Date: Tue, 28 Apr 2020 17:11:04 +0800 Subject: [PATCH 145/401] fix The create user is always loading when the queue does not select any value --- .../security/pages/users/_source/createUser.vue | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue index 39dc27855a..b2cf89e1c7 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue @@ -150,7 +150,7 @@ // Mobile phone number regular let regPhone = /^1(3|4|5|6|7|8)\d{9}$/; // eslint-disable-line - + let regPassword = /^(?![0-9]+$)(?![a-z]+$)(?![A-Z]+$)(?![`~!@#$%^&*()_\-+=<>?:"{}|,.\/;'\\[\]·~！@#￥%……&*（）——\-+={}|《》？：“”【】、；‘’，。、]+$)[`~!@#$%^&*()_\-+=<>?:"{}|,.\/;'\\[\]·~！@#￥%……&*（）——\-+={}|《》？：“”【】、；‘’，。、0-9A-Za-z]{6,22}$/; // user name @@ -194,7 +194,7 @@ _getQueueList () { return new Promise((resolve, reject) => { this.store.dispatch('security/getQueueList').then(res => { - + this.queueList = _.map(res, v => { return { id: v.id, @@ -229,12 +229,18 @@ }, _submit () { this.$refs['popup'].spinnerLoading = true + + let queueCode = ''; + //get queue code + if (this.queueName != ''){ + queueCode = this.queueList.length > 0 ? _.find(this.queueList, ['id', this.queueName]).code : '' + } let param = { userName: this.userName, userPassword: this.userPassword, tenantId: this.tenantId, email: this.email, - queue: this.queueList.length>0? _.find(this.queueList, ['id', this.queueName]).code : '', + queue: queueCode, phone: this.phone } From 043a20f2c72993b63b3963b0057fce3c6a83d00b Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Wed, 29 Apr 2020 17:36:21 +0800 Subject: [PATCH 146/401] No master don't create command #2571 (#2575) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * dispatch task fail will set task status failed * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，no worker condition , master will while ture wait for worker startup 2，worker response task status sync wait for result * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * 1，task status statistics and process status statistics bug fix (#2357) 2，worker group bug fix * send mail error, #2466 bug fix * send mail error, #2466 bug fix * send mail error, #2466 bug fix * send mail error, #2466 bug fix * #2486 bug fix * host and workergroup compatible * EnterpriseWeChatUtils modify * EnterpriseWeChatUtils modify * EnterpriseWeChatUtils modify * #2499 bug fix * add comment * revert comment * revert comment * #2499 buf fix * #2499 bug fix * #2499 bug fix * #2499 bug fix * #2499 bug fix * #2499 bug fix * #2499 bug fix * no valid worker group，master can kill task directly * no valid worker group，master can kill task directly * no valid worker group，master can kill task directly * no valid worker group，master can kill task directly * no valid worker group，master can kill task directly * no valid worker group，master can kill task directly * no valid worker group，master can kill task directly * no valid worker group，master can kill task directly * no valid worker group，master can kill task directly * No master don't create command #2571 * No master don't create command #2571 Co-authored-by: qiaozhanwei --- .../api/service/ExecutorService.java | 11 ++++- .../api/service/SchedulerService.java | 1 + .../api/service/ExecutorService2Test.java | 43 ++++++++++++++++--- 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index 4b95ad385a..7776ca3b57 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.api.enums.ExecuteType; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.*; +import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; @@ -59,7 +60,7 @@ public class ExecutorService extends BaseService{ private ProcessDefinitionMapper processDefinitionMapper; @Autowired - private ProcessDefinitionService processDefinitionService; + private MonitorService monitorService; @Autowired @@ -123,6 +124,14 @@ public class ExecutorService extends BaseService{ return result; } + // check master server exists + List masterServers = monitorService.getServerListFromZK(true); + + + if (masterServers.size() == 0) { + putMsg(result, Status.MASTER_NOT_EXISTS); + return result; + } /** * create command */ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java index 72122100a1..e2b482da00 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java @@ -365,6 +365,7 @@ public class SchedulerService extends BaseService { if (masterServers.size() == 0) { putMsg(result, Status.MASTER_NOT_EXISTS); + return result; } // set status diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java index 07d7477930..c4f0bef4f0 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.RunMode; +import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; @@ -63,6 +64,9 @@ public class ExecutorService2Test { @Mock private ProjectService projectService; + @Mock + private MonitorService monitorService; + private int processDefinitionId = 1; private int tenantId = 1; @@ -102,6 +106,7 @@ public class ExecutorService2Test { Mockito.when(processDefinitionMapper.selectById(processDefinitionId)).thenReturn(processDefinition); Mockito.when(processService.getTenantForProcess(tenantId, userId)).thenReturn(new Tenant()); Mockito.when(processService.createCommand(any(Command.class))).thenReturn(1); + Mockito.when(monitorService.getServerListFromZK(true)).thenReturn(getMasterServersList()); } /** @@ -121,7 +126,6 @@ public class ExecutorService2Test { Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); verify(processService, times(1)).createCommand(any(Command.class)); }catch (Exception e){ - Assert.assertTrue(false); } } @@ -142,7 +146,6 @@ public class ExecutorService2Test { Assert.assertEquals(Status.START_PROCESS_INSTANCE_ERROR, result.get(Constants.STATUS)); verify(processService, times(0)).createCommand(any(Command.class)); }catch (Exception e){ - Assert.assertTrue(false); } } @@ -163,7 +166,6 @@ public class ExecutorService2Test { Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); verify(processService, times(1)).createCommand(any(Command.class)); }catch (Exception e){ - Assert.assertTrue(false); } } @@ -184,7 +186,6 @@ public class ExecutorService2Test { Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); verify(processService, times(31)).createCommand(any(Command.class)); }catch (Exception e){ - Assert.assertTrue(false); } } @@ -205,10 +206,42 @@ public class ExecutorService2Test { Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); verify(processService, times(15)).createCommand(any(Command.class)); }catch (Exception e){ - Assert.assertTrue(false); } } + + @Test + public void testNoMsterServers() throws ParseException{ + Mockito.when(monitorService.getServerListFromZK(true)).thenReturn(new ArrayList()); + + Map result = executorService.execProcessInstance(loginUser, projectName, + processDefinitionId, cronTime, CommandType.COMPLEMENT_DATA, + null, null, + null, null, 0, + "", "", RunMode.RUN_MODE_PARALLEL, + Priority.LOW, Constants.DEFAULT_WORKER_GROUP, 110); + Assert.assertEquals(result.get(Constants.STATUS),Status.MASTER_NOT_EXISTS); + + } + + private List getMasterServersList(){ + List masterServerList = new ArrayList<>(); + Server masterServer1 = new Server(); + masterServer1.setId(1); + masterServer1.setHost("192.168.220.188"); + masterServer1.setPort(1121); + masterServerList.add(masterServer1); + + Server masterServer2 = new Server(); + masterServer2.setId(2); + masterServer2.setHost("192.168.220.189"); + masterServer2.setPort(1122); + masterServerList.add(masterServer2); + + return masterServerList; + + } + private List zeroSchedulerList(){ return Collections.EMPTY_LIST; } From 077829c59a9159e0e24dfa60833d602184d59114 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Thu, 30 Apr 2020 09:25:53 +0800 Subject: [PATCH 147/401] Fix condition judgment state value --- .../src/js/conf/home/pages/dag/_source/dag.vue | 7 +++++++ .../dag/_source/formModel/tasks/_source/nodeStatus.vue | 4 ++-- .../pages/dag/_source/formModel/tasks/conditions.vue | 9 ++++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue index 6f630071c1..ae15e20f74 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue @@ -259,8 +259,15 @@ if (v2.name === v1.name) { let dom = $(`#${v2.id}`) let state = dom.find('.state-p') + let depState = '' + taskList.forEach(item=>{ + if(item.name==v1.name) { + depState = item.state + } + }) dom.attr('data-state-id', v1.stateId) dom.attr('data-dependent-result', v1.dependentResult || '') + dom.attr('data-dependent-depState', depState) state.append(``) state.find('strong').attr('title', titleTpl(v2, v1.desc)) } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue index 0c3f7433a3..972782207c 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue @@ -28,8 +28,8 @@ diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue index 639b2f4ad7..ab48ce7ab3 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue @@ -143,13 +143,20 @@ created () { let o = this.backfillItem let dependentResult = $(`#${o.id}`).data('dependent-result') || {} + // Does not represent an empty object backfill if (!_.isEmpty(o)) { this.relation = _.cloneDeep(o.dependence.relation) || 'AND' this.dependTaskList = _.cloneDeep(o.dependence.dependTaskList) || [] let defaultState = this.isDetails ? 'WAITING' : '' // Process instance return status display matches by key - _.map(this.dependTaskList, v => _.map(v.dependItemList, v1 => v1.state = dependentResult[`${v1.definitionId}-${v1.depTasks}-${v1.cycle}-${v1.dateValue}`] || defaultState)) + _.map(this.dependTaskList, v => _.map(v.dependItemList, v1 => { + $(`#${o.id}`).siblings().each(function(){ + if(v1.depTasks == $(this).text()) { + v1.state = $(this).attr('data-dependent-depstate') + } + }); + })) } }, mounted () { From 3813549a290783cfac5f4143ad49eed558f647a7 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Thu, 30 Apr 2020 10:53:02 +0800 Subject: [PATCH 148/401] fix --- .../pages/dag/_source/formModel/tasks/_source/nodeStatus.vue | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue index 972782207c..549cb119f9 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue @@ -176,8 +176,7 @@ } }) }, - mounted () { - }, + mounted () {}, components: {} } From e8f0fcbbf2cfbb1898ad953a7e320fc9ac13627b Mon Sep 17 00:00:00 2001 From: "mzjnumber1@163.com" Date: Tue, 28 Apr 2020 17:31:14 +0800 Subject: [PATCH 149/401] fix bug "Password display in plain text" --- .../conf/home/pages/security/pages/users/_source/createUser.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue index b2cf89e1c7..c02b205d79 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue @@ -37,7 +37,7 @@