diff --git a/.github/workflows/ci_backend.yml b/.github/workflows/ci_backend.yml index afd241d94e..0273251e99 100644 --- a/.github/workflows/ci_backend.yml +++ b/.github/workflows/ci_backend.yml @@ -45,7 +45,13 @@ jobs: Compile-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: @@ -55,7 +61,13 @@ jobs: License-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml index 13f12641fe..fe818d0668 100644 --- a/.github/workflows/ci_e2e.yml +++ b/.github/workflows/ci_e2e.yml @@ -15,7 +15,7 @@ # limitations under the License. # -on: ["push", "pull_request"] +on: ["pull_request"] env: DOCKER_DIR: ./docker LOG_DIR: /tmp/dolphinscheduler @@ -29,9 +29,13 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - with: - submodules: true + - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - uses: actions/cache@v1 with: path: ~/.m2/repository diff --git a/.github/workflows/ci_frontend.yml b/.github/workflows/ci_frontend.yml index fab75c6341..494d12dbae 100644 --- a/.github/workflows/ci_frontend.yml +++ b/.github/workflows/ci_frontend.yml @@ -34,7 +34,13 @@ jobs: matrix: os: [ubuntu-latest, macos-latest] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - name: Set up Node.js uses: actions/setup-node@v1 with: @@ -49,7 +55,13 @@ jobs: License-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 6f3eda0362..12f7c04ed6 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -15,7 +15,7 @@ # limitations under the License. # -on: ["push", "pull_request"] +on: ["pull_request", "push"] env: DOCKER_DIR: ./docker LOG_DIR: /tmp/dolphinscheduler @@ -29,9 +29,13 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - with: - submodules: true + - uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 - uses: actions/cache@v1 with: path: ~/.m2/repository @@ -48,7 +52,15 @@ jobs: run: | export MAVEN_OPTS='-Dmaven.repo.local=.m2/repository -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:-UseGCOverheadLimit -Xmx3g' mvn test -B -Dmaven.test.skip=false + - name: Upload coverage report to codecov + if: github.event_name == 'pull_request' + run: | CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash) + - name: Git fetch unshallow + run: | + git fetch --unshallow + git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" + git fetch origin - name: Run SonarCloud Analysis run: > mvn verify --batch-mode diff --git a/README.md b/README.md index 0a9a164b85..ebd620efee 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Dolphin Scheduler Official Website ### Design features: -A distributed and easy-to-expand visual DAG workflow scheduling system. Dedicated to solving the complex dependencies in data processing, making the scheduling system `out of the box` for data processing. +A distributed and easy-to-extend visual DAG workflow scheduling system. Dedicated to solving the complex dependencies in data processing, making the scheduling system `out of the box` for data processing. Its main objectives are as follows: - Associate the Tasks according to the dependencies of the tasks in a DAG graph, which can visualize the running state of task in real time. diff --git a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py b/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py index 230ad14565..3780f6c27e 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py +++ b/ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py @@ -30,7 +30,7 @@ sys.setdefaultencoding('utf-8') config = Script.get_config() # conf_dir = "/etc/" -dolphin_home = "/opt/soft/apache-dolphinscheduler-incubating-1.2.1" +dolphin_home = "/opt/soft/dolphinscheduler" dolphin_conf_dir = dolphin_home + "/conf" dolphin_log_dir = dolphin_home + "/logs" dolphin_bin_dir = dolphin_home + "/bin" diff --git a/dockerfile/Dockerfile b/dockerfile/Dockerfile index 1fc064c489..acd2cb99e5 100644 --- a/dockerfile/Dockerfile +++ b/dockerfile/Dockerfile @@ -23,11 +23,11 @@ ENV TZ Asia/Shanghai ENV LANG C.UTF-8 ENV DEBIAN_FRONTEND noninteractive -#1. install dos2unix shadow bash openrc python sudo vim wget iputils net-tools ssh pip kazoo. +#1. install dos2unix shadow bash openrc python sudo vim wget iputils net-tools ssh pip tini kazoo. #If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example: #RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories RUN apk update && \ - apk add dos2unix shadow bash openrc python sudo vim wget iputils net-tools openssh-server py2-pip && \ + apk add dos2unix shadow bash openrc python sudo vim wget iputils net-tools openssh-server py2-pip tini && \ apk add --update procps && \ openrc boot && \ pip install kazoo @@ -63,19 +63,22 @@ RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \ ADD ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d #7. add configuration and modify permissions and set soft links +ADD ./checkpoint.sh /root/checkpoint.sh ADD ./startup-init-conf.sh /root/startup-init-conf.sh ADD ./startup.sh /root/startup.sh ADD ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/ -ADD ./conf/dolphinscheduler/env/dolphinscheduler_env /opt/dolphinscheduler/conf/env/ -RUN chmod +x /root/startup-init-conf.sh && \ +ADD conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/ +RUN chmod +x /root/checkpoint.sh && \ + chmod +x /root/startup-init-conf.sh && \ chmod +x /root/startup.sh && \ - chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env && \ + chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \ chmod +x /opt/dolphinscheduler/script/*.sh && \ chmod +x /opt/dolphinscheduler/bin/*.sh && \ chmod +x /opt/zookeeper/bin/*.sh && \ + dos2unix /root/checkpoint.sh && \ dos2unix /root/startup-init-conf.sh && \ dos2unix /root/startup.sh && \ - dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env && \ + dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \ dos2unix /opt/dolphinscheduler/script/*.sh && \ dos2unix /opt/dolphinscheduler/bin/*.sh && \ dos2unix /opt/zookeeper/bin/*.sh && \ @@ -87,6 +90,6 @@ RUN chmod +x /root/startup-init-conf.sh && \ RUN rm -rf /var/cache/apk/* #9. expose port -EXPOSE 2181 2888 3888 5432 12345 8888 +EXPOSE 2181 2888 3888 5432 12345 50051 8888 -ENTRYPOINT ["/root/startup.sh"] \ No newline at end of file +ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"] \ No newline at end of file diff --git a/dockerfile/README.md b/dockerfile/README.md index b5a9d0d3aa..60af7fad8f 100644 --- a/dockerfile/README.md +++ b/dockerfile/README.md @@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org #### You can start a dolphinscheduler instance ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test \ +-e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \ -p 8888:8888 \ dolphinscheduler all ``` @@ -25,13 +25,13 @@ The default postgres user `root`, postgres password `root` and database `dolphin The default zookeeper is created in the `startup.sh`. -#### Or via Environment Variables **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`ZOOKEEPER_QUORUM`** +#### Or via Environment Variables **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`POSTGRESQL_DATABASE`** **`ZOOKEEPER_QUORUM`** You can specify **existing postgres service**. Example: ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -p 8888:8888 \ dolphinscheduler all @@ -42,7 +42,7 @@ You can specify **existing zookeeper service**. Example: ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -p 8888:8888 \ dolphinscheduler all ``` @@ -56,7 +56,7 @@ You can start a standalone dolphinscheduler server. ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler master-server ``` @@ -66,7 +66,7 @@ dolphinscheduler master-server ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler worker-server ``` @@ -75,7 +75,7 @@ dolphinscheduler worker-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -p 12345:12345 \ dolphinscheduler api-server @@ -85,7 +85,7 @@ dolphinscheduler api-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler alert-server ``` @@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \ dolphinscheduler frontend ``` -**Note**: You must be specify `POSTGRESQL_HOST` `POSTGRESQL_PORT` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server. +**Note**: You must be specify `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server. ## How to build a docker image @@ -140,14 +140,36 @@ This environment variable sets the port for PostgreSQL. The default value is `54 This environment variable sets the username for PostgreSQL. The default value is `root`. +**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. + **`POSTGRESQL_PASSWORD`** This environment variable sets the password for PostgreSQL. The default value is `root`. +**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. + +**`POSTGRESQL_DATABASE`** + +This environment variable sets the database for PostgreSQL. The default value is `dolphinscheduler`. + +**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. + **`DOLPHINSCHEDULER_ENV_PATH`** This environment variable sets the runtime environment for task. The default value is `/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh`. +**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`** + +User data directory path, self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler` + +**`DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH`** + +Directory path for user data download. self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler/download` + +**`DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH`** + +Process execute directory. self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler/exec` + **`TASK_QUEUE`** This environment variable sets the task queue for `master-server` and `worker-serverr`. The default value is `zookeeper`. diff --git a/dockerfile/README_zh_CN.md b/dockerfile/README_zh_CN.md index 5830e30b07..900c8b50d9 100644 --- a/dockerfile/README_zh_CN.md +++ b/dockerfile/README_zh_CN.md @@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org #### 你可以运行一个dolphinscheduler实例 ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test \ +-e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \ -p 8888:8888 \ dolphinscheduler all ``` @@ -31,7 +31,7 @@ dolphinscheduler all ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -p 8888:8888 \ dolphinscheduler all @@ -42,7 +42,7 @@ dolphinscheduler all ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ +-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -p 8888:8888 \ dolphinscheduler all ``` @@ -56,7 +56,7 @@ dolphinscheduler all ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler master-server ``` @@ -66,7 +66,7 @@ dolphinscheduler master-server ``` $ docker run -dit --name dolphinscheduler \ -e ZOOKEEPER_QUORUM="l92.168.x.x:2181" --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler worker-server ``` @@ -75,7 +75,7 @@ dolphinscheduler worker-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -p 12345:12345 \ dolphinscheduler api-server @@ -85,7 +85,7 @@ dolphinscheduler api-server ``` $ docker run -dit --name dolphinscheduler \ --e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ +-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ dolphinscheduler alert-server ``` @@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \ dolphinscheduler frontend ``` -**注意**: 当你运行dolphinscheduler中的部分服务时，你必须指定这些环境变量 `POSTGRESQL_HOST` `POSTGRESQL_PORT` `ZOOKEEPER_QUORUM`。 +**注意**: 当你运行dolphinscheduler中的部分服务时，你必须指定这些环境变量 `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM`。 ## 如何构建一个docker镜像 @@ -140,14 +140,36 @@ Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些 配置`PostgreSQL`的`USERNAME`， 默认值 `root`。 +**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 + **`POSTGRESQL_PASSWORD`** 配置`PostgreSQL`的`PASSWORD`， 默认值 `root`。 +**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 + +**`POSTGRESQL_DATABASE`** + +配置`PostgreSQL`的`DATABASE`， 默认值 `dolphinscheduler`。 + +**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 + **`DOLPHINSCHEDULER_ENV_PATH`** 任务执行时的环境变量配置文件， 默认值 `/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh`。 +**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`** + +用户数据目录, 用户自己配置, 请确保这个目录存在并且用户读写权限， 默认值 `/tmp/dolphinscheduler`。 + +**`DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH`** + +用户数据下载目录, 用户自己配置, 请确保这个目录存在并且用户读写权限， 默认值 `/tmp/dolphinscheduler/download`。 + +**`DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH`** + +任务执行目录, 用户自己配置, 请确保这个目录存在并且用户读写权限， 默认值 `/tmp/dolphinscheduler/exec`。 + **`TASK_QUEUE`** 配置`master-server`和`worker-serverr`的`Zookeeper`任务队列名， 默认值 `zookeeper`。 diff --git a/dockerfile/checkpoint.sh b/dockerfile/checkpoint.sh new file mode 100644 index 0000000000..cd2774f9ce --- /dev/null +++ b/dockerfile/checkpoint.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +if [ "$(ps -ef | grep java | grep -c $1)" -eq 0 ]; then + echo "[ERROR] $1 process not exits." + exit 1 +else + echo "[INFO] $1 process exits." + exit 0 +fi diff --git a/dockerfile/conf/dolphinscheduler/application.properties.tpl b/dockerfile/conf/dolphinscheduler/application.properties.tpl index c643c414cd..6dd8a18e11 100644 --- a/dockerfile/conf/dolphinscheduler/application.properties.tpl +++ b/dockerfile/conf/dolphinscheduler/application.properties.tpl @@ -19,7 +19,7 @@ spring.datasource.type=com.alibaba.druid.pool.DruidDataSource # postgre spring.datasource.driver-class-name=org.postgresql.Driver -spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/dolphinscheduler?characterEncoding=utf8 +spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8 # mysql #spring.datasource.driver-class-name=com.mysql.jdbc.Driver #spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 diff --git a/dockerfile/conf/dolphinscheduler/common.properties.tpl b/dockerfile/conf/dolphinscheduler/common.properties.tpl index ea03e0b78e..8134fc7a9b 100644 --- a/dockerfile/conf/dolphinscheduler/common.properties.tpl +++ b/dockerfile/conf/dolphinscheduler/common.properties.tpl @@ -38,6 +38,12 @@ dolphinscheduler.env.path=${DOLPHINSCHEDULER_ENV_PATH} resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties # is development state? default "false" development.state=true +# user data directory path, self configuration, please make sure the directory exists and have read write permissions +data.basedir.path=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH} +# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions +data.download.basedir.path=${DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH} +# process execute directory. self configuration, please make sure the directory exists and have read write permissions +process.exec.basepath=${DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH} # resource upload startup type : HDFS,S3,NONE res.upload.startup.type=NONE @@ -49,12 +55,6 @@ res.upload.startup.type=NONE hdfs.root.user=hdfs # data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended data.store2hdfs.basepath=/dolphinscheduler -# user data directory path, self configuration, please make sure the directory exists and have read write permissions -data.basedir.path=/tmp/dolphinscheduler -# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions -data.download.basedir.path=/tmp/dolphinscheduler/download -# process execute directory. self configuration, please make sure the directory exists and have read write permissions -process.exec.basepath=/tmp/dolphinscheduler/exec # whether kerberos starts hadoop.security.authentication.startup.state=false # java.security.krb5.conf path diff --git a/dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env b/dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh similarity index 100% rename from dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env rename to dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh diff --git a/dockerfile/conf/dolphinscheduler/quartz.properties.tpl b/dockerfile/conf/dolphinscheduler/quartz.properties.tpl index de5496bc3d..7c7c92e425 100644 --- a/dockerfile/conf/dolphinscheduler/quartz.properties.tpl +++ b/dockerfile/conf/dolphinscheduler/quartz.properties.tpl @@ -22,7 +22,7 @@ org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.PostgreSQLDelegate # postgre org.quartz.dataSource.myDs.driver = org.postgresql.Driver -org.quartz.dataSource.myDs.URL = jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/dolphinscheduler?characterEncoding=utf8 +org.quartz.dataSource.myDs.URL = jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8 org.quartz.dataSource.myDs.user = ${POSTGRESQL_USERNAME} org.quartz.dataSource.myDs.password = ${POSTGRESQL_PASSWORD} org.quartz.scheduler.instanceName = DolphinScheduler diff --git a/dockerfile/hooks/check b/dockerfile/hooks/check index fbc3f8bd44..21f3f4db12 100644 --- a/dockerfile/hooks/check +++ b/dockerfile/hooks/check @@ -17,7 +17,7 @@ # echo "------ dolphinscheduler check - server - status -------" sleep 20 -server_num=$(docker top `docker container list | grep startup | awk '{print $1}'`| grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) +server_num=$(docker top `docker container list | grep '/sbin/tini' | awk '{print $1}'`| grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l) if [ $server_num -eq 5 ] then echo "Server all start successfully" diff --git a/dockerfile/startup-init-conf.sh b/dockerfile/startup-init-conf.sh index db37976168..d2b50fa3e7 100644 --- a/dockerfile/startup-init-conf.sh +++ b/dockerfile/startup-init-conf.sh @@ -28,11 +28,15 @@ export POSTGRESQL_HOST=${POSTGRESQL_HOST:-"127.0.0.1"} export POSTGRESQL_PORT=${POSTGRESQL_PORT:-"5432"} export POSTGRESQL_USERNAME=${POSTGRESQL_USERNAME:-"root"} export POSTGRESQL_PASSWORD=${POSTGRESQL_PASSWORD:-"root"} +export POSTGRESQL_DATABASE=${POSTGRESQL_DATABASE:-"dolphinscheduler"} #============================================================================ # System #============================================================================ export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"} +export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"} +export DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH:-"/tmp/dolphinscheduler/download"} +export DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH=${DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH:-"/tmp/dolphinscheduler/exec"} #============================================================================ # Zookeeper diff --git a/dockerfile/startup.sh b/dockerfile/startup.sh index 610a86f5d3..30439c2321 100644 --- a/dockerfile/startup.sh +++ b/dockerfile/startup.sh @@ -164,6 +164,7 @@ case "$1" in LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log ;; (api-server) + initZK initPostgreSQL initApiServer LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log @@ -187,6 +188,9 @@ case "$1" in ;; esac -echo "tee begin" -exec tee ${LOGFILE} +# init directories and log files +mkdir -p ${DOLPHINSCHEDULER_LOGS} && mkdir -p /var/log/nginx/ && cat /dev/null >> ${LOGFILE} + +echo "tail begin" +exec bash -c "tail -n 1 -f ${LOGFILE}" diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java index 9bcad56c24..bb06be6561 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java @@ -42,8 +42,8 @@ public class EnterpriseWeChatManager { public Map send(Alert alert, String token){ Map retMap = new HashMap<>(); retMap.put(Constants.STATUS, false); - String agentId = EnterpriseWeChatUtils.enterpriseWeChatAgentId; - String users = EnterpriseWeChatUtils.enterpriseWeChatUsers; + String agentId = EnterpriseWeChatUtils.ENTERPRISE_WE_CHAT_AGENT_ID; + String users = EnterpriseWeChatUtils.ENTERPRISE_WE_CHAT_USERS; List userList = Arrays.asList(users.split(",")); logger.info("send message {}",alert); String msg = EnterpriseWeChatUtils.makeUserSendMsg(userList, agentId,EnterpriseWeChatUtils.markdownByAlert(alert)); diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java index 428fa4cb62..79123a1e80 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.alert.template.impl; import org.apache.dolphinscheduler.alert.template.AlertTemplate; import org.apache.dolphinscheduler.alert.utils.Constants; import org.apache.dolphinscheduler.alert.utils.JSONUtils; -import org.apache.dolphinscheduler.alert.utils.MailUtils; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java index ff8822421a..900c120cd4 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java @@ -43,24 +43,24 @@ public class EnterpriseWeChatUtils { public static final Logger logger = LoggerFactory.getLogger(EnterpriseWeChatUtils.class); - private static final String enterpriseWeChatCorpId = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_CORP_ID); + private static final String ENTERPRISE_WE_CHAT_CORP_ID = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_CORP_ID); - private static final String enterpriseWeChatSecret = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_SECRET); + private static final String ENTERPRISE_WE_CHAT_SECRET = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_SECRET); - private static final String enterpriseWeChatTokenUrl = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TOKEN_URL); - private static String enterpriseWeChatTokenUrlReplace = enterpriseWeChatTokenUrl - .replaceAll("\\$corpId", enterpriseWeChatCorpId) - .replaceAll("\\$secret", enterpriseWeChatSecret); + private static final String ENTERPRISE_WE_CHAT_TOKEN_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TOKEN_URL); + private static final String ENTERPRISE_WE_CHAT_TOKEN_URL_REPLACE = ENTERPRISE_WE_CHAT_TOKEN_URL + .replaceAll("\\$corpId", ENTERPRISE_WE_CHAT_CORP_ID) + .replaceAll("\\$secret", ENTERPRISE_WE_CHAT_SECRET); - private static final String enterpriseWeChatPushUrl = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_PUSH_URL); + private static final String ENTERPRISE_WE_CHAT_PUSH_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_PUSH_URL); - private static final String enterpriseWeChatTeamSendMsg = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TEAM_SEND_MSG); + private static final String ENTERPRISE_WE_CHAT_TEAM_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TEAM_SEND_MSG); - private static final String enterpriseWeChatUserSendMsg = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USER_SEND_MSG); + private static final String ENTERPRISE_WE_CHAT_USER_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USER_SEND_MSG); - public static final String enterpriseWeChatAgentId = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_AGENT_ID); + public static final String ENTERPRISE_WE_CHAT_AGENT_ID = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_AGENT_ID); - public static final String enterpriseWeChatUsers = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USERS); + public static final String ENTERPRISE_WE_CHAT_USERS = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USERS); /** * get Enterprise WeChat is enable @@ -87,7 +87,7 @@ public class EnterpriseWeChatUtils { CloseableHttpClient httpClient = HttpClients.createDefault(); try { - HttpGet httpGet = new HttpGet(enterpriseWeChatTokenUrlReplace); + HttpGet httpGet = new HttpGet(ENTERPRISE_WE_CHAT_TOKEN_URL_REPLACE); CloseableHttpResponse response = httpClient.execute(httpGet); try { HttpEntity entity = response.getEntity(); @@ -114,7 +114,7 @@ public class EnterpriseWeChatUtils { * @return Enterprise WeChat send message */ public static String makeTeamSendMsg(String toParty, String agentId, String msg) { - return enterpriseWeChatTeamSendMsg.replaceAll("\\$toParty", toParty) + return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\$toParty", toParty) .replaceAll("\\$agentId", agentId) .replaceAll("\\$msg", msg); } @@ -128,7 +128,7 @@ public class EnterpriseWeChatUtils { */ public static String makeTeamSendMsg(Collection toParty, String agentId, String msg) { String listParty = FuncUtils.mkString(toParty, "|"); - return enterpriseWeChatTeamSendMsg.replaceAll("\\$toParty", listParty) + return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\$toParty", listParty) .replaceAll("\\$agentId", agentId) .replaceAll("\\$msg", msg); } @@ -141,7 +141,7 @@ public class EnterpriseWeChatUtils { * @return Enterprise WeChat send message */ public static String makeUserSendMsg(String toUser, String agentId, String msg) { - return enterpriseWeChatUserSendMsg.replaceAll("\\$toUser", toUser) + return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\$toUser", toUser) .replaceAll("\\$agentId", agentId) .replaceAll("\\$msg", msg); } @@ -155,7 +155,7 @@ public class EnterpriseWeChatUtils { */ public static String makeUserSendMsg(Collection toUser, String agentId, String msg) { String listUser = FuncUtils.mkString(toUser, "|"); - return enterpriseWeChatUserSendMsg.replaceAll("\\$toUser", listUser) + return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\$toUser", listUser) .replaceAll("\\$agentId", agentId) .replaceAll("\\$msg", msg); } @@ -169,7 +169,7 @@ public class EnterpriseWeChatUtils { * @throws IOException the IOException */ public static String sendEnterpriseWeChat(String charset, String data, String token) throws IOException { - String enterpriseWeChatPushUrlReplace = enterpriseWeChatPushUrl.replaceAll("\\$token", token); + String enterpriseWeChatPushUrlReplace = ENTERPRISE_WE_CHAT_PUSH_URL.replaceAll("\\$token", token); CloseableHttpClient httpClient = HttpClients.createDefault(); try { @@ -184,8 +184,8 @@ public class EnterpriseWeChatUtils { } finally { response.close(); } - logger.info("Enterprise WeChat send [{}], param:{}, resp:{}", - enterpriseWeChatPushUrl, data, resp); + logger.info("Enterprise WeChat send [{}], param:{}, resp:{}", + ENTERPRISE_WE_CHAT_PUSH_URL, data, resp); return resp; } finally { httpClient.close(); diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java index dd6ca4b8a6..d68532a82b 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java @@ -20,7 +20,7 @@ import org.apache.dolphinscheduler.common.utils.StringUtils; public class FuncUtils { - static public String mkString(Iterable list, String split) { + public static String mkString(Iterable list, String split) { if (null == list || StringUtils.isEmpty(split)){ return null; diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java index 1cd9f490b2..5f8371de2d 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java @@ -16,12 +16,13 @@ */ package org.apache.dolphinscheduler.alert.utils; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; -import com.alibaba.fastjson.JSONObject; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collections; import java.util.List; /** @@ -38,7 +39,7 @@ public class JSONUtils { */ public static String toJsonString(Object object) { try{ - return JSONObject.toJSONString(object,false); + return JSON.toJSONString(object,false); } catch (Exception e) { throw new RuntimeException("Json deserialization exception.", e); } @@ -50,19 +51,19 @@ public class JSONUtils { * @param json the json * @param clazz c * @param the generic clazz - * @return the result list + * @return the result list or empty list */ public static List toList(String json, Class clazz) { if (StringUtils.isEmpty(json)) { - return null; + return Collections.emptyList(); } try { - return JSONArray.parseArray(json, clazz); + return JSON.parseArray(json, clazz); } catch (Exception e) { logger.error("JSONArray.parseArray exception!",e); } - return null; + return Collections.emptyList(); } } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java index 99efdc8a6a..66a63dcff4 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java @@ -39,29 +39,29 @@ public class MailUtils { public static final Logger logger = LoggerFactory.getLogger(MailUtils.class); - public static final String mailProtocol = PropertyUtils.getString(Constants.MAIL_PROTOCOL); + public static final String MAIL_PROTOCOL = PropertyUtils.getString(Constants.MAIL_PROTOCOL); - public static final String mailServerHost = PropertyUtils.getString(Constants.MAIL_SERVER_HOST); + public static final String MAIL_SERVER_HOST = PropertyUtils.getString(Constants.MAIL_SERVER_HOST); - public static final Integer mailServerPort = PropertyUtils.getInt(Constants.MAIL_SERVER_PORT); + public static final Integer MAIL_SERVER_PORT = PropertyUtils.getInt(Constants.MAIL_SERVER_PORT); - public static final String mailSender = PropertyUtils.getString(Constants.MAIL_SENDER); + public static final String MAIL_SENDER = PropertyUtils.getString(Constants.MAIL_SENDER); - public static final String mailUser = PropertyUtils.getString(Constants.MAIL_USER); + public static final String MAIL_USER = PropertyUtils.getString(Constants.MAIL_USER); - public static final String mailPasswd = PropertyUtils.getString(Constants.MAIL_PASSWD); + public static final String MAIL_PASSWD = PropertyUtils.getString(Constants.MAIL_PASSWD); - public static final Boolean mailUseStartTLS = PropertyUtils.getBoolean(Constants.MAIL_SMTP_STARTTLS_ENABLE); + public static final Boolean MAIL_USE_START_TLS = PropertyUtils.getBoolean(Constants.MAIL_SMTP_STARTTLS_ENABLE); - public static final Boolean mailUseSSL = PropertyUtils.getBoolean(Constants.MAIL_SMTP_SSL_ENABLE); + public static final Boolean MAIL_USE_SSL = PropertyUtils.getBoolean(Constants.MAIL_SMTP_SSL_ENABLE); - public static final String xlsFilePath = PropertyUtils.getString(Constants.XLS_FILE_PATH); + public static final String XLS_FILE_PATH = PropertyUtils.getString(Constants.XLS_FILE_PATH); - public static final String starttlsEnable = PropertyUtils.getString(Constants.MAIL_SMTP_STARTTLS_ENABLE); + public static final String STARTTLS_ENABLE = PropertyUtils.getString(Constants.MAIL_SMTP_STARTTLS_ENABLE); - public static final String sslEnable = PropertyUtils.getString(Constants.MAIL_SMTP_SSL_ENABLE); + public static final String SSL_ENABLE = PropertyUtils.getString(Constants.MAIL_SMTP_SSL_ENABLE); - public static final String sslTrust = PropertyUtils.getString(Constants.MAIL_SMTP_SSL_TRUST); + public static final String SSL_TRUST = PropertyUtils.getString(Constants.MAIL_SMTP_SSL_TRUST); public static final AlertTemplate alertTemplate = AlertTemplateFactory.getMessageTemplate(); @@ -105,7 +105,7 @@ public class MailUtils { try { Session session = getSession(); email.setMailSession(session); - email.setFrom(mailSender); + email.setFrom(MAIL_SENDER); email.setCharset(Constants.UTF_8); if (CollectionUtils.isNotEmpty(receivers)){ // receivers mail @@ -199,10 +199,10 @@ public class MailUtils { // 2. creating mail: Creating a MimeMessage MimeMessage msg = new MimeMessage(session); // 3. set sender - msg.setFrom(new InternetAddress(mailSender)); + msg.setFrom(new InternetAddress(MAIL_SENDER)); // 4. set receivers for (String receiver : receivers) { - msg.addRecipients(MimeMessage.RecipientType.TO, InternetAddress.parse(receiver)); + msg.addRecipients(Message.RecipientType.TO, InternetAddress.parse(receiver)); } return msg; } @@ -213,19 +213,19 @@ public class MailUtils { */ private static Session getSession() { Properties props = new Properties(); - props.setProperty(Constants.MAIL_HOST, mailServerHost); - props.setProperty(Constants.MAIL_PORT, String.valueOf(mailServerPort)); + props.setProperty(Constants.MAIL_HOST, MAIL_SERVER_HOST); + props.setProperty(Constants.MAIL_PORT, String.valueOf(MAIL_SERVER_PORT)); props.setProperty(Constants.MAIL_SMTP_AUTH, Constants.STRING_TRUE); - props.setProperty(Constants.MAIL_TRANSPORT_PROTOCOL, mailProtocol); - props.setProperty(Constants.MAIL_SMTP_STARTTLS_ENABLE, starttlsEnable); - props.setProperty(Constants.MAIL_SMTP_SSL_ENABLE, sslEnable); - props.setProperty(Constants.MAIL_SMTP_SSL_TRUST, sslTrust); + props.setProperty(Constants.MAIL_TRANSPORT_PROTOCOL, MAIL_PROTOCOL); + props.setProperty(Constants.MAIL_SMTP_STARTTLS_ENABLE, STARTTLS_ENABLE); + props.setProperty(Constants.MAIL_SMTP_SSL_ENABLE, SSL_ENABLE); + props.setProperty(Constants.MAIL_SMTP_SSL_TRUST, SSL_TRUST); Authenticator auth = new Authenticator() { @Override protected PasswordAuthentication getPasswordAuthentication() { // mail username and password - return new PasswordAuthentication(mailUser, mailPasswd); + return new PasswordAuthentication(MAIL_USER, MAIL_PASSWD); } }; @@ -248,12 +248,10 @@ public class MailUtils { */ if(CollectionUtils.isNotEmpty(receiversCc)){ for (String receiverCc : receiversCc){ - msg.addRecipients(MimeMessage.RecipientType.CC, InternetAddress.parse(receiverCc)); + msg.addRecipients(Message.RecipientType.CC, InternetAddress.parse(receiverCc)); } } - // set receivers type to cc - // msg.addRecipients(MimeMessage.RecipientType.CC, InternetAddress.parse(propMap.get("${CC}"))); // set subject msg.setSubject(title); MimeMultipart partList = new MimeMultipart(); @@ -263,8 +261,8 @@ public class MailUtils { // set attach file MimeBodyPart part2 = new MimeBodyPart(); // make excel file - ExcelUtils.genExcelFile(content,title,xlsFilePath); - File file = new File(xlsFilePath + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS); + ExcelUtils.genExcelFile(content,title, XLS_FILE_PATH); + File file = new File(XLS_FILE_PATH + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS); part2.attachFile(file); part2.setFileName(MimeUtility.encodeText(title + Constants.EXCEL_SUFFIX_XLS,Constants.UTF_8,"B")); // add components to collection @@ -334,7 +332,7 @@ public class MailUtils { * @param e the exception */ private static void handleException(Collection receivers, Map retMap, Exception e) { - logger.error("Send email to {} failed {}", receivers, e); + logger.error("Send email to {} failed", receivers, e); retMap.put(Constants.MESSAGE, "Send email to {" + String.join(",", receivers) + "} failed，" + e.toString()); } diff --git a/dolphinscheduler-alert/src/main/resources/alert.properties b/dolphinscheduler-alert/src/main/resources/alert.properties index 000d0653b7..9f5acea188 100644 --- a/dolphinscheduler-alert/src/main/resources/alert.properties +++ b/dolphinscheduler-alert/src/main/resources/alert.properties @@ -28,7 +28,6 @@ mail.server.port=25 mail.sender=xxx@xxx.com mail.user=xxx@xxx.com mail.passwd=111111 - # TLS mail.smtp.starttls.enable=true # SSL diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java index 3471f6efdd..15b92a622e 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java @@ -53,7 +53,7 @@ public class EnterpriseWeChatUtilsTest { String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); String errmsg = JSON.parseObject(resp).getString("errmsg"); - Assert.assertEquals(errmsg, "ok"); + Assert.assertEquals("ok",errmsg); } catch (IOException e) { e.printStackTrace(); } @@ -68,7 +68,7 @@ public class EnterpriseWeChatUtilsTest { String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); String errmsg = JSON.parseObject(resp).getString("errmsg"); - Assert.assertEquals(errmsg, "ok"); + Assert.assertEquals("ok",errmsg); } catch (IOException e) { e.printStackTrace(); } @@ -95,7 +95,7 @@ public class EnterpriseWeChatUtilsTest { String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); String errmsg = JSON.parseObject(resp).getString("errmsg"); - Assert.assertEquals(errmsg, "ok"); + Assert.assertEquals("ok",errmsg); } catch (IOException e) { e.printStackTrace(); } @@ -110,7 +110,7 @@ public class EnterpriseWeChatUtilsTest { String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); String errmsg = JSON.parseObject(resp).getString("errmsg"); - Assert.assertEquals(errmsg, "ok"); + Assert.assertEquals("ok",errmsg); } catch (IOException e) { e.printStackTrace(); } diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java index e2b25d961d..a4aeea9c0c 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java @@ -46,7 +46,7 @@ public class FuncUtilsTest { logger.info(result); //Expected result string - assertEquals(result, "user1|user2|user3"); + assertEquals("user1|user2|user3", result); //Null list expected return null result = FuncUtils.mkString(null, split); diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java index cb63a22d79..a151abc714 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.alert.utils; -import org.junit.After; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; @@ -27,8 +26,7 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.Assert.*; public class JSONUtilsTest { @@ -74,7 +72,7 @@ public class JSONUtilsTest { result = JSONUtils.toJsonString(null); logger.info(result); - assertEquals(result,"null"); + assertEquals("null", result); } @@ -87,25 +85,27 @@ public class JSONUtilsTest { //Invoke toList List result = JSONUtils.toList(expected ,LinkedHashMap.class); //Equal list size=1 - assertEquals(result.size(),1); + assertEquals(1,result.size()); //Transform entity to LinkedHashMap LinkedHashMap entity = result.get(0); //Equal expected values - assertEquals(entity.get("mysql service name"),"mysql200"); - assertEquals(entity.get("mysql address"),"192.168.xx.xx"); - assertEquals(entity.get("port"),"3306"); - assertEquals(entity.get("no index of number"),"80"); - assertEquals(entity.get("database client connections"),"190"); + assertEquals("mysql200",entity.get("mysql service name")); + assertEquals("192.168.xx.xx", entity.get("mysql address")); + assertEquals("3306", entity.get("port")); + assertEquals("80", entity.get("no index of number")); + assertEquals("190", entity.get("database client connections")); - //If param is null, then return null + //If param is null, then return empty list result = JSONUtils.toList(null ,LinkedHashMap.class); - assertNull(result); + assertNotNull(result); + assertTrue(result.isEmpty()); - //If param is incorrect, then return null and log error message + //If param is incorrect, then return empty list and log error message result = JSONUtils.toList("}{" ,LinkedHashMap.class); - assertNull(result); + assertNotNull(result); + assertTrue(result.isEmpty()); } diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 41971734ec..b7c3f3da69 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -140,6 +140,12 @@ org.apache.curator curator-recipes + + + org.apache.zookeeper + zookeeper + + diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java index 8376c2876d..5998ec5a4d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java @@ -25,7 +25,9 @@ import springfox.documentation.swagger2.annotations.EnableSwagger2; @SpringBootApplication @ServletComponentScan -@ComponentScan("org.apache.dolphinscheduler") +@ComponentScan({"org.apache.dolphinscheduler.api", + "org.apache.dolphinscheduler.dao", + "org.apache.dolphinscheduler.service"}) public class ApiApplicationServer extends SpringBootServletInitializer { public static void main(String[] args) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java index 1b1dc65e01..f93e7d6944 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java @@ -103,7 +103,7 @@ public class DataAnalysisController extends BaseController{ @RequestParam(value="endDate", required=false) String endDate, @RequestParam(value="projectId", required=false, defaultValue = "0") int projectId){ try{ - logger.info("count process instance state, user:{}, start date: {}, end date:{}, project id", + logger.info("count process instance state, user:{}, start date: {}, end date:{}, project id:{}", loginUser.getUserName(), startDate, endDate, projectId); Map result = dataAnalysisService.countProcessInstanceStateByProject(loginUser, projectId, startDate, endDate); return returnDataList(result); @@ -129,7 +129,7 @@ public class DataAnalysisController extends BaseController{ public Result countDefinitionByUser(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value="projectId", required=false, defaultValue = "0") int projectId){ try{ - logger.info("count process definition , user:{}, project id", + logger.info("count process definition , user:{}, project id:{}", loginUser.getUserName(), projectId); Map result = dataAnalysisService.countDefinitionByUser(loginUser, projectId); return returnDataList(result); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java index 93b095ddf8..ffedd5703c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java @@ -149,7 +149,7 @@ public class ExecutorController extends BaseController { ) { try { logger.info("execute command, login user: {}, project:{}, process instance id:{}, execute type:{}", - loginUser.getUserName(), projectName, processInstanceId, executeType.toString()); + loginUser.getUserName(), projectName, processInstanceId, executeType); Map result = execService.execute(loginUser, projectName, processInstanceId, executeType); return returnDataList(result); } catch (Exception e) { @@ -173,7 +173,7 @@ public class ExecutorController extends BaseController { @ResponseStatus(HttpStatus.OK) public Result startCheckProcessDefinition(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "processDefinitionId") int processDefinitionId) { - logger.info("login user {}, check process definition", loginUser.getUserName(), processDefinitionId); + logger.info("login user {}, check process definition {}", loginUser.getUserName(), processDefinitionId); try { Map result = execService.startCheckByProcessDefinedId(processDefinitionId); return returnDataList(result); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java index 380eea5774..98bac42f72 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java @@ -27,7 +27,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.servlet.HandlerInterceptor; -import org.springframework.web.servlet.ModelAndView; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -90,14 +89,4 @@ public class LoginHandlerInterceptor implements HandlerInterceptor { return true; } - @Override - public void postHandle(HttpServletRequest request, HttpServletResponse response, Object handler, ModelAndView modelAndView) throws Exception { - - } - - @Override - public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) throws Exception { - - } - } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java index f6d8903dd8..6a732fed0e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.api.service; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; @@ -303,7 +304,7 @@ public class DataSourceService extends BaseService{ for (DataSource dataSource : dataSourceList) { String connectionParams = dataSource.getConnectionParams(); - JSONObject object = JSONObject.parseObject(connectionParams); + JSONObject object = JSON.parseObject(connectionParams); object.put(Constants.PASSWORD, Constants.XXXXXX); dataSource.setConnectionParams(JSONUtils.toJson(object)); @@ -367,11 +368,11 @@ public class DataSourceService extends BaseService{ try { switch (dbType) { case POSTGRESQL: - datasource = JSONObject.parseObject(parameter, PostgreDataSource.class); + datasource = JSON.parseObject(parameter, PostgreDataSource.class); Class.forName(Constants.ORG_POSTGRESQL_DRIVER); break; case MYSQL: - datasource = JSONObject.parseObject(parameter, MySQLDataSource.class); + datasource = JSON.parseObject(parameter, MySQLDataSource.class); Class.forName(Constants.COM_MYSQL_JDBC_DRIVER); break; case HIVE: @@ -386,26 +387,26 @@ public class DataSourceService extends BaseService{ getString(org.apache.dolphinscheduler.common.Constants.LOGIN_USER_KEY_TAB_PATH)); } if (dbType == DbType.HIVE){ - datasource = JSONObject.parseObject(parameter, HiveDataSource.class); + datasource = JSON.parseObject(parameter, HiveDataSource.class); }else if (dbType == DbType.SPARK){ - datasource = JSONObject.parseObject(parameter, SparkDataSource.class); + datasource = JSON.parseObject(parameter, SparkDataSource.class); } Class.forName(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER); break; case CLICKHOUSE: - datasource = JSONObject.parseObject(parameter, ClickHouseDataSource.class); + datasource = JSON.parseObject(parameter, ClickHouseDataSource.class); Class.forName(Constants.COM_CLICKHOUSE_JDBC_DRIVER); break; case ORACLE: - datasource = JSONObject.parseObject(parameter, OracleDataSource.class); + datasource = JSON.parseObject(parameter, OracleDataSource.class); Class.forName(Constants.COM_ORACLE_JDBC_DRIVER); break; case SQLSERVER: - datasource = JSONObject.parseObject(parameter, SQLServerDataSource.class); + datasource = JSON.parseObject(parameter, SQLServerDataSource.class); Class.forName(Constants.COM_SQLSERVER_JDBC_DRIVER); break; case DB2: - datasource = JSONObject.parseObject(parameter, DB2ServerDataSource.class); + datasource = JSON.parseObject(parameter, DB2ServerDataSource.class); Class.forName(Constants.COM_DB2_JDBC_DRIVER); break; default: @@ -507,7 +508,7 @@ public class DataSourceService extends BaseService{ parameterMap.put(Constants.PRINCIPAL,principal); } if (other != null && !"".equals(other)) { - LinkedHashMap map = JSONObject.parseObject(other, new TypeReference>() { + LinkedHashMap map = JSON.parseObject(other, new TypeReference>() { }); if (map.size() > 0) { StringBuilder otherSb = new StringBuilder(); @@ -523,9 +524,9 @@ public class DataSourceService extends BaseService{ } if(logger.isDebugEnabled()){ - logger.info("parameters map-----" + JSONObject.toJSONString(parameterMap)); + logger.info("parameters map-----" + JSON.toJSONString(parameterMap)); } - return JSONObject.toJSONString(parameterMap); + return JSON.toJSONString(parameterMap); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index 152292a21b..86b507f0a0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -259,10 +259,7 @@ public class ExecutorService extends BaseService{ // checkTenantExists(); Tenant tenant = processService.getTenantForProcess(processDefinition.getTenantId(), processDefinition.getUserId()); - if(tenant == null){ - return false; - } - return true; + return tenant != null; } /** @@ -298,6 +295,7 @@ public class ExecutorService extends BaseService{ if (executionStatus.typeIsPause()|| executionStatus.typeIsCancel()) { checkResult = true; } + break; default: break; } @@ -369,7 +367,7 @@ public class ExecutorService extends BaseService{ * @return check result code */ public Map startCheckByProcessDefinedId(int processDefineId) { - Map result = new HashMap(); + Map result = new HashMap<>(); if (processDefineId == 0){ logger.error("process definition id is null"); @@ -378,10 +376,9 @@ public class ExecutorService extends BaseService{ List ids = new ArrayList<>(); processService.recurseFindSubProcessId(processDefineId, ids); Integer[] idArray = ids.toArray(new Integer[ids.size()]); - if (ids.size() > 0){ - List processDefinitionList; - processDefinitionList = processDefinitionMapper.queryDefinitionListByIdList(idArray); - if (processDefinitionList != null && processDefinitionList.size() > 0){ + if (!ids.isEmpty()){ + List processDefinitionList = processDefinitionMapper.queryDefinitionListByIdList(idArray); + if (processDefinitionList != null){ for (ProcessDefinition processDefinition : processDefinitionList){ /** * if there is no online process, exit directly diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java index 118c5ce936..3370961fd4 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java @@ -28,7 +28,6 @@ import org.apache.dolphinscheduler.dao.entity.ZookeeperRecord; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java index 4f81d89505..3530c66ff6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.api.service; +import java.nio.charset.StandardCharsets; import org.apache.dolphinscheduler.api.dto.gantt.GanttDto; import org.apache.dolphinscheduler.api.dto.gantt.Task; import org.apache.dolphinscheduler.api.enums.Status; @@ -49,7 +50,6 @@ import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStreamReader; -import java.nio.charset.Charset; import java.text.ParseException; import java.util.*; import java.util.stream.Collectors; @@ -273,7 +273,8 @@ public class ProcessInstanceService extends BaseDAGService { return resultMap; } - BufferedReader br = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(log.getBytes(Charset.forName("utf8"))), Charset.forName("utf8"))); + BufferedReader br = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(log.getBytes( + StandardCharsets.UTF_8)), StandardCharsets.UTF_8)); String line; while ((line = br.readLine()) != null) { if(line.contains(DEPENDENT_SPLIT)){ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java index 5aa6be858a..a33754397d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java @@ -121,7 +121,7 @@ public class ZooKeeperState { private class SendThread extends Thread { private String cmd; - public String ret = ""; + private String ret = ""; public SendThread(String cmd) { this.cmd = cmd; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java index bddc055de3..f5a28d01ae 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java @@ -28,7 +28,6 @@ import org.springframework.test.web.servlet.MvcResult; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java index 8fc055daf1..41674d3e54 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import com.alibaba.fastjson.JSONObject; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java index bab82df59d..42cdd1705a 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java @@ -29,8 +29,6 @@ import org.springframework.test.web.servlet.MvcResult; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; -import javax.ws.rs.POST; - import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java index 46d85f4d8d..a56e3f83ef 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java @@ -16,15 +16,14 @@ */ package org.apache.dolphinscheduler.api.controller; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.utils.JSONUtils; import com.alibaba.fastjson.JSONObject; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,7 +55,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -80,7 +79,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -283,7 +282,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -305,7 +304,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -326,7 +325,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -346,7 +345,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -367,7 +366,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -388,7 +387,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -408,7 +407,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -429,7 +428,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -448,7 +447,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); result.getCode().equals(Status.SUCCESS.getCode()); - JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java index 8bddb0f905..943e14607b 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java @@ -18,9 +18,6 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.enums.FailureStrategy; -import org.apache.dolphinscheduler.common.enums.Priority; -import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.junit.Assert; import org.junit.Test; @@ -31,9 +28,7 @@ import org.springframework.test.web.servlet.MvcResult; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; -import static org.junit.Assert.*; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java index 65ecd3f759..8517ad2d45 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java @@ -29,7 +29,6 @@ import org.springframework.test.web.servlet.MvcResult; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; -import static org.junit.Assert.*; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java index 31c8c0222d..d6fb6b219c 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java @@ -175,6 +175,14 @@ public class TenantServiceTest { logger.info(result.toString()); List tenantList = (List) result.get(Constants.DATA_LIST); Assert.assertTrue(CollectionUtils.isNotEmpty(tenantList)); + + Mockito.when( tenantMapper.queryByTenantCode("1")).thenReturn(getList()); + Map successRes = tenantService.queryTenantList("1"); + Assert.assertEquals(Status.SUCCESS,successRes.get(Constants.STATUS)); + + Mockito.when( tenantMapper.queryByTenantCode("1")).thenReturn(null); + Map tenantNotExistRes = tenantService.queryTenantList("1"); + Assert.assertEquals(Status.TENANT_NOT_EXIST,tenantNotExistRes.get(Constants.STATUS)); } @Test diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMainTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMainTest.java index e8adc6ca9c..69d1f21c37 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMainTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMainTest.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.utils; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitorUtilsTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitorUtilsTest.java index b3626fa8a9..0d89d4b6e3 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitorUtilsTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitorUtilsTest.java @@ -28,7 +28,7 @@ public class ZookeeperMonitorUtilsTest { @Test - public void testGetMasterLsit(){ + public void testGetMasterList(){ ZookeeperMonitor zookeeperMonitor = new ZookeeperMonitor(); diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index e7789f724b..ca75a84a62 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -25,7 +25,7 @@ dolphinscheduler-common dolphinscheduler-common - http://maven.apache.org + jar UTF-8 diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 73125f4926..b0a7b74d39 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -746,7 +746,7 @@ public final class Constants { * application regex */ public static final String APPLICATION_REGEX = "application_\\d+_\\d+"; - public static final String PID = "pid"; + public static final String PID = OSUtils.isWindows() ? "handle" : "pid"; /** * month_begin */ @@ -813,6 +813,11 @@ public final class Constants { */ public static final String KERBEROS = "kerberos"; + /** + * kerberos expire time + */ + public static final String KERBEROS_EXPIRE_TIME = "kerberos.expire.time"; + /** * java.security.krb5.conf */ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/WorkerLogFilter.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/WorkerLogFilter.java index abcc8bc619..6240ed9a2e 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/WorkerLogFilter.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/WorkerLogFilter.java @@ -20,9 +20,6 @@ import ch.qos.logback.classic.Level; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.filter.Filter; import ch.qos.logback.core.spi.FilterReply; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; - -import java.util.Arrays; /** * worker log filter diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java index b45bd8aeb8..c0ad907dca 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.common.model; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; @@ -23,7 +24,6 @@ import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import com.alibaba.fastjson.JSONObject; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -294,7 +294,7 @@ public class TaskNode { if(StringUtils.isNotEmpty(this.getTimeout())){ String formatStr = String.format("%s,%s", TaskTimeoutStrategy.WARN.name(), TaskTimeoutStrategy.FAILED.name()); String timeout = this.getTimeout().replace(formatStr,TaskTimeoutStrategy.WARNFAILED.name()); - return JSONObject.parseObject(timeout,TaskTimeoutParameter.class); + return JSON.parseObject(timeout,TaskTimeoutParameter.class); } return new TaskTimeoutParameter(false); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java index 101da18b2c..f846b19741 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java @@ -335,7 +335,7 @@ public abstract class AbstractShell { try{ entry.getValue().destroy(); } catch (Exception e) { - e.printStackTrace(); + logger.error("Destroy All Processes error", e); } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java index 0638b3858e..1fbd9ab354 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -207,12 +208,15 @@ public class FlinkParameters extends AbstractParameters { @Override public List getResourceFilesList() { - if(resourceList !=null ) { - this.resourceList.add(mainJar); - return resourceList.stream() - .map(p -> p.getRes()).collect(Collectors.toList()); + if(resourceList != null ) { + List resourceFiles = resourceList.stream() + .map(ResourceInfo::getRes).collect(Collectors.toList()); + if(mainJar != null) { + resourceFiles.add(mainJar.getRes()); + } + return resourceFiles; } - return null; + return Collections.emptyList(); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/mr/MapreduceParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/mr/MapreduceParameters.java index b8fd6ebcbf..31c9c7292f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/mr/MapreduceParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/mr/MapreduceParameters.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -125,12 +126,15 @@ public class MapreduceParameters extends AbstractParameters { @Override public List getResourceFilesList() { - if (resourceList != null) { - this.resourceList.add(mainJar); - return resourceList.stream() - .map(p -> p.getRes()).collect(Collectors.toList()); + if(resourceList != null ) { + List resourceFiles = resourceList.stream() + .map(ResourceInfo::getRes).collect(Collectors.toList()); + if(mainJar != null) { + resourceFiles.add(mainJar.getRes()); + } + return resourceFiles; } - return null; + return Collections.emptyList(); } @Override diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java index 0a4ed9b5ac..d8ef0bb38d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java @@ -120,12 +120,24 @@ public class ThreadUtils { /** * Wrapper over ScheduledThreadPoolExecutor + * @param threadName * @param corePoolSize * @return */ - public static ScheduledExecutorService newDaemonThreadScheduledExecutor(String threadName,int corePoolSize) { + public static ScheduledExecutorService newDaemonThreadScheduledExecutor(String threadName, int corePoolSize) { + return newThreadScheduledExecutor(threadName, corePoolSize, true); + } + + /** + * Wrapper over ScheduledThreadPoolExecutor + * @param threadName + * @param corePoolSize + * @param isDaemon + * @return + */ + public static ScheduledExecutorService newThreadScheduledExecutor(String threadName, int corePoolSize, boolean isDaemon) { ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setDaemon(true) + .setDaemon(isDaemon) .setNameFormat(threadName) .build(); ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(corePoolSize, threadFactory); @@ -135,7 +147,6 @@ public class ThreadUtils { return executor; } - public static ThreadInfo getThreadInfo(Thread t) { long tid = t.getId(); return threadBean.getThreadInfo(tid, STACK_DEPTH); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java index c84848fbae..dc60b04c59 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java @@ -44,7 +44,7 @@ public class FileUtils { String fileSuffix = ""; if (StringUtils.isNotEmpty(filename)) { - int lastIndex = filename.lastIndexOf("."); + int lastIndex = filename.lastIndexOf('.'); if (lastIndex > 0) { fileSuffix = filename.substring(lastIndex + 1); } @@ -325,10 +325,8 @@ public class FileUtils { } } else { File parent = file.getParentFile(); - if (parent != null) { - if (!parent.mkdirs() && !parent.isDirectory()) { + if (parent != null && !parent.mkdirs() && !parent.isDirectory()) { throw new IOException("Directory '" + parent + "' could not be created"); - } } } return new FileOutputStream(file, append); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 541281f793..6c42704b47 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -16,6 +16,9 @@ */ package org.apache.dolphinscheduler.common.utils; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ResUploadType; @@ -32,9 +35,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; +import java.nio.file.Files; import java.security.PrivilegedExceptionAction; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -46,32 +52,37 @@ public class HadoopUtils implements Closeable { private static final Logger logger = LoggerFactory.getLogger(HadoopUtils.class); - private static String hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER); - private static volatile HadoopUtils instance = new HadoopUtils(); - private static volatile Configuration configuration; - private static FileSystem fs; + private static final String HADOOP_UTILS_KEY = "HADOOP_UTILS_KEY"; + private static final LoadingCache cache = CacheBuilder + .newBuilder() + .expireAfterWrite(PropertyUtils.getInt(Constants.KERBEROS_EXPIRE_TIME, 7), TimeUnit.DAYS) + .build(new CacheLoader() { + @Override + public HadoopUtils load(String key) throws Exception { + return new HadoopUtils(); + } + }); - private HadoopUtils(){ - if(StringUtils.isEmpty(hdfsUser)){ - hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER); - } + private Configuration configuration; + private FileSystem fs; + + private static String hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER); + + private HadoopUtils() { init(); initHdfsPath(); } - public static HadoopUtils getInstance(){ - // if kerberos startup , renew HadoopUtils - if (CommonUtils.getKerberosStartupState()){ - return new HadoopUtils(); - } - return instance; + public static HadoopUtils getInstance() { + + return cache.getUnchecked(HADOOP_UTILS_KEY); } /** * init dolphinscheduler root path in hdfs */ - private void initHdfsPath(){ + private void initHdfsPath() { String hdfsPath = PropertyUtils.getString(Constants.DATA_STORE_2_HDFS_BASEPATH); Path path = new Path(hdfsPath); @@ -80,7 +91,7 @@ public class HadoopUtils implements Closeable { fs.mkdirs(path); } } catch (Exception e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); } } @@ -89,82 +100,74 @@ public class HadoopUtils implements Closeable { * init hadoop configuration */ private void init() { - if (configuration == null) { - synchronized (HadoopUtils.class) { - if (configuration == null) { - try { - configuration = new Configuration(); - - String resUploadStartupType = PropertyUtils.getString(Constants.RES_UPLOAD_STARTUP_TYPE); - ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); - - if (resUploadType == ResUploadType.HDFS){ - if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE)){ - System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, - PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)); - configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION,"kerberos"); - UserGroupInformation.setConfiguration(configuration); - UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), - PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)); - } + try { + configuration = new Configuration(); + + String resUploadStartupType = PropertyUtils.getString(Constants.RES_UPLOAD_STARTUP_TYPE); + ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); + + if (resUploadType == ResUploadType.HDFS) { + if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE)) { + System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, + PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)); + configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + UserGroupInformation.setConfiguration(configuration); + UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), + PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)); + } - String defaultFS = configuration.get(Constants.FS_DEFAULTFS); - //first get key from core-site.xml hdfs-site.xml ,if null ,then try to get from properties file - // the default is the local file system - if(defaultFS.startsWith("file")){ - String defaultFSProp = PropertyUtils.getString(Constants.FS_DEFAULTFS); - if(StringUtils.isNotBlank(defaultFSProp)){ - Map fsRelatedProps = PropertyUtils.getPrefixedProperties("fs."); - configuration.set(Constants.FS_DEFAULTFS,defaultFSProp); - fsRelatedProps.forEach((key, value) -> configuration.set(key, value)); - }else{ - logger.error("property:{} can not to be empty, please set!", Constants.FS_DEFAULTFS ); - throw new RuntimeException( - String.format("property: %s can not to be empty, please set!", Constants.FS_DEFAULTFS) - ); - } - }else{ - logger.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", Constants.FS_DEFAULTFS, defaultFS); - } + String defaultFS = configuration.get(Constants.FS_DEFAULTFS); + //first get key from core-site.xml hdfs-site.xml ,if null ,then try to get from properties file + // the default is the local file system + if (defaultFS.startsWith("file")) { + String defaultFSProp = PropertyUtils.getString(Constants.FS_DEFAULTFS); + if (StringUtils.isNotBlank(defaultFSProp)) { + Map fsRelatedProps = PropertyUtils.getPrefixedProperties("fs."); + configuration.set(Constants.FS_DEFAULTFS, defaultFSProp); + fsRelatedProps.forEach((key, value) -> configuration.set(key, value)); + } else { + logger.error("property:{} can not to be empty, please set!", Constants.FS_DEFAULTFS); + throw new RuntimeException( + String.format("property: %s can not to be empty, please set!", Constants.FS_DEFAULTFS) + ); + } + } else { + logger.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", Constants.FS_DEFAULTFS, defaultFS); + } - if (fs == null) { - if(StringUtils.isNotEmpty(hdfsUser)){ - //UserGroupInformation ugi = UserGroupInformation.createProxyUser(hdfsUser,UserGroupInformation.getLoginUser()); - UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser); - ugi.doAs(new PrivilegedExceptionAction() { - @Override - public Boolean run() throws Exception { - fs = FileSystem.get(configuration); - return true; - } - }); - }else{ - logger.warn("hdfs.root.user is not set value!"); - fs = FileSystem.get(configuration); - } + if (fs == null) { + if (StringUtils.isNotEmpty(hdfsUser)) { + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser); + ugi.doAs(new PrivilegedExceptionAction() { + @Override + public Boolean run() throws Exception { + fs = FileSystem.get(configuration); + return true; } - }else if (resUploadType == ResUploadType.S3){ - configuration.set(Constants.FS_DEFAULTFS, PropertyUtils.getString(Constants.FS_DEFAULTFS)); - configuration.set(Constants.FS_S3A_ENDPOINT, PropertyUtils.getString(Constants.FS_S3A_ENDPOINT)); - configuration.set(Constants.FS_S3A_ACCESS_KEY, PropertyUtils.getString(Constants.FS_S3A_ACCESS_KEY)); - configuration.set(Constants.FS_S3A_SECRET_KEY, PropertyUtils.getString(Constants.FS_S3A_SECRET_KEY)); - fs = FileSystem.get(configuration); - } - - - String rmHaIds = PropertyUtils.getString(Constants.YARN_RESOURCEMANAGER_HA_RM_IDS); - String appAddress = PropertyUtils.getString(Constants.YARN_APPLICATION_STATUS_ADDRESS); - if (!StringUtils.isEmpty(rmHaIds)) { - appAddress = getAppAddress(appAddress, rmHaIds); - logger.info("appAddress : {}", appAddress); - } - configuration.set(Constants.YARN_APPLICATION_STATUS_ADDRESS, appAddress); - } catch (Exception e) { - logger.error(e.getMessage(), e); + }); + } else { + logger.warn("hdfs.root.user is not set value!"); + fs = FileSystem.get(configuration); } - } + } else if (resUploadType == ResUploadType.S3) { + configuration.set(Constants.FS_DEFAULTFS, PropertyUtils.getString(Constants.FS_DEFAULTFS)); + configuration.set(Constants.FS_S3A_ENDPOINT, PropertyUtils.getString(Constants.FS_S3A_ENDPOINT)); + configuration.set(Constants.FS_S3A_ACCESS_KEY, PropertyUtils.getString(Constants.FS_S3A_ACCESS_KEY)); + configuration.set(Constants.FS_S3A_SECRET_KEY, PropertyUtils.getString(Constants.FS_S3A_SECRET_KEY)); + fs = FileSystem.get(configuration); + } + + + String rmHaIds = PropertyUtils.getString(Constants.YARN_RESOURCEMANAGER_HA_RM_IDS); + String appAddress = PropertyUtils.getString(Constants.YARN_APPLICATION_STATUS_ADDRESS); + if (!StringUtils.isEmpty(rmHaIds)) { + appAddress = getAppAddress(appAddress, rmHaIds); + logger.info("appAddress : {}", appAddress); } + configuration.set(Constants.YARN_APPLICATION_STATUS_ADDRESS, appAddress); + } catch (Exception e) { + logger.error(e.getMessage(), e); } } @@ -188,15 +191,15 @@ public class HadoopUtils implements Closeable { /** * cat file on hdfs * - * @param hdfsFilePath hdfs file path + * @param hdfsFilePath hdfs file path * @return byte[] byte array * @throws IOException errors */ public byte[] catFile(String hdfsFilePath) throws IOException { - if(StringUtils.isBlank(hdfsFilePath)){ - logger.error("hdfs file path:{} is blank",hdfsFilePath); - return null; + if (StringUtils.isBlank(hdfsFilePath)) { + logger.error("hdfs file path:{} is blank", hdfsFilePath); + return new byte[0]; } FSDataInputStream fsDataInputStream = fs.open(new Path(hdfsFilePath)); @@ -204,29 +207,28 @@ public class HadoopUtils implements Closeable { } - /** * cat file on hdfs * - * @param hdfsFilePath hdfs file path - * @param skipLineNums skip line numbers - * @param limit read how many lines + * @param hdfsFilePath hdfs file path + * @param skipLineNums skip line numbers + * @param limit read how many lines * @return content of file * @throws IOException errors */ public List catFile(String hdfsFilePath, int skipLineNums, int limit) throws IOException { - if (StringUtils.isBlank(hdfsFilePath)){ - logger.error("hdfs file path:{} is blank",hdfsFilePath); - return null; + if (StringUtils.isBlank(hdfsFilePath)) { + logger.error("hdfs file path:{} is blank", hdfsFilePath); + return Collections.emptyList(); } - try (FSDataInputStream in = fs.open(new Path(hdfsFilePath))){ + try (FSDataInputStream in = fs.open(new Path(hdfsFilePath))) { BufferedReader br = new BufferedReader(new InputStreamReader(in)); Stream stream = br.lines().skip(skipLineNums).limit(limit); return stream.collect(Collectors.toList()); } - + } /** @@ -259,17 +261,17 @@ public class HadoopUtils implements Closeable { /** * the src file is on the local disk. Add it to FS at * the given dst name. - - * @param srcFile local file - * @param dstHdfsPath destination hdfs path - * @param deleteSource whether to delete the src - * @param overwrite whether to overwrite an existing file + * + * @param srcFile local file + * @param dstHdfsPath destination hdfs path + * @param deleteSource whether to delete the src + * @param overwrite whether to overwrite an existing file * @return if success or not * @throws IOException errors */ public boolean copyLocalToHdfs(String srcFile, String dstHdfsPath, boolean deleteSource, boolean overwrite) throws IOException { Path srcPath = new Path(srcFile); - Path dstPath= new Path(dstHdfsPath); + Path dstPath = new Path(dstHdfsPath); fs.copyFromLocalFile(deleteSource, overwrite, srcPath, dstPath); @@ -279,10 +281,10 @@ public class HadoopUtils implements Closeable { /** * copy hdfs file to local * - * @param srcHdfsFilePath source hdfs file path - * @param dstFile destination file - * @param deleteSource delete source - * @param overwrite overwrite + * @param srcHdfsFilePath source hdfs file path + * @param dstFile destination file + * @param deleteSource delete source + * @param overwrite overwrite * @return result of copy hdfs file to local * @throws IOException errors */ @@ -293,14 +295,14 @@ public class HadoopUtils implements Closeable { if (dstPath.exists()) { if (dstPath.isFile()) { if (overwrite) { - dstPath.delete(); + Files.delete(dstPath.toPath()); } } else { logger.error("destination file must be a file"); } } - if(!dstPath.getParentFile().exists()){ + if (!dstPath.getParentFile().exists()) { dstPath.getParentFile().mkdirs(); } @@ -308,14 +310,13 @@ public class HadoopUtils implements Closeable { } /** - * * delete a file * * @param hdfsFilePath the path to delete. - * @param recursive if path is a directory and set to - * true, the directory is deleted else throws an exception. In - * case of a file the recursive can be set to either true or false. - * @return true if delete is successful else false. + * @param recursive if path is a directory and set to + * true, the directory is deleted else throws an exception. In + * case of a file the recursive can be set to either true or false. + * @return true if delete is successful else false. * @throws IOException errors */ public boolean delete(String hdfsFilePath, boolean recursive) throws IOException { @@ -340,7 +341,7 @@ public class HadoopUtils implements Closeable { * @return {@link FileStatus} file status * @throws Exception errors */ - public FileStatus[] listFileStatus(String filePath)throws Exception{ + public FileStatus[] listFileStatus(String filePath) throws Exception { try { return fs.listStatus(new Path(filePath)); } catch (IOException e) { @@ -352,10 +353,11 @@ public class HadoopUtils implements Closeable { /** * Renames Path src to Path dst. Can take place on local fs * or remote DFS. + * * @param src path to be renamed * @param dst new path after rename - * @throws IOException on failure * @return true if rename is successful + * @throws IOException on failure */ public boolean rename(String src, String dst) throws IOException { return fs.rename(new Path(src), new Path(dst)); @@ -378,7 +380,7 @@ public class HadoopUtils implements Closeable { String responseContent = HttpUtils.get(applicationUrl); - JSONObject jsonObject = JSONObject.parseObject(responseContent); + JSONObject jsonObject = JSON.parseObject(responseContent); String result = jsonObject.getJSONObject("app").getString("finalStatus"); switch (result) { @@ -401,7 +403,6 @@ public class HadoopUtils implements Closeable { } /** - * * @return data hdfs path */ public static String getHdfsDataBasePath() { @@ -428,11 +429,11 @@ public class HadoopUtils implements Closeable { * hdfs user dir * * @param tenantCode tenant code - * @param userId user id + * @param userId user id * @return hdfs resource dir */ - public static String getHdfsUserDir(String tenantCode,int userId) { - return String.format("%s/home/%d", getHdfsTenantDir(tenantCode),userId); + public static String getHdfsUserDir(String tenantCode, int userId) { + return String.format("%s/home/%d", getHdfsTenantDir(tenantCode), userId); } /** @@ -480,7 +481,7 @@ public class HadoopUtils implements Closeable { * getAppAddress * * @param appAddress app address - * @param rmHa resource manager ha + * @param rmHa resource manager ha * @return app address */ public static String getAppAddress(String appAddress, String rmHa) { @@ -525,8 +526,6 @@ public class HadoopUtils implements Closeable { */ private static final class YarnHAAdminUtils extends RMAdminCLI { - private static final Logger logger = LoggerFactory.getLogger(YarnHAAdminUtils.class); - /** * get active resourcemanager * @@ -585,8 +584,7 @@ public class HadoopUtils implements Closeable { JSONObject jsonObject = JSON.parseObject(retStr); //get ResourceManager state - String state = jsonObject.getJSONObject("clusterInfo").getString("haState"); - return state; + return jsonObject.getJSONObject("clusterInfo").getString("haState"); } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java index c163dcab3d..7de198f28b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java @@ -81,17 +81,15 @@ public class HttpUtils { logger.error(e.getMessage(),e); } - if (httpget != null && !httpget.isAborted()) { + if (!httpget.isAborted()) { httpget.releaseConnection(); httpget.abort(); } - if (httpclient != null) { - try { - httpclient.close(); - } catch (IOException e) { - logger.error(e.getMessage(),e); - } + try { + httpclient.close(); + } catch (IOException e) { + logger.error(e.getMessage(),e); } } return responseContent; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java index 73df158aa3..ce551d8405 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java @@ -19,26 +19,17 @@ package org.apache.dolphinscheduler.common.utils; +import java.io.Closeable; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; public class IOUtils { - public static void closeQuietly(InputStream fis){ - if(fis != null){ + public static void closeQuietly(Closeable closeable){ + if(closeable != null){ try { - fis.close(); - } catch (IOException ignore) { - } - } - } - - public static void closeQuietly(InputStreamReader reader){ - if(reader != null){ - try { - reader.close(); + closeable.close(); } catch (IOException ignore) { + // nothing need to do } } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java index e7e0b34bdd..3b068c60d2 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java @@ -17,16 +17,11 @@ package org.apache.dolphinscheduler.common.utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - /** * http utils */ public class IpUtils { - private static final Logger logger = LoggerFactory.getLogger(IpUtils.class); public static final String DOT = "."; /** diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java index ec523b1ff2..f0aed91a0d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.TypeReference; @@ -41,12 +42,6 @@ public class JSONUtils { */ private static final ObjectMapper objectMapper = new ObjectMapper(); - /** - * init - */ - private static final JSONUtils instance = new JSONUtils(); - - private JSONUtils() { //Feature that determines whether encountering of unknown properties, false means not analyzer unknown properties objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false).setTimeZone(TimeZone.getDefault()); @@ -59,7 +54,7 @@ public class JSONUtils { */ public static String toJson(Object object) { try{ - return JSONObject.toJSONString(object,false); + return JSON.toJSONString(object,false); } catch (Exception e) { logger.error("object to json exception!",e); } @@ -89,7 +84,7 @@ public class JSONUtils { } try { - return JSONObject.parseObject(json, clazz); + return JSON.parseObject(json, clazz); } catch (Exception e) { logger.error("parse object exception!",e); } @@ -178,7 +173,7 @@ public class JSONUtils { } try { - return JSONObject.parseObject(json, new TypeReference>(){}); + return JSON.parseObject(json, new TypeReference>(){}); } catch (Exception e) { logger.error("json to map exception!",e); } @@ -203,7 +198,7 @@ public class JSONUtils { } try { - return JSONObject.parseObject(json, new TypeReference>() {}); + return JSON.parseObject(json, new TypeReference>() {}); } catch (Exception e) { logger.error("json to map exception!",e); } @@ -218,23 +213,23 @@ public class JSONUtils { */ public static String toJsonString(Object object) { try{ - return JSONObject.toJSONString(object,false); + return JSON.toJSONString(object,false); } catch (Exception e) { - throw new RuntimeException("Json deserialization exception.", e); + throw new RuntimeException("Object json deserialization exception.", e); } } public static JSONObject parseObject(String text) { try{ - return JSONObject.parseObject(text); + return JSON.parseObject(text); } catch (Exception e) { - throw new RuntimeException("Json deserialization exception.", e); + throw new RuntimeException("String json deserialization exception.", e); } } public static JSONArray parseArray(String text) { try{ - return JSONObject.parseArray(text); + return JSON.parseArray(text); } catch (Exception e) { throw new RuntimeException("Json deserialization exception.", e); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java index fc08eb645b..191df335c5 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java @@ -79,7 +79,7 @@ public class LoggerUtils { */ public static List getAppIds(String log, Logger logger) { - List appIds = new ArrayList(); + List appIds = new ArrayList<>(); Matcher matcher = APPLICATION_REGEX.matcher(log); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index acfca77c8c..b011c0bc4e 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -400,8 +400,7 @@ public class OSUtils { * @return true if mac */ public static boolean isMacOS() { - String os = System.getProperty("os.name"); - return os.startsWith("Mac"); + return getOSName().startsWith("Mac"); } @@ -409,9 +408,16 @@ public class OSUtils { * whether is windows * @return true if windows */ - public static boolean isWindows() { - String os = System.getProperty("os.name"); - return os.startsWith("Windows"); + public static boolean isWindows() { ; + return getOSName().startsWith("Windows"); + } + + /** + * get current OS name + * @return current OS name + */ + public static String getOSName() { + return System.getProperty("os.name"); } /** diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java index 7f2888384e..7a4553aaf5 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DataType; @@ -23,7 +24,6 @@ import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils; import org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils; -import com.alibaba.fastjson.JSONObject; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.time.DateUtils; import org.slf4j.Logger; @@ -119,10 +119,15 @@ public class ParameterUtils { */ public static String curingGlobalParams(Map globalParamMap, List globalParamList, CommandType commandType, Date scheduleTime){ - Map globalMap = new HashMap<>(); - if(globalParamMap!= null){ - globalMap.putAll(globalParamMap); - } + + if (globalParamList == null || globalParamList.isEmpty()) { + return null; + } + + Map globalMap = new HashMap<>(); + if (globalParamMap!= null){ + globalMap.putAll(globalParamMap); + } Map allParamMap = new HashMap<>(); //If it is a complement, a complement time needs to be passed in, according to the task type Map timeParams = BusinessTimeUtils @@ -132,9 +137,7 @@ public class ParameterUtils { allParamMap.putAll(timeParams); } - if (globalMap != null) { - allParamMap.putAll(globalMap); - } + allParamMap.putAll(globalMap); Set> entries = allParamMap.entrySet(); @@ -146,22 +149,15 @@ public class ParameterUtils { resolveMap.put(entry.getKey(),str); } } + globalMap.putAll(resolveMap); - if (globalMap != null){ - globalMap.putAll(resolveMap); - } - - if (globalParamList != null && globalParamList.size() > 0){ - - for (Property property : globalParamList){ - String val = globalMap.get(property.getProp()); - if (val != null){ - property.setValue(val); - } + for (Property property : globalParamList){ + String val = globalMap.get(property.getProp()); + if (val != null){ + property.setValue(val); } - return JSONObject.toJSONString(globalParamList); } - return null; + return JSON.toJSONString(globalParamList); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index c3e8197079..ca928c94d5 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -43,13 +43,11 @@ public class PropertyUtils { private static final Properties properties = new Properties(); - private static final PropertyUtils propertyUtils = new PropertyUtils(); - - private PropertyUtils(){ - init(); + private PropertyUtils() { + throw new IllegalStateException("PropertyUtils class"); } - private void init(){ + static { String[] propertyFiles = new String[]{COMMON_PROPERTIES_PATH}; for (String fileName : propertyFiles) { InputStream fis = null; @@ -125,7 +123,7 @@ public class PropertyUtils { * @param key property name * @return property value */ - public static Boolean getBoolean(String key) { + public static boolean getBoolean(String key) { String value = properties.getProperty(key.trim()); if(null != value){ return Boolean.parseBoolean(value); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/PlaceholderUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/PlaceholderUtils.java index 0c756cb0b3..53a0d5af5b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/PlaceholderUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/PlaceholderUtils.java @@ -31,12 +31,12 @@ public class PlaceholderUtils { /** * Prefix of the position to be replaced */ - public static final String placeholderPrefix = "${"; + public static final String PLACEHOLDER_PREFIX = "${"; /** * The suffix of the position to be replaced */ - public static final String placeholderSuffix = "}"; + public static final String PLACEHOLDER_SUFFIX = "}"; /** @@ -68,7 +68,7 @@ public class PlaceholderUtils { */ public static PropertyPlaceholderHelper getPropertyPlaceholderHelper(boolean ignoreUnresolvablePlaceholders) { - return new PropertyPlaceholderHelper(placeholderPrefix, placeholderSuffix, null, ignoreUnresolvablePlaceholders); + return new PropertyPlaceholderHelper(PLACEHOLDER_PREFIX, PLACEHOLDER_SUFFIX, null, ignoreUnresolvablePlaceholders); } /** diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32.java new file mode 100644 index 0000000000..eee456d019 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32.java @@ -0,0 +1,1071 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils.process; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * This class is used to create operating system processes. + * + *

Each {@code ProcessBuilderForWindows} instance manages a collection + * of process attributes. The {@link #start()} method creates a new + * {@link Process} instance with those attributes. The {@link + * #start()} method can be invoked repeatedly from the same instance + * to create new subprocesses with identical or related attributes. + * + *

Each process builder manages these process attributes: + * + *

    + * + *
  • a command, a list of strings which signifies the + * external program file to be invoked and its arguments, if any. + * Which string lists represent a valid operating system command is + * system-dependent. For example, it is common for each conceptual + * argument to be an element in this list, but there are operating + * systems where programs are expected to tokenize command line + * strings themselves - on such a system a Java implementation might + * require commands to contain exactly two elements. + * + *
  • an environment, which is a system-dependent mapping from + * variables to values. The initial value is a copy of + * the environment of the current process (see {@link System#getenv()}). + * + *
  • a working directory. The default value is the current + * working directory of the current process, usually the directory + * named by the system property {@code user.dir}. + * + *
  • a source of standard input. + * By default, the subprocess reads input from a pipe. Java code + * can access this pipe via the output stream returned by + * {@link Process#getOutputStream()}. However, standard input may + * be redirected to another source using + * {@link #redirectInput(ProcessBuilderForWin32.Redirect) redirectInput}. + * In this case, {@link Process#getOutputStream()} will return a + * null output stream, for which: + * + *
      + *
    • the {@link OutputStream#write(int) write} methods always + * throw {@code IOException} + *
    • the {@link OutputStream#close() close} method does nothing + *
    + * + *
  • a destination for standard output + * and standard error. By default, the subprocess writes standard + * output and standard error to pipes. Java code can access these pipes + * via the input streams returned by {@link Process#getInputStream()} and + * {@link Process#getErrorStream()}. However, standard output and + * standard error may be redirected to other destinations using + * {@link #redirectOutput(ProcessBuilderForWin32.Redirect) redirectOutput} and + * {@link #redirectError(ProcessBuilderForWin32.Redirect) redirectError}. + * In this case, {@link Process#getInputStream()} and/or + * {@link Process#getErrorStream()} will return a null input + * stream, for which: + * + *
      + *
    • the {@link InputStream#read() read} methods always return + * {@code -1} + *
    • the {@link InputStream#available() available} method always returns + * {@code 0} + *
    • the {@link InputStream#close() close} method does nothing + *
    + * + *
  • a redirectErrorStream property. Initially, this property + * is {@code false}, meaning that the standard output and error + * output of a subprocess are sent to two separate streams, which can + * be accessed using the {@link Process#getInputStream()} and {@link + * Process#getErrorStream()} methods. + * + *

    If the value is set to {@code true}, then: + * + *

      + *
    • standard error is merged with the standard output and always sent + * to the same destination (this makes it easier to correlate error + * messages with the corresponding output) + *
    • the common destination of standard error and standard output can be + * redirected using + * {@link #redirectOutput(ProcessBuilderForWin32.Redirect) redirectOutput} + *
    • any redirection set by the + * {@link #redirectError(ProcessBuilderForWin32.Redirect) redirectError} + * method is ignored when creating a subprocess + *
    • the stream returned from {@link Process#getErrorStream()} will + * always be a null input stream + *
    + * + *
+ * + *

Modifying a process builder's attributes will affect processes + * subsequently started by that object's {@link #start()} method, but + * will never affect previously started processes or the Java process + * itself. + * + *

Most error checking is performed by the {@link #start()} method. + * It is possible to modify the state of an object so that {@link + * #start()} will fail. For example, setting the command attribute to + * an empty list will not throw an exception unless {@link #start()} + * is invoked. + * + *

Note that this class is not synchronized. + * If multiple threads access a {@code ProcessBuilderForWindows} instance + * concurrently, and at least one of the threads modifies one of the + * attributes structurally, it must be synchronized externally. + * + *

Starting a new process which uses the default working directory + * and environment is easy: + * + * 

 {@code
+ * Process p = new ProcessBuilderForWindows("myCommand", "myArg").start();
+ * }
+ * + *

Here is an example that starts a process with a modified working + * directory and environment, and redirects standard output and error + * to be appended to a log file: + * + * 

 {@code
+ * ProcessBuilderForWindows pb =
+ *   new ProcessBuilderForWindows("myCommand", "myArg1", "myArg2");
+ * Map env = pb.environment();
+ * env.put("VAR1", "myValue");
+ * env.remove("OTHERVAR");
+ * env.put("VAR2", env.get("VAR1") + "suffix");
+ * pb.directory(new File("myDir"));
+ * File log = new File("log");
+ * pb.redirectErrorStream(true);
+ * pb.redirectOutput(Redirect.appendTo(log));
+ * Process p = pb.start();
+ * assert pb.redirectInput() == Redirect.PIPE;
+ * assert pb.redirectOutput().file() == log;
+ * assert p.getInputStream().read() == -1;
+ * }
+ * + *

To start a process with an explicit set of environment + * variables, first call {@link Map#clear() Map.clear()} + * before adding environment variables. + * + * @author Martin Buchholz + * @since 1.5 + */ + +public class ProcessBuilderForWin32 { + + private String username; + private String password; + private List command; + private File directory; + private Map environment; + private boolean redirectErrorStream; + private ProcessBuilderForWin32.Redirect[] redirects; + + /** + * Constructs a process builder with the specified operating + * system program and arguments. This constructor does not + * make a copy of the {@code command} list. Subsequent + * updates to the list will be reflected in the state of the + * process builder. It is not checked whether + * {@code command} corresponds to a valid operating system + * command. + * + * @param command the list containing the program and its arguments + * @throws NullPointerException if the argument is null + */ + public ProcessBuilderForWin32(List command) { + if (command == null) + throw new NullPointerException(); + this.command = command; + } + + /** + * Constructs a process builder with the specified operating + * system program and arguments. This is a convenience + * constructor that sets the process builder's command to a string + * list containing the same strings as the {@code command} + * array, in the same order. It is not checked whether + * {@code command} corresponds to a valid operating system + * command. + * + * @param command a string array containing the program and its arguments + */ + public ProcessBuilderForWin32(String... command) { + this.command = new ArrayList<>(command.length); + for (String arg : command) + this.command.add(arg); + } + + /** + * set username and password for process + * + * @param username username + * @param password password + * @return this process builder + */ + public ProcessBuilderForWin32 user(String username, String password) { + this.username = username; + this.password = password; + return this; + } + + /** + * Sets this process builder's operating system program and + * arguments. This method does not make a copy of the + * {@code command} list. Subsequent updates to the list will + * be reflected in the state of the process builder. It is not + * checked whether {@code command} corresponds to a valid + * operating system command. + * + * @param command the list containing the program and its arguments + * @return this process builder + * + * @throws NullPointerException if the argument is null + */ + public ProcessBuilderForWin32 command(List command) { + if (command == null) + throw new NullPointerException(); + this.command = command; + return this; + } + + /** + * Sets this process builder's operating system program and + * arguments. This is a convenience method that sets the command + * to a string list containing the same strings as the + * {@code command} array, in the same order. It is not + * checked whether {@code command} corresponds to a valid + * operating system command. + * + * @param command a string array containing the program and its arguments + * @return this process builder + */ + public ProcessBuilderForWin32 command(String... command) { + this.command = new ArrayList<>(command.length); + for (String arg : command) + this.command.add(arg); + return this; + } + + /** + * Returns this process builder's operating system program and + * arguments. The returned list is not a copy. Subsequent + * updates to the list will be reflected in the state of this + * process builder. + * + * @return this process builder's program and its arguments + */ + public List command() { + return command; + } + + /** + * Returns a string map view of this process builder's environment. + * + * Whenever a process builder is created, the environment is + * initialized to a copy of the current process environment (see + * {@link System#getenv()}). Subprocesses subsequently started by + * this object's {@link #start()} method will use this map as + * their environment. + * + *

The returned object may be modified using ordinary {@link + * Map Map} operations. These modifications will be + * visible to subprocesses started via the {@link #start()} + * method. Two {@code ProcessBuilderForWindows} instances always + * contain independent process environments, so changes to the + * returned map will never be reflected in any other + * {@code ProcessBuilderForWindows} instance or the values returned by + * {@link System#getenv System.getenv}. + * + *

If the system does not support environment variables, an + * empty map is returned. + * + *

The returned map does not permit null keys or values. + * Attempting to insert or query the presence of a null key or + * value will throw a {@link NullPointerException}. + * Attempting to query the presence of a key or value which is not + * of type {@link String} will throw a {@link ClassCastException}. + * + *

The behavior of the returned map is system-dependent. A + * system may not allow modifications to environment variables or + * may forbid certain variable names or values. For this reason, + * attempts to modify the map may fail with + * {@link UnsupportedOperationException} or + * {@link IllegalArgumentException} + * if the modification is not permitted by the operating system. + * + *

Since the external format of environment variable names and + * values is system-dependent, there may not be a one-to-one + * mapping between them and Java's Unicode strings. Nevertheless, + * the map is implemented in such a way that environment variables + * which are not modified by Java code will have an unmodified + * native representation in the subprocess. + * + *

The returned map and its collection views may not obey the + * general contract of the {@link Object#equals} and + * {@link Object#hashCode} methods. + * + *

The returned map is typically case-sensitive on all platforms. + * + *

If a security manager exists, its + * {@link SecurityManager#checkPermission checkPermission} method + * is called with a + * {@link RuntimePermission}{@code ("getenv.*")} permission. + * This may result in a {@link SecurityException} being thrown. + * + *

When passing information to a Java subprocess, + * system properties + * are generally preferred over environment variables. + * + * @return this process builder's environment + * + * @throws SecurityException + * if a security manager exists and its + * {@link SecurityManager#checkPermission checkPermission} + * method doesn't allow access to the process environment + * + * @see Runtime#exec(String[],String[], File) + * @see System#getenv() + */ + public Map environment() { + SecurityManager security = System.getSecurityManager(); + if (security != null) + security.checkPermission(new RuntimePermission("getenv.*")); + + if (environment == null) + environment = ProcessEnvironmentForWin32.environment(); + + assert environment != null; + + return environment; + } + + // Only for use by Runtime.exec(...envp...) + ProcessBuilderForWin32 environment(String[] envp) { + assert environment == null; + if (envp != null) { + environment = ProcessEnvironmentForWin32.emptyEnvironment(envp.length); + assert environment != null; + + for (String envstring : envp) { + // Before 1.5, we blindly passed invalid envstrings + // to the child process. + // We would like to throw an exception, but do not, + // for compatibility with old broken code. + + // Silently discard any trailing junk. + if (envstring.indexOf((int) '\u0000') != -1) + envstring = envstring.replaceFirst("\u0000.*", ""); + + int eqlsign = + envstring.indexOf('=', ProcessEnvironmentForWin32.MIN_NAME_LENGTH); + // Silently ignore envstrings lacking the required `='. + if (eqlsign != -1) + environment.put(envstring.substring(0,eqlsign), + envstring.substring(eqlsign+1)); + } + } + return this; + } + + /** + * Returns this process builder's working directory. + * + * Subprocesses subsequently started by this object's {@link + * #start()} method will use this as their working directory. + * The returned value may be {@code null} -- this means to use + * the working directory of the current Java process, usually the + * directory named by the system property {@code user.dir}, + * as the working directory of the child process. + * + * @return this process builder's working directory + */ + public File directory() { + return directory; + } + + /** + * Sets this process builder's working directory. + * + * Subprocesses subsequently started by this object's {@link + * #start()} method will use this as their working directory. + * The argument may be {@code null} -- this means to use the + * working directory of the current Java process, usually the + * directory named by the system property {@code user.dir}, + * as the working directory of the child process. + * + * @param directory the new working directory + * @return this process builder + */ + public ProcessBuilderForWin32 directory(File directory) { + this.directory = directory; + return this; + } + + // ---------------- I/O Redirection ---------------- + + /** + * Implements a null input stream. + */ + static class NullInputStream extends InputStream { + static final ProcessBuilderForWin32.NullInputStream INSTANCE = new ProcessBuilderForWin32.NullInputStream(); + private NullInputStream() {} + public int read() { return -1; } + @Override + public int available() { return 0; } + } + + /** + * Implements a null output stream. + */ + static class NullOutputStream extends OutputStream { + static final ProcessBuilderForWin32.NullOutputStream INSTANCE = new ProcessBuilderForWin32.NullOutputStream(); + private NullOutputStream() {} + public void write(int b) throws IOException { + throw new IOException("Stream closed"); + } + } + + /** + * Represents a source of subprocess input or a destination of + * subprocess output. + * + * Each {@code Redirect} instance is one of the following: + * + *

    + *
  • the special value {@link #PIPE Redirect.PIPE} + *
  • the special value {@link #INHERIT Redirect.INHERIT} + *
  • a redirection to read from a file, created by an invocation of + * {@link ProcessBuilderForWin32.Redirect#from Redirect.from(File)} + *
  • a redirection to write to a file, created by an invocation of + * {@link ProcessBuilderForWin32.Redirect#to Redirect.to(File)} + *
  • a redirection to append to a file, created by an invocation of + * {@link ProcessBuilderForWin32.Redirect#appendTo Redirect.appendTo(File)} + *
+ * + *

Each of the above categories has an associated unique + * {@link ProcessBuilderForWin32.Redirect.Type Type}. + * + * @since 1.7 + */ + public abstract static class Redirect { + /** + * The type of a {@link ProcessBuilderForWin32.Redirect}. + */ + public enum Type { + /** + * The type of {@link ProcessBuilderForWin32.Redirect#PIPE Redirect.PIPE}. + */ + PIPE, + + /** + * The type of {@link ProcessBuilderForWin32.Redirect#INHERIT Redirect.INHERIT}. + */ + INHERIT, + + /** + * The type of redirects returned from + * {@link ProcessBuilderForWin32.Redirect#from Redirect.from(File)}. + */ + READ, + + /** + * The type of redirects returned from + * {@link ProcessBuilderForWin32.Redirect#to Redirect.to(File)}. + */ + WRITE, + + /** + * The type of redirects returned from + * {@link ProcessBuilderForWin32.Redirect#appendTo Redirect.appendTo(File)}. + */ + APPEND + } + + /** + * Returns the type of this {@code Redirect}. + * @return the type of this {@code Redirect} + */ + public abstract ProcessBuilderForWin32.Redirect.Type type(); + + /** + * Indicates that subprocess I/O will be connected to the + * current Java process over a pipe. + * + * This is the default handling of subprocess standard I/O. + * + *

It will always be true that + * 

 {@code
+         * Redirect.PIPE.file() == null &&
+         * Redirect.PIPE.type() == Redirect.Type.PIPE
+         * }
+ */ + public static final ProcessBuilderForWin32.Redirect PIPE = new ProcessBuilderForWin32.Redirect() { + public Type type() { return Type.PIPE; } + public String toString() { return type().toString(); }}; + + /** + * Indicates that subprocess I/O source or destination will be the + * same as those of the current process. This is the normal + * behavior of most operating system command interpreters (shells). + * + *

It will always be true that + * 

 {@code
+         * Redirect.INHERIT.file() == null &&
+         * Redirect.INHERIT.type() == Redirect.Type.INHERIT
+         * }
+ */ + public static final ProcessBuilderForWin32.Redirect INHERIT = new ProcessBuilderForWin32.Redirect() { + public Type type() { return Type.INHERIT; } + public String toString() { return type().toString(); }}; + + /** + * Returns the {@link File} source or destination associated + * with this redirect, or {@code null} if there is no such file. + * + * @return the file associated with this redirect, + * or {@code null} if there is no such file + */ + public File file() { return null; } + + /** + * When redirected to a destination file, indicates if the output + * is to be written to the end of the file. + */ + boolean append() { + throw new UnsupportedOperationException(); + } + + /** + * Returns a redirect to read from the specified file. + * + *

It will always be true that + * 

 {@code
+         * Redirect.from(file).file() == file &&
+         * Redirect.from(file).type() == Redirect.Type.READ
+         * }
+ * + * @param file The {@code File} for the {@code Redirect}. + * @throws NullPointerException if the specified file is null + * @return a redirect to read from the specified file + */ + public static ProcessBuilderForWin32.Redirect from(final File file) { + if (file == null) + throw new NullPointerException(); + return new ProcessBuilderForWin32.Redirect() { + public Type type() { return Type.READ; } + @Override + public File file() { return file; } + public String toString() { + return "redirect to read from file \"" + file + "\""; + } + }; + } + + /** + * Returns a redirect to write to the specified file. + * If the specified file exists when the subprocess is started, + * its previous contents will be discarded. + * + *

It will always be true that + * 

 {@code
+         * Redirect.to(file).file() == file &&
+         * Redirect.to(file).type() == Redirect.Type.WRITE
+         * }
+ * + * @param file The {@code File} for the {@code Redirect}. + * @throws NullPointerException if the specified file is null + * @return a redirect to write to the specified file + */ + public static ProcessBuilderForWin32.Redirect to(final File file) { + if (file == null) + throw new NullPointerException(); + return new ProcessBuilderForWin32.Redirect() { + public Type type() { return Type.WRITE; } + @Override + public File file() { return file; } + public String toString() { + return "redirect to write to file \"" + file + "\""; + } + @Override + boolean append() { return false; } + }; + } + + /** + * Returns a redirect to append to the specified file. + * Each write operation first advances the position to the + * end of the file and then writes the requested data. + * Whether the advancement of the position and the writing + * of the data are done in a single atomic operation is + * system-dependent and therefore unspecified. + * + *

It will always be true that + * 

 {@code
+         * Redirect.appendTo(file).file() == file &&
+         * Redirect.appendTo(file).type() == Redirect.Type.APPEND
+         * }
+ * + * @param file The {@code File} for the {@code Redirect}. + * @throws NullPointerException if the specified file is null + * @return a redirect to append to the specified file + */ + public static ProcessBuilderForWin32.Redirect appendTo(final File file) { + if (file == null) + throw new NullPointerException(); + return new ProcessBuilderForWin32.Redirect() { + public Type type() { return Type.APPEND; } + @Override + public File file() { return file; } + public String toString() { + return "redirect to append to file \"" + file + "\""; + } + @Override + boolean append() { return true; } + }; + } + + /** + * Compares the specified object with this {@code Redirect} for + * equality. Returns {@code true} if and only if the two + * objects are identical or both objects are {@code Redirect} + * instances of the same type associated with non-null equal + * {@code File} instances. + */ + public boolean equals(Object obj) { + if (obj == this) + return true; + if (! (obj instanceof ProcessBuilderForWin32.Redirect)) + return false; + ProcessBuilderForWin32.Redirect r = (ProcessBuilderForWin32.Redirect) obj; + if (r.type() != this.type()) + return false; + assert this.file() != null; + return this.file().equals(r.file()); + } + + /** + * Returns a hash code value for this {@code Redirect}. + * @return a hash code value for this {@code Redirect} + */ + public int hashCode() { + File file = file(); + if (file == null) + return super.hashCode(); + else + return file.hashCode(); + } + + /** + * No public constructors. Clients must use predefined + * static {@code Redirect} instances or factory methods. + */ + private Redirect() {} + } + + private ProcessBuilderForWin32.Redirect[] redirects() { + if (redirects == null) + redirects = new ProcessBuilderForWin32.Redirect[] { + ProcessBuilderForWin32.Redirect.PIPE, ProcessBuilderForWin32.Redirect.PIPE, ProcessBuilderForWin32.Redirect.PIPE + }; + return redirects; + } + + /** + * Sets this process builder's standard input source. + * + * Subprocesses subsequently started by this object's {@link #start()} + * method obtain their standard input from this source. + * + *

If the source is {@link ProcessBuilderForWin32.Redirect#PIPE Redirect.PIPE} + * (the initial value), then the standard input of a + * subprocess can be written to using the output stream + * returned by {@link Process#getOutputStream()}. + * If the source is set to any other value, then + * {@link Process#getOutputStream()} will return a + * null output stream. + * + * @param source the new standard input source + * @return this process builder + * @throws IllegalArgumentException + * if the redirect does not correspond to a valid source + * of data, that is, has type + * {@link ProcessBuilderForWin32.Redirect.Type#WRITE WRITE} or + * {@link ProcessBuilderForWin32.Redirect.Type#APPEND APPEND} + * @since 1.7 + */ + public ProcessBuilderForWin32 redirectInput(ProcessBuilderForWin32.Redirect source) { + if (source.type() == ProcessBuilderForWin32.Redirect.Type.WRITE || + source.type() == ProcessBuilderForWin32.Redirect.Type.APPEND) + throw new IllegalArgumentException( + "Redirect invalid for reading: " + source); + redirects()[0] = source; + return this; + } + + /** + * Sets this process builder's standard output destination. + * + * Subprocesses subsequently started by this object's {@link #start()} + * method send their standard output to this destination. + * + *

If the destination is {@link ProcessBuilderForWin32.Redirect#PIPE Redirect.PIPE} + * (the initial value), then the standard output of a subprocess + * can be read using the input stream returned by {@link + * Process#getInputStream()}. + * If the destination is set to any other value, then + * {@link Process#getInputStream()} will return a + * null input stream. + * + * @param destination the new standard output destination + * @return this process builder + * @throws IllegalArgumentException + * if the redirect does not correspond to a valid + * destination of data, that is, has type + * {@link ProcessBuilderForWin32.Redirect.Type#READ READ} + * @since 1.7 + */ + public ProcessBuilderForWin32 redirectOutput(ProcessBuilderForWin32.Redirect destination) { + if (destination.type() == ProcessBuilderForWin32.Redirect.Type.READ) + throw new IllegalArgumentException( + "Redirect invalid for writing: " + destination); + redirects()[1] = destination; + return this; + } + + /** + * Sets this process builder's standard error destination. + * + * Subprocesses subsequently started by this object's {@link #start()} + * method send their standard error to this destination. + * + *

If the destination is {@link ProcessBuilderForWin32.Redirect#PIPE Redirect.PIPE} + * (the initial value), then the error output of a subprocess + * can be read using the input stream returned by {@link + * Process#getErrorStream()}. + * If the destination is set to any other value, then + * {@link Process#getErrorStream()} will return a + * null input stream. + * + *

If the {@link #redirectErrorStream redirectErrorStream} + * attribute has been set {@code true}, then the redirection set + * by this method has no effect. + * + * @param destination the new standard error destination + * @return this process builder + * @throws IllegalArgumentException + * if the redirect does not correspond to a valid + * destination of data, that is, has type + * {@link ProcessBuilderForWin32.Redirect.Type#READ READ} + * @since 1.7 + */ + public ProcessBuilderForWin32 redirectError(ProcessBuilderForWin32.Redirect destination) { + if (destination.type() == ProcessBuilderForWin32.Redirect.Type.READ) + throw new IllegalArgumentException( + "Redirect invalid for writing: " + destination); + redirects()[2] = destination; + return this; + } + + /** + * Sets this process builder's standard input source to a file. + * + *

This is a convenience method. An invocation of the form + * {@code redirectInput(file)} + * behaves in exactly the same way as the invocation + * {@link #redirectInput(ProcessBuilderForWin32.Redirect) redirectInput} + * {@code (Redirect.from(file))}. + * + * @param file the new standard input source + * @return this process builder + * @since 1.7 + */ + public ProcessBuilderForWin32 redirectInput(File file) { + return redirectInput(ProcessBuilderForWin32.Redirect.from(file)); + } + + /** + * Sets this process builder's standard output destination to a file. + * + *

This is a convenience method. An invocation of the form + * {@code redirectOutput(file)} + * behaves in exactly the same way as the invocation + * {@link #redirectOutput(ProcessBuilderForWin32.Redirect) redirectOutput} + * {@code (Redirect.to(file))}. + * + * @param file the new standard output destination + * @return this process builder + * @since 1.7 + */ + public ProcessBuilderForWin32 redirectOutput(File file) { + return redirectOutput(ProcessBuilderForWin32.Redirect.to(file)); + } + + /** + * Sets this process builder's standard error destination to a file. + * + *

This is a convenience method. An invocation of the form + * {@code redirectError(file)} + * behaves in exactly the same way as the invocation + * {@link #redirectError(ProcessBuilderForWin32.Redirect) redirectError} + * {@code (Redirect.to(file))}. + * + * @param file the new standard error destination + * @return this process builder + * @since 1.7 + */ + public ProcessBuilderForWin32 redirectError(File file) { + return redirectError(ProcessBuilderForWin32.Redirect.to(file)); + } + + /** + * Returns this process builder's standard input source. + * + * Subprocesses subsequently started by this object's {@link #start()} + * method obtain their standard input from this source. + * The initial value is {@link ProcessBuilderForWin32.Redirect#PIPE Redirect.PIPE}. + * + * @return this process builder's standard input source + * @since 1.7 + */ + public ProcessBuilderForWin32.Redirect redirectInput() { + return (redirects == null) ? ProcessBuilderForWin32.Redirect.PIPE : redirects[0]; + } + + /** + * Returns this process builder's standard output destination. + * + * Subprocesses subsequently started by this object's {@link #start()} + * method redirect their standard output to this destination. + * The initial value is {@link ProcessBuilderForWin32.Redirect#PIPE Redirect.PIPE}. + * + * @return this process builder's standard output destination + * @since 1.7 + */ + public ProcessBuilderForWin32.Redirect redirectOutput() { + return (redirects == null) ? ProcessBuilderForWin32.Redirect.PIPE : redirects[1]; + } + + /** + * Returns this process builder's standard error destination. + * + * Subprocesses subsequently started by this object's {@link #start()} + * method redirect their standard error to this destination. + * The initial value is {@link ProcessBuilderForWin32.Redirect#PIPE Redirect.PIPE}. + * + * @return this process builder's standard error destination + * @since 1.7 + */ + public ProcessBuilderForWin32.Redirect redirectError() { + return (redirects == null) ? ProcessBuilderForWin32.Redirect.PIPE : redirects[2]; + } + + /** + * Sets the source and destination for subprocess standard I/O + * to be the same as those of the current Java process. + * + *

This is a convenience method. An invocation of the form + * 

 {@code
+     * pb.inheritIO()
+     * }
+ * behaves in exactly the same way as the invocation + * 
 {@code
+     * pb.redirectInput(Redirect.INHERIT)
+     *   .redirectOutput(Redirect.INHERIT)
+     *   .redirectError(Redirect.INHERIT)
+     * }
+ * + * This gives behavior equivalent to most operating system + * command interpreters, or the standard C library function + * {@code system()}. + * + * @return this process builder + * @since 1.7 + */ + public ProcessBuilderForWin32 inheritIO() { + Arrays.fill(redirects(), ProcessBuilderForWin32.Redirect.INHERIT); + return this; + } + + /** + * Tells whether this process builder merges standard error and + * standard output. + * + *

If this property is {@code true}, then any error output + * generated by subprocesses subsequently started by this object's + * {@link #start()} method will be merged with the standard + * output, so that both can be read using the + * {@link Process#getInputStream()} method. This makes it easier + * to correlate error messages with the corresponding output. + * The initial value is {@code false}. + * + * @return this process builder's {@code redirectErrorStream} property + */ + public boolean redirectErrorStream() { + return redirectErrorStream; + } + + /** + * Sets this process builder's {@code redirectErrorStream} property. + * + *

If this property is {@code true}, then any error output + * generated by subprocesses subsequently started by this object's + * {@link #start()} method will be merged with the standard + * output, so that both can be read using the + * {@link Process#getInputStream()} method. This makes it easier + * to correlate error messages with the corresponding output. + * The initial value is {@code false}. + * + * @param redirectErrorStream the new property value + * @return this process builder + */ + public ProcessBuilderForWin32 redirectErrorStream(boolean redirectErrorStream) { + this.redirectErrorStream = redirectErrorStream; + return this; + } + + /** + * Starts a new process using the attributes of this process builder. + * + *

The new process will + * invoke the command and arguments given by {@link #command()}, + * in a working directory as given by {@link #directory()}, + * with a process environment as given by {@link #environment()}. + * + *

This method checks that the command is a valid operating + * system command. Which commands are valid is system-dependent, + * but at the very least the command must be a non-empty list of + * non-null strings. + * + *

A minimal set of system dependent environment variables may + * be required to start a process on some operating systems. + * As a result, the subprocess may inherit additional environment variable + * settings beyond those in the process builder's {@link #environment()}. + * + *

If there is a security manager, its + * {@link SecurityManager#checkExec checkExec} + * method is called with the first component of this object's + * {@code command} array as its argument. This may result in + * a {@link SecurityException} being thrown. + * + *

Starting an operating system process is highly system-dependent. + * Among the many things that can go wrong are: + *

    + *
  • The operating system program file was not found. + *
  • Access to the program file was denied. + *
  • The working directory does not exist. + *
+ * + *

In such cases an exception will be thrown. The exact nature + * of the exception is system-dependent, but it will always be a + * subclass of {@link IOException}. + * + *

Subsequent modifications to this process builder will not + * affect the returned {@link Process}. + * + * @return a new {@link Process} object for managing the subprocess + * + * @throws NullPointerException + * if an element of the command list is null + * + * @throws IndexOutOfBoundsException + * if the command is an empty list (has size {@code 0}) + * + * @throws SecurityException + * if a security manager exists and + *

    + * + *
  • its + * {@link SecurityManager#checkExec checkExec} + * method doesn't allow creation of the subprocess, or + * + *
  • the standard input to the subprocess was + * {@linkplain #redirectInput redirected from a file} + * and the security manager's + * {@link SecurityManager#checkRead checkRead} method + * denies read access to the file, or + * + *
  • the standard output or standard error of the + * subprocess was + * {@linkplain #redirectOutput redirected to a file} + * and the security manager's + * {@link SecurityManager#checkWrite checkWrite} method + * denies write access to the file + * + *
+ * + * @throws IOException if an I/O error occurs + * + * @see Runtime#exec(String[], String[], File) + */ + public Process start() throws IOException { + // Must convert to array first -- a malicious user-supplied + // list might try to circumvent the security check. + String[] cmdarray = command.toArray(new String[command.size()]); + cmdarray = cmdarray.clone(); + + for (String arg : cmdarray) + if (arg == null) + throw new NullPointerException(); + // Throws IndexOutOfBoundsException if command is empty + String prog = cmdarray[0]; + + SecurityManager security = System.getSecurityManager(); + if (security != null) + security.checkExec(prog); + + String dir = directory == null ? null : directory.toString(); + + for (int i = 1; i < cmdarray.length; i++) { + if (cmdarray[i].indexOf('\u0000') >= 0) { + throw new IOException("invalid null character in command"); + } + } + + try { + return ProcessImplForWin32.start( + username, + password, + cmdarray, + environment, + dir, + redirects, + redirectErrorStream); + } catch (IOException | IllegalArgumentException e) { + String exceptionInfo = ": " + e.getMessage(); + Throwable cause = e; + if ((e instanceof IOException) && security != null) { + // Can not disclose the fail reason for read-protected files. + try { + security.checkRead(prog); + } catch (SecurityException se) { + exceptionInfo = ""; + cause = se; + } + } + // It's much easier for us to create a high-quality error + // message than the low-level C code which found the problem. + throw new IOException( + "Cannot run program \"" + prog + "\"" + + (dir == null ? "" : " (in directory \"" + dir + "\")") + + exceptionInfo, + cause); + } + } + +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32.java new file mode 100644 index 0000000000..3dbe7cb50f --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils.process; + +import com.sun.jna.platform.win32.Kernel32Util; + +import java.util.*; + +final class ProcessEnvironmentForWin32 extends HashMap { + + private static final long serialVersionUID = -8017839552603542824L; + + private static String validateName(String name) { + // An initial `=' indicates a magic Windows variable name -- OK + if (name.indexOf('=', 1) != -1 || + name.indexOf('\u0000') != -1) + throw new IllegalArgumentException + ("Invalid environment variable name: \"" + name + "\""); + return name; + } + + private static String validateValue(String value) { + if (value.indexOf('\u0000') != -1) + throw new IllegalArgumentException + ("Invalid environment variable value: \"" + value + "\""); + return value; + } + + private static String nonNullString(Object o) { + if (o == null) + throw new NullPointerException(); + return (String) o; + } + + public String put(String key, String value) { + return super.put(validateName(key), validateValue(value)); + } + + public String get(Object key) { + return super.get(nonNullString(key)); + } + + public boolean containsKey(Object key) { + return super.containsKey(nonNullString(key)); + } + + public boolean containsValue(Object value) { + return super.containsValue(nonNullString(value)); + } + + public String remove(Object key) { + return super.remove(nonNullString(key)); + } + + private static class CheckedEntry implements Entry { + private final Entry e; + public CheckedEntry(Entry e) {this.e = e;} + public String getKey() { return e.getKey();} + public String getValue() { return e.getValue();} + public String setValue(String value) { + return e.setValue(validateValue(value)); + } + public String toString() { return getKey() + "=" + getValue();} + public boolean equals(Object o) {return e.equals(o);} + public int hashCode() {return e.hashCode();} + } + + private static class CheckedEntrySet extends AbstractSet> { + private final Set> s; + public CheckedEntrySet(Set> s) {this.s = s;} + public int size() {return s.size();} + public boolean isEmpty() {return s.isEmpty();} + public void clear() { s.clear();} + public Iterator> iterator() { + return new Iterator>() { + Iterator> i = s.iterator(); + public boolean hasNext() { return i.hasNext();} + public Entry next() { + return new CheckedEntry(i.next()); + } + public void remove() { i.remove();} + }; + } + private static Entry checkedEntry(Object o) { + @SuppressWarnings("unchecked") + Entry e = (Entry) o; + nonNullString(e.getKey()); + nonNullString(e.getValue()); + return e; + } + public boolean contains(Object o) {return s.contains(checkedEntry(o));} + public boolean remove(Object o) {return s.remove(checkedEntry(o));} + } + + private static class CheckedValues extends AbstractCollection { + private final Collection c; + public CheckedValues(Collection c) {this.c = c;} + public int size() {return c.size();} + public boolean isEmpty() {return c.isEmpty();} + public void clear() { c.clear();} + public Iterator iterator() {return c.iterator();} + public boolean contains(Object o) {return c.contains(nonNullString(o));} + public boolean remove(Object o) {return c.remove(nonNullString(o));} + } + + private static class CheckedKeySet extends AbstractSet { + private final Set s; + public CheckedKeySet(Set s) {this.s = s;} + public int size() {return s.size();} + public boolean isEmpty() {return s.isEmpty();} + public void clear() { s.clear();} + public Iterator iterator() {return s.iterator();} + public boolean contains(Object o) {return s.contains(nonNullString(o));} + public boolean remove(Object o) {return s.remove(nonNullString(o));} + } + + public Set keySet() { + return new CheckedKeySet(super.keySet()); + } + + public Collection values() { + return new CheckedValues(super.values()); + } + + public Set> entrySet() { + return new CheckedEntrySet(super.entrySet()); + } + + private static final class NameComparator implements Comparator { + public int compare(String s1, String s2) { + // We can't use String.compareToIgnoreCase since it + // canonicalizes to lower case, while Windows + // canonicalizes to upper case! For example, "_" should + // sort *after* "Z", not before. + int n1 = s1.length(); + int n2 = s2.length(); + int min = Math.min(n1, n2); + for (int i = 0; i < min; i++) { + char c1 = s1.charAt(i); + char c2 = s2.charAt(i); + if (c1 != c2) { + c1 = Character.toUpperCase(c1); + c2 = Character.toUpperCase(c2); + if (c1 != c2) + // No overflow because of numeric promotion + return c1 - c2; + } + } + return n1 - n2; + } + } + + private static final class EntryComparator implements Comparator> { + public int compare(Entry e1, + Entry e2) { + return nameComparator.compare(e1.getKey(), e2.getKey()); + } + } + + // Allow `=' as first char in name, e.g. =C:=C:\DIR + static final int MIN_NAME_LENGTH = 1; + + private static final NameComparator nameComparator; + private static final EntryComparator entryComparator; + private static final ProcessEnvironmentForWin32 theEnvironment; + private static final Map theUnmodifiableEnvironment; + private static final Map theCaseInsensitiveEnvironment; + + static { + nameComparator = new NameComparator(); + entryComparator = new EntryComparator(); + theEnvironment = new ProcessEnvironmentForWin32(); + theUnmodifiableEnvironment = Collections.unmodifiableMap(theEnvironment); + + theEnvironment.putAll(environmentBlock()); + + theCaseInsensitiveEnvironment = new TreeMap<>(nameComparator); + theCaseInsensitiveEnvironment.putAll(theEnvironment); + } + + private ProcessEnvironmentForWin32() { + super(); + } + + private ProcessEnvironmentForWin32(int capacity) { + super(capacity); + } + + // Only for use by System.getenv(String) + static String getenv(String name) { + // The original implementation used a native call to _wgetenv, + // but it turns out that _wgetenv is only consistent with + // GetEnvironmentStringsW (for non-ASCII) if `wmain' is used + // instead of `main', even in a process created using + // CREATE_UNICODE_ENVIRONMENT. Instead we perform the + // case-insensitive comparison ourselves. At least this + // guarantees that System.getenv().get(String) will be + // consistent with System.getenv(String). + return theCaseInsensitiveEnvironment.get(name); + } + + // Only for use by System.getenv() + static Map getenv() { + return theUnmodifiableEnvironment; + } + + // Only for use by ProcessBuilder.environment() + @SuppressWarnings("unchecked") + static Map environment() { + return (Map) theEnvironment.clone(); + } + + // Only for use by ProcessBuilder.environment(String[] envp) + static Map emptyEnvironment(int capacity) { + return new ProcessEnvironmentForWin32(capacity); + } + + private static Map environmentBlock() { + return Kernel32Util.getEnvironmentVariables(); + } + + // Only for use by ProcessImpl.start() + String toEnvironmentBlock() { + // Sort Unicode-case-insensitively by name + List> list = new ArrayList<>(entrySet()); + Collections.sort(list, entryComparator); + + StringBuilder sb = new StringBuilder(size()*30); + int cmp = -1; + + // Some versions of MSVCRT.DLL require SystemRoot to be set. + // So, we make sure that it is always set, even if not provided + // by the caller. + final String SYSTEMROOT = "SystemRoot"; + + for (Entry e : list) { + String key = e.getKey(); + String value = e.getValue(); + if (cmp < 0 && (cmp = nameComparator.compare(key, SYSTEMROOT)) > 0) { + // Not set, so add it here + addToEnvIfSet(sb, SYSTEMROOT); + } + addToEnv(sb, key, value); + } + if (cmp < 0) { + // Got to end of list and still not found + addToEnvIfSet(sb, SYSTEMROOT); + } + if (sb.length() == 0) { + // Environment was empty and SystemRoot not set in parent + sb.append('\u0000'); + } + // Block is double NUL terminated + sb.append('\u0000'); + return sb.toString(); + } + + // add the environment variable to the child, if it exists in parent + private static void addToEnvIfSet(StringBuilder sb, String name) { + String s = getenv(name); + if (s != null) + addToEnv(sb, name, s); + } + + private static void addToEnv(StringBuilder sb, String name, String val) { + sb.append(name).append('=').append(val).append('\u0000'); + } + + static String toEnvironmentBlock(Map map) { + return map == null ? null : ((ProcessEnvironmentForWin32)map).toEnvironmentBlock(); + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java new file mode 100644 index 0000000000..9f2716a096 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java @@ -0,0 +1,785 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils.process; + +import com.sun.jna.Pointer; +import com.sun.jna.platform.win32.*; +import com.sun.jna.ptr.IntByReference; +import java.lang.reflect.Field; +import org.apache.dolphinscheduler.common.utils.OSUtils; +import sun.security.action.GetPropertyAction; + +import java.io.*; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.sun.jna.platform.win32.WinBase.STILL_ACTIVE; +import static java.util.Objects.requireNonNull; + +public class ProcessImplForWin32 extends Process { + + private static final Field FD_HANDLE; + + static { + if (!OSUtils.isWindows()) { + throw new RuntimeException("ProcessImplForWin32 can be only initialized in " + + "Windows environment, but current OS is " + OSUtils.getOSName()); + } + + try { + FD_HANDLE = requireNonNull(FileDescriptor.class.getDeclaredField("handle")); + FD_HANDLE.setAccessible(true); + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); + } + } + + private static final int PIPE_SIZE = 4096 + 24; + + private static final int HANDLE_STORAGE_SIZE = 6; + + private static final int OFFSET_READ = 0; + + private static final int OFFSET_WRITE = 1; + + private static final WinNT.HANDLE JAVA_INVALID_HANDLE_VALUE = new WinNT.HANDLE(Pointer.createConstant(-1)); + + private static void setHandle(FileDescriptor obj, long handle) { + try { + FD_HANDLE.set(obj, handle); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + private static long getHandle(FileDescriptor obj) { + try { + return (Long) FD_HANDLE.get(obj); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + /** + * Open a file for writing. If {@code append} is {@code true} then the file + * is opened for atomic append directly and a FileOutputStream constructed + * with the resulting handle. This is because a FileOutputStream created + * to append to a file does not open the file in a manner that guarantees + * that writes by the child process will be atomic. + */ + private static FileOutputStream newFileOutputStream(File f, boolean append) + throws IOException + { + if (append) { + String path = f.getPath(); + SecurityManager sm = System.getSecurityManager(); + if (sm != null) + sm.checkWrite(path); + long handle = openForAtomicAppend(path); + final FileDescriptor fd = new FileDescriptor(); + setHandle(fd, handle); + return AccessController.doPrivileged( + new PrivilegedAction() { + public FileOutputStream run() { + return new FileOutputStream(fd); + } + } + ); + } else { + return new FileOutputStream(f); + } + } + + // System-dependent portion of ProcessBuilderForWindows.start() + static Process start(String username, + String password, + String cmdarray[], + java.util.Map environment, + String dir, + ProcessBuilderForWin32.Redirect[] redirects, + boolean redirectErrorStream) + throws IOException + { + String envblock = ProcessEnvironmentForWin32.toEnvironmentBlock(environment); + + FileInputStream f0 = null; + FileOutputStream f1 = null; + FileOutputStream f2 = null; + + try { + long[] stdHandles; + if (redirects == null) { + stdHandles = new long[] { -1L, -1L, -1L }; + } else { + stdHandles = new long[3]; + + if (redirects[0] == ProcessBuilderForWin32.Redirect.PIPE) + stdHandles[0] = -1L; + else if (redirects[0] == ProcessBuilderForWin32.Redirect.INHERIT) + stdHandles[0] = getHandle(FileDescriptor.in); + else { + f0 = new FileInputStream(redirects[0].file()); + stdHandles[0] = getHandle(f0.getFD()); + } + + if (redirects[1] == ProcessBuilderForWin32.Redirect.PIPE) + stdHandles[1] = -1L; + else if (redirects[1] == ProcessBuilderForWin32.Redirect.INHERIT) + stdHandles[1] = getHandle(FileDescriptor.out); + else { + f1 = newFileOutputStream(redirects[1].file(), + redirects[1].append()); + stdHandles[1] = getHandle(f1.getFD()); + } + + if (redirects[2] == ProcessBuilderForWin32.Redirect.PIPE) + stdHandles[2] = -1L; + else if (redirects[2] == ProcessBuilderForWin32.Redirect.INHERIT) + stdHandles[2] = getHandle(FileDescriptor.err); + else { + f2 = newFileOutputStream(redirects[2].file(), + redirects[2].append()); + stdHandles[2] = getHandle(f2.getFD()); + } + } + + return new ProcessImplForWin32(username, password, cmdarray, envblock, dir, stdHandles, redirectErrorStream); + } finally { + // In theory, close() can throw IOException + // (although it is rather unlikely to happen here) + try { if (f0 != null) f0.close(); } + finally { + try { if (f1 != null) f1.close(); } + finally { if (f2 != null) f2.close(); } + } + } + + } + + private static class LazyPattern { + // Escape-support version: + // "(\")((?:\\\\\\1|.)+?)\\1|([^\\s\"]+)"; + private static final Pattern PATTERN = + Pattern.compile("[^\\s\"]+|\"[^\"]*\""); + }; + + /* Parses the command string parameter into the executable name and + * program arguments. + * + * The command string is broken into tokens. The token separator is a space + * or quota character. The space inside quotation is not a token separator. + * There are no escape sequences. + */ + private static String[] getTokensFromCommand(String command) { + ArrayList matchList = new ArrayList<>(8); + Matcher regexMatcher = ProcessImplForWin32.LazyPattern.PATTERN.matcher(command); + while (regexMatcher.find()) + matchList.add(regexMatcher.group()); + return matchList.toArray(new String[matchList.size()]); + } + + private static final int VERIFICATION_CMD_BAT = 0; + private static final int VERIFICATION_WIN32 = 1; + private static final int VERIFICATION_WIN32_SAFE = 2; // inside quotes not allowed + private static final int VERIFICATION_LEGACY = 3; + // See Command shell overview for documentation of special characters. + // https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-xp/bb490954(v=technet.10) + private static final char ESCAPE_VERIFICATION[][] = { + // We guarantee the only command file execution for implicit [cmd.exe] run. + // http://technet.microsoft.com/en-us/library/bb490954.aspx + {' ', '\t', '<', '>', '&', '|', '^'}, + {' ', '\t', '<', '>'}, + {' ', '\t', '<', '>'}, + {' ', '\t'} + }; + + private static String createCommandLine(int verificationType, + final String executablePath, + final String cmd[]) + { + StringBuilder cmdbuf = new StringBuilder(80); + + cmdbuf.append(executablePath); + + for (int i = 1; i < cmd.length; ++i) { + cmdbuf.append(' '); + String s = cmd[i]; + if (needsEscaping(verificationType, s)) { + cmdbuf.append('"'); + + if (verificationType == VERIFICATION_WIN32_SAFE) { + // Insert the argument, adding '\' to quote any interior quotes + int length = s.length(); + for (int j = 0; j < length; j++) { + char c = s.charAt(j); + if (c == DOUBLEQUOTE) { + int count = countLeadingBackslash(verificationType, s, j); + while (count-- > 0) { + cmdbuf.append(BACKSLASH); // double the number of backslashes + } + cmdbuf.append(BACKSLASH); // backslash to quote the quote + } + cmdbuf.append(c); + } + } else { + cmdbuf.append(s); + } + // The code protects the [java.exe] and console command line + // parser, that interprets the [\"] combination as an escape + // sequence for the ["] char. + // http://msdn.microsoft.com/en-us/library/17w5ykft.aspx + // + // If the argument is an FS path, doubling of the tail [\] + // char is not a problem for non-console applications. + // + // The [\"] sequence is not an escape sequence for the [cmd.exe] + // command line parser. The case of the [""] tail escape + // sequence could not be realized due to the argument validation + // procedure. + int count = countLeadingBackslash(verificationType, s, s.length()); + while (count-- > 0) { + cmdbuf.append(BACKSLASH); // double the number of backslashes + } + cmdbuf.append('"'); + } else { + cmdbuf.append(s); + } + } + return cmdbuf.toString(); + } + + /** + * Return the argument without quotes (1st and last) if present, else the arg. + * @param str a string + * @return the string without 1st and last quotes + */ + private static String unQuote(String str) { + int len = str.length(); + return (len >= 2 && str.charAt(0) == DOUBLEQUOTE && str.charAt(len - 1) == DOUBLEQUOTE) + ? str.substring(1, len - 1) + : str; + } + + private static boolean needsEscaping(int verificationType, String arg) { + // Switch off MS heuristic for internal ["]. + // Please, use the explicit [cmd.exe] call + // if you need the internal ["]. + // Example: "cmd.exe", "/C", "Extended_MS_Syntax" + + // For [.exe] or [.com] file the unpaired/internal ["] + // in the argument is not a problem. + String unquotedArg = unQuote(arg); + boolean argIsQuoted = !arg.equals(unquotedArg); + boolean embeddedQuote = unquotedArg.indexOf(DOUBLEQUOTE) >= 0; + + switch (verificationType) { + case VERIFICATION_CMD_BAT: + if (embeddedQuote) { + throw new IllegalArgumentException("Argument has embedded quote, " + + "use the explicit CMD.EXE call."); + } + break; // break determine whether to quote + case VERIFICATION_WIN32_SAFE: + if (argIsQuoted && embeddedQuote) { + throw new IllegalArgumentException("Malformed argument has embedded quote: " + + unquotedArg); + } + break; + default: + break; + } + + if (!argIsQuoted) { + char testEscape[] = ESCAPE_VERIFICATION[verificationType]; + for (int i = 0; i < testEscape.length; ++i) { + if (arg.indexOf(testEscape[i]) >= 0) { + return true; + } + } + } + return false; + } + + private static String getExecutablePath(String path) + throws IOException + { + String name = unQuote(path); + if (name.indexOf(DOUBLEQUOTE) >= 0) { + throw new IllegalArgumentException("Executable name has embedded quote, " + + "split the arguments: " + name); + } + // Win32 CreateProcess requires path to be normalized + File fileToRun = new File(name); + + // From the [CreateProcess] function documentation: + // + // "If the file name does not contain an extension, .exe is appended. + // Therefore, if the file name extension is .com, this parameter + // must include the .com extension. If the file name ends in + // a period (.) with no extension, or if the file name contains a path, + // .exe is not appended." + // + // "If the file name !does not contain a directory path!, + // the system searches for the executable file in the following + // sequence:..." + // + // In practice ANY non-existent path is extended by [.exe] extension + // in the [CreateProcess] function with the only exception: + // the path ends by (.) + + return fileToRun.getPath(); + } + + /** + * An executable is any program that is an EXE or does not have an extension + * and the Windows createProcess will be looking for .exe. + * The comparison is case insensitive based on the name. + * @param executablePath the executable file + * @return true if the path ends in .exe or does not have an extension. + */ + private boolean isExe(String executablePath) { + File file = new File(executablePath); + String upName = file.getName().toUpperCase(Locale.ROOT); + return (upName.endsWith(".EXE") || upName.indexOf('.') < 0); + } + + // Old version that can be bypassed + private boolean isShellFile(String executablePath) { + String upPath = executablePath.toUpperCase(); + return (upPath.endsWith(".CMD") || upPath.endsWith(".BAT")); + } + + private String quoteString(String arg) { + StringBuilder argbuf = new StringBuilder(arg.length() + 2); + return argbuf.append('"').append(arg).append('"').toString(); + } + + // Count backslashes before start index of string. + // .bat files don't include backslashes as part of the quote + private static int countLeadingBackslash(int verificationType, + CharSequence input, int start) { + if (verificationType == VERIFICATION_CMD_BAT) + return 0; + int j; + for (j = start - 1; j >= 0 && input.charAt(j) == BACKSLASH; j--) { + // just scanning backwards + } + return (start - 1) - j; // number of BACKSLASHES + } + + private static final char DOUBLEQUOTE = '\"'; + private static final char BACKSLASH = '\\'; + + private WinNT.HANDLE handle; + private OutputStream stdin_stream; + private InputStream stdout_stream; + private InputStream stderr_stream; + + private ProcessImplForWin32( + String username, + String password, + String cmd[], + final String envblock, + final String path, + final long[] stdHandles, + final boolean redirectErrorStream) + throws IOException + { + String cmdstr; + final SecurityManager security = System.getSecurityManager(); + GetPropertyAction action = new GetPropertyAction("jdk.lang.Process.allowAmbiguousCommands", + (security == null) ? "true" : "false"); + final boolean allowAmbiguousCommands = !"false".equalsIgnoreCase(action.run()); + if (allowAmbiguousCommands && security == null) { + // Legacy mode. + + // Normalize path if possible. + String executablePath = new File(cmd[0]).getPath(); + + // No worry about internal, unpaired ["], and redirection/piping. + if (needsEscaping(VERIFICATION_LEGACY, executablePath) ) + executablePath = quoteString(executablePath); + + cmdstr = createCommandLine( + //legacy mode doesn't worry about extended verification + VERIFICATION_LEGACY, + executablePath, + cmd); + } else { + String executablePath; + try { + executablePath = getExecutablePath(cmd[0]); + } catch (IllegalArgumentException e) { + // Workaround for the calls like + // Runtime.getRuntime().exec("\"C:\\Program Files\\foo\" bar") + + // No chance to avoid CMD/BAT injection, except to do the work + // right from the beginning. Otherwise we have too many corner + // cases from + // Runtime.getRuntime().exec(String[] cmd [, ...]) + // calls with internal ["] and escape sequences. + + // Restore original command line. + StringBuilder join = new StringBuilder(); + // terminal space in command line is ok + for (String s : cmd) + join.append(s).append(' '); + + // Parse the command line again. + cmd = getTokensFromCommand(join.toString()); + executablePath = getExecutablePath(cmd[0]); + + // Check new executable name once more + if (security != null) + security.checkExec(executablePath); + } + + // Quotation protects from interpretation of the [path] argument as + // start of longer path with spaces. Quotation has no influence to + // [.exe] extension heuristic. + boolean isShell = allowAmbiguousCommands ? isShellFile(executablePath) + : !isExe(executablePath); + cmdstr = createCommandLine( + // We need the extended verification procedures + isShell ? VERIFICATION_CMD_BAT + : (allowAmbiguousCommands ? VERIFICATION_WIN32 : VERIFICATION_WIN32_SAFE), + quoteString(executablePath), + cmd); + } + + handle = create(username, password, cmdstr, envblock, path, stdHandles, redirectErrorStream); + + AccessController.doPrivileged( + new PrivilegedAction() { + public Void run() { + if (stdHandles[0] == -1L) + stdin_stream = ProcessBuilderForWin32.NullOutputStream.INSTANCE; + else { + FileDescriptor stdin_fd = new FileDescriptor(); + setHandle(stdin_fd, stdHandles[0]); + stdin_stream = new BufferedOutputStream( + new FileOutputStream(stdin_fd)); + } + + if (stdHandles[1] == -1L) + stdout_stream = ProcessBuilderForWin32.NullInputStream.INSTANCE; + else { + FileDescriptor stdout_fd = new FileDescriptor(); + setHandle(stdout_fd, stdHandles[1]); + stdout_stream = new BufferedInputStream( + new FileInputStream(stdout_fd)); + } + + if (stdHandles[2] == -1L) + stderr_stream = ProcessBuilderForWin32.NullInputStream.INSTANCE; + else { + FileDescriptor stderr_fd = new FileDescriptor(); + setHandle(stderr_fd, stdHandles[2]); + stderr_stream = new FileInputStream(stderr_fd); + } + + return null; }}); + } + + public OutputStream getOutputStream() { + return stdin_stream; + } + + public InputStream getInputStream() { + return stdout_stream; + } + + public InputStream getErrorStream() { + return stderr_stream; + } + + protected void finalize() { + closeHandle(handle); + } + + public int exitValue() { + int exitCode = getExitCodeProcess(handle); + if (exitCode == STILL_ACTIVE) + throw new IllegalThreadStateException("process has not exited"); + return exitCode; + } + + public int waitFor() throws InterruptedException { + waitForInterruptibly(handle); + if (Thread.interrupted()) + throw new InterruptedException(); + return exitValue(); + } + + @Override + public boolean waitFor(long timeout, TimeUnit unit) + throws InterruptedException + { + if (getExitCodeProcess(handle) != STILL_ACTIVE) return true; + if (timeout <= 0) return false; + + long remainingNanos = unit.toNanos(timeout); + long deadline = System.nanoTime() + remainingNanos ; + + do { + // Round up to next millisecond + long msTimeout = TimeUnit.NANOSECONDS.toMillis(remainingNanos + 999_999L); + waitForTimeoutInterruptibly(handle, msTimeout); + if (Thread.interrupted()) + throw new InterruptedException(); + if (getExitCodeProcess(handle) != STILL_ACTIVE) { + return true; + } + remainingNanos = deadline - System.nanoTime(); + } while (remainingNanos > 0); + + return (getExitCodeProcess(handle) != STILL_ACTIVE); + } + + public void destroy() { terminateProcess(handle); } + + public Process destroyForcibly() { + destroy(); + return this; + } + + public boolean isAlive() { + return isProcessAlive(handle); + } + + private static boolean initHolder(WinNT.HANDLEByReference pjhandles, + WinNT.HANDLEByReference[] pipe, + int offset, + WinNT.HANDLEByReference phStd) { + if (!pjhandles.getValue().equals(JAVA_INVALID_HANDLE_VALUE)) { + phStd.setValue(pjhandles.getValue()); + pjhandles.setValue(JAVA_INVALID_HANDLE_VALUE); + } else { + if (!Kernel32.INSTANCE.CreatePipe(pipe[0], pipe[1], null, PIPE_SIZE)) { + throw new Win32Exception(Kernel32.INSTANCE.GetLastError()); + } else { + WinNT.HANDLE thisProcessEnd = offset == OFFSET_READ ? pipe[1].getValue() : pipe[0].getValue(); + phStd.setValue(pipe[offset].getValue()); + pjhandles.setValue(thisProcessEnd); + } + } + Kernel32.INSTANCE.SetHandleInformation(phStd.getValue(), Kernel32.HANDLE_FLAG_INHERIT, Kernel32.HANDLE_FLAG_INHERIT); + return true; + } + + private static void releaseHolder(boolean complete, WinNT.HANDLEByReference[] pipe, int offset) { + closeHandle(pipe[offset].getValue()); + if (complete) { + closeHandle(pipe[offset == OFFSET_READ ? OFFSET_WRITE : OFFSET_READ].getValue()); + } + } + + private static void prepareIOEHandleState(WinNT.HANDLE[] stdIOE, Boolean[] inherit) { + for(int i = 0; i < HANDLE_STORAGE_SIZE; ++i) { + WinNT.HANDLE hstd = stdIOE[i]; + if (!Kernel32.INVALID_HANDLE_VALUE.equals(hstd)) { + inherit[i] = Boolean.TRUE; + Kernel32.INSTANCE.SetHandleInformation(hstd, Kernel32.HANDLE_FLAG_INHERIT, 0); + } + } + } + + private static void restoreIOEHandleState(WinNT.HANDLE[] stdIOE, Boolean[] inherit) { + for (int i = HANDLE_STORAGE_SIZE - 1; i >= 0; --i) { + if (!Kernel32.INVALID_HANDLE_VALUE.equals(stdIOE[i])) { + Kernel32.INSTANCE.SetHandleInformation(stdIOE[i], Kernel32.HANDLE_FLAG_INHERIT, inherit[i] ? Kernel32.HANDLE_FLAG_INHERIT : 0); + } + } + } + + private static WinNT.HANDLE processCreate(String username, + String password, + String cmd, + final String envblock, + final String path, + final WinNT.HANDLEByReference[] stdHandles, + final boolean redirectErrorStream) { + WinNT.HANDLE ret = new WinNT.HANDLE(Pointer.createConstant(0)); + + WinNT.HANDLE[] stdIOE = new WinNT.HANDLE[] { + Kernel32.INVALID_HANDLE_VALUE, Kernel32.INVALID_HANDLE_VALUE, Kernel32.INVALID_HANDLE_VALUE, + stdHandles[0].getValue(), stdHandles[1].getValue(), stdHandles[2].getValue() + }; + stdIOE[0] = Kernel32.INSTANCE.GetStdHandle(Kernel32.STD_INPUT_HANDLE); + stdIOE[1] = Kernel32.INSTANCE.GetStdHandle(Kernel32.STD_OUTPUT_HANDLE); + stdIOE[2] = Kernel32.INSTANCE.GetStdHandle(Kernel32.STD_ERROR_HANDLE); + + Boolean[] inherit = new Boolean[] { + Boolean.FALSE, Boolean.FALSE, Boolean.FALSE, + Boolean.FALSE, Boolean.FALSE, Boolean.FALSE + }; + + prepareIOEHandleState(stdIOE, inherit); + + // input + WinNT.HANDLEByReference hStdInput = new WinNT.HANDLEByReference(); + WinNT.HANDLEByReference[] pipeIn = new WinNT.HANDLEByReference[] { + new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE), new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE) }; + + // output + WinNT.HANDLEByReference hStdOutput = new WinNT.HANDLEByReference(); + WinNT.HANDLEByReference[] pipeOut = new WinNT.HANDLEByReference[] { + new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE), new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE) }; + + // error + WinNT.HANDLEByReference hStdError = new WinNT.HANDLEByReference(); + WinNT.HANDLEByReference[] pipeError = new WinNT.HANDLEByReference[] { + new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE), new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE) }; + + boolean success; + if (initHolder(stdHandles[0], pipeIn, OFFSET_READ, hStdInput)) { + if (initHolder(stdHandles[1], pipeOut, OFFSET_WRITE, hStdOutput)) { + WinBase.STARTUPINFO si = new WinBase.STARTUPINFO(); + si.hStdInput = hStdInput.getValue(); + si.hStdOutput = hStdOutput.getValue(); + + if (redirectErrorStream) { + si.hStdError = si.hStdOutput; + stdHandles[2].setValue(JAVA_INVALID_HANDLE_VALUE); + success = true; + } else { + success = initHolder(stdHandles[2], pipeError, OFFSET_WRITE, hStdError); + si.hStdError = hStdError.getValue(); + } + + if (success) { + WTypes.LPSTR lpEnvironment = envblock == null ? new WTypes.LPSTR() : new WTypes.LPSTR(envblock); + Kernel32.PROCESS_INFORMATION pi = new WinBase.PROCESS_INFORMATION(); + si.dwFlags = Kernel32.STARTF_USESTDHANDLES; + if (!Advapi32.INSTANCE.CreateProcessWithLogonW( + username + , null + , password + , Advapi32.LOGON_WITH_PROFILE + , null + , cmd + , Kernel32.CREATE_NO_WINDOW + , lpEnvironment.getPointer() + , path + , si + , pi)) { + throw new Win32Exception(Kernel32.INSTANCE.GetLastError()); + } else { + closeHandle(pi.hThread); + ret = pi.hProcess; + } + } + releaseHolder(ret.getPointer().equals(Pointer.createConstant(0)), pipeError, OFFSET_WRITE); + releaseHolder(ret.getPointer().equals(Pointer.createConstant(0)), pipeOut, OFFSET_WRITE); + } + releaseHolder(ret.getPointer().equals(Pointer.createConstant(0)), pipeIn, OFFSET_READ); + } + restoreIOEHandleState(stdIOE, inherit); + return ret; + } + + private static synchronized WinNT.HANDLE create(String username, + String password, + String cmd, + final String envblock, + final String path, + final long[] stdHandles, + final boolean redirectErrorStream) { + WinNT.HANDLE ret = new WinNT.HANDLE(Pointer.createConstant(0)); + WinNT.HANDLEByReference[] handles = new WinNT.HANDLEByReference[stdHandles.length]; + for (int i = 0; i < stdHandles.length; i++) { + handles[i] = new WinNT.HANDLEByReference(new WinNT.HANDLE(Pointer.createConstant(stdHandles[i]))); + } + + if (cmd != null) { + if (username != null && password != null) { + ret = processCreate(username, password, cmd, envblock, path, handles, redirectErrorStream); + } + } + + for (int i = 0; i < stdHandles.length; i++) { + stdHandles[i] = handles[i].getPointer().getLong(0); + } + + return ret; + } + + private static int getExitCodeProcess(WinNT.HANDLE handle) { + IntByReference exitStatus = new IntByReference(); + if (!Kernel32.INSTANCE.GetExitCodeProcess(handle, exitStatus)) { + throw new Win32Exception(Kernel32.INSTANCE.GetLastError()); + } + return exitStatus.getValue(); + } + + private static void terminateProcess(WinNT.HANDLE handle) { + Kernel32.INSTANCE.TerminateProcess(handle, 1); + } + + private static boolean isProcessAlive(WinNT.HANDLE handle) { + IntByReference exitStatus = new IntByReference(); + Kernel32.INSTANCE.GetExitCodeProcess(handle, exitStatus); + return exitStatus.getValue() == STILL_ACTIVE; + } + + private static void closeHandle(WinNT.HANDLE handle) { + Kernel32Util.closeHandle(handle); + } + + /** + * Opens a file for atomic append. The file is created if it doesn't + * already exist. + * + * @param path the file to open or create + * @return the native HANDLE + */ + private static long openForAtomicAppend(String path) throws IOException { + int access = Kernel32.GENERIC_READ | Kernel32.GENERIC_WRITE; + int sharing = Kernel32.FILE_SHARE_READ | Kernel32.FILE_SHARE_WRITE; + int disposition = Kernel32.OPEN_ALWAYS; + int flagsAndAttributes = Kernel32.FILE_ATTRIBUTE_NORMAL; + if (path == null || path.isEmpty()) { + return -1; + } else { + WinNT.HANDLE handle = Kernel32.INSTANCE.CreateFile(path, access, sharing, null, disposition, flagsAndAttributes, null); + if (handle == Kernel32.INVALID_HANDLE_VALUE) { + throw new Win32Exception(Kernel32.INSTANCE.GetLastError()); + } + return handle.getPointer().getLong(0); + } + } + + private static void waitForInterruptibly(WinNT.HANDLE handle) { + int result = Kernel32.INSTANCE.WaitForMultipleObjects(1, new WinNT.HANDLE[]{handle}, false, Kernel32.INFINITE); + if (result == Kernel32.WAIT_FAILED) { + throw new Win32Exception(Kernel32.INSTANCE.GetLastError()); + } + } + + private static void waitForTimeoutInterruptibly(WinNT.HANDLE handle, long timeout) { + int result = Kernel32.INSTANCE.WaitForMultipleObjects(1, new WinNT.HANDLE[]{handle}, false, (int) timeout); + if (result == Kernel32.WAIT_FAILED) { + throw new Win32Exception(Kernel32.INSTANCE.GetLastError()); + } + } + +} diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index 5a4aa1441f..8391a9e2d6 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -91,4 +91,4 @@ yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx # If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s - +kerberos.expire.time=7 diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java new file mode 100644 index 0000000000..3280a9629f --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common; + +import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.junit.Assert; +import org.junit.Test; + +/** + * Constants Test + */ +public class ConstantsTest { + + /** + * Test PID via env + */ + @Test + public void testPID() { + if (OSUtils.isWindows()) { + Assert.assertEquals(Constants.PID, "handle"); + } else { + Assert.assertEquals(Constants.PID, "pid"); + } + } + +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java index 8745a4f6b4..ff298000f5 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java @@ -27,8 +27,6 @@ import org.slf4j.Marker; import java.util.Map; -import static org.junit.Assert.*; - public class TaskLogDiscriminatorTest { /** diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java index 52767074da..5cca6403c8 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java @@ -21,7 +21,6 @@ import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.classic.spi.IThrowableProxy; import ch.qos.logback.classic.spi.LoggerContextVO; import ch.qos.logback.core.spi.FilterReply; -import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.junit.Assert; import org.junit.Test; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java new file mode 100644 index 0000000000..7ce00e875a --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.task; + +import org.apache.dolphinscheduler.common.process.ResourceInfo; +import org.apache.dolphinscheduler.common.task.flink.FlinkParameters; +import org.junit.Assert; +import org.junit.Test; + +import java.util.LinkedList; +import java.util.List; + +public class FlinkParametersTest { + @Test + public void getResourceFilesList() { + FlinkParameters flinkParameters = new FlinkParameters(); + Assert.assertNotNull(flinkParameters.getResourceFilesList()); + Assert.assertTrue(flinkParameters.getResourceFilesList().isEmpty()); + + ResourceInfo mainResource = new ResourceInfo(); + mainResource.setRes("testFlinkMain-1.0.0-SNAPSHOT.jar"); + flinkParameters.setMainJar(mainResource); + + List resourceInfos = new LinkedList<>(); + ResourceInfo resourceInfo1 = new ResourceInfo(); + resourceInfo1.setRes("testFlinkParameters1.jar"); + resourceInfos.add(resourceInfo1); + + flinkParameters.setResourceList(resourceInfos); + Assert.assertNotNull(flinkParameters.getResourceFilesList()); + Assert.assertEquals(2, flinkParameters.getResourceFilesList().size()); + + ResourceInfo resourceInfo2 = new ResourceInfo(); + resourceInfo2.setRes("testFlinkParameters2.jar"); + resourceInfos.add(resourceInfo2); + + flinkParameters.setResourceList(resourceInfos); + Assert.assertNotNull(flinkParameters.getResourceFilesList()); + Assert.assertEquals(3, flinkParameters.getResourceFilesList().size()); + } +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java index 89458f6f1c..96217842bf 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java @@ -30,29 +30,32 @@ public class FileUtilsTest { @Test public void suffix() { - Assert.assertEquals(FileUtils.suffix("ninfor.java"),"java"); + Assert.assertEquals("java", FileUtils.suffix("ninfor.java")); + Assert.assertEquals("", FileUtils.suffix(null)); + Assert.assertEquals("", FileUtils.suffix("")); + Assert.assertEquals("", FileUtils.suffix("ninfor-java")); } @Test public void testGetDownloadFilename() { PowerMockito.mockStatic(DateUtils.class); PowerMockito.when(DateUtils.getCurrentTime(YYYYMMDDHHMMSS)).thenReturn("20190101101059"); - Assert.assertEquals(FileUtils.getDownloadFilename("test"), - "/tmp/dolphinscheduler/download/20190101101059/test"); + Assert.assertEquals("/tmp/dolphinscheduler/download/20190101101059/test", + FileUtils.getDownloadFilename("test")); } @Test public void testGetUploadFilename() { - Assert.assertEquals(FileUtils.getUploadFilename("aaa","bbb"), - "/tmp/dolphinscheduler/aaa/resources/bbb"); + Assert.assertEquals("/tmp/dolphinscheduler/aaa/resources/bbb", + FileUtils.getUploadFilename("aaa","bbb")); } @Test public void testGetProcessExecDir() { String dir = FileUtils.getProcessExecDir(1,2,3, 4); - Assert.assertEquals(dir, "/tmp/dolphinscheduler/exec/process/1/2/3/4"); + Assert.assertEquals("/tmp/dolphinscheduler/exec/process/1/2/3/4", dir); dir = FileUtils.getProcessExecDir(1,2,3); - Assert.assertEquals(dir, "/tmp/dolphinscheduler/exec/process/1/2/3"); + Assert.assertEquals("/tmp/dolphinscheduler/exec/process/1/2/3", dir); } @Test diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java index 20994ac99f..17929f9344 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.common.utils; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,7 +38,7 @@ public class HttpUtilsTest { String result = HttpUtils.get("https://github.com/manifest.json"); Assert.assertNotNull(result); JSONObject jsonObject = JSON.parseObject(result); - Assert.assertEquals(jsonObject.getString("name"), "GitHub"); + Assert.assertEquals("GitHub", jsonObject.getString("name")); result = HttpUtils.get("https://123.333.111.33/ccc"); Assert.assertNull(result); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/IpUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/IpUtilsTest.java index e65bcd219b..ec6ffa35a7 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/IpUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/IpUtilsTest.java @@ -29,13 +29,13 @@ public class IpUtilsTest { long longNumber = IpUtils.ipToLong(ip); long longNumber2 = IpUtils.ipToLong(ip2); System.out.println(longNumber); - Assert.assertEquals(longNumber, 3232263681L); - Assert.assertEquals(longNumber2, 0L); + Assert.assertEquals(3232263681L, longNumber); + Assert.assertEquals(0L, longNumber2); String ip3 = "255.255.255.255"; long longNumber3 = IpUtils.ipToLong(ip3); System.out.println(longNumber3); - Assert.assertEquals(longNumber3, 4294967295L); + Assert.assertEquals(4294967295L, longNumber3); } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java index bd924e4852..8ce60349ed 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java @@ -16,10 +16,10 @@ */ package org.apache.dolphinscheduler.common.utils; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.Direct; import org.apache.dolphinscheduler.common.process.Property; -import com.alibaba.fastjson.JSONObject; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import org.junit.Assert; @@ -40,8 +40,8 @@ public class JSONUtilsTest { String jsonStr = "{\"id\":\"1001\",\"name\":\"Jobs\"}"; Map models = JSONUtils.toMap(jsonStr); - Assert.assertEquals(models.get("id"), "1001"); - Assert.assertEquals(models.get("name"), "Jobs"); + Assert.assertEquals("1001", models.get("id")); + Assert.assertEquals("Jobs", models.get("name")); } @@ -53,9 +53,9 @@ public class JSONUtilsTest { property.setType(DataType.VARCHAR); property.setValue("sssssss"); String str = "{\"direct\":\"IN\",\"prop\":\"ds\",\"type\":\"VARCHAR\",\"value\":\"sssssss\"}"; - Property property1 = JSONObject.parseObject(str, Property.class); + Property property1 = JSON.parseObject(str, Property.class); Direct direct = property1.getDirect(); - Assert.assertEquals(direct , Direct.IN); + Assert.assertEquals(Direct.IN, direct); } @@ -66,12 +66,12 @@ public class JSONUtilsTest { List maps = JSONUtils.toList(str, LinkedHashMap.class); - Assert.assertEquals(maps.size(), 1); - Assert.assertEquals(maps.get(0).get("mysql service name"), "mysql200"); - Assert.assertEquals(maps.get(0).get("mysql address"), "192.168.xx.xx"); - Assert.assertEquals(maps.get(0).get("port"), "3306"); - Assert.assertEquals(maps.get(0).get("no index of number"), "80"); - Assert.assertEquals(maps.get(0).get("database client connections"), "190"); + Assert.assertEquals(1, maps.size()); + Assert.assertEquals("mysql200", maps.get(0).get("mysql service name")); + Assert.assertEquals("192.168.xx.xx", maps.get(0).get("mysql address")); + Assert.assertEquals("3306", maps.get(0).get("port")); + Assert.assertEquals("80", maps.get(0).get("no index of number")); + Assert.assertEquals("190", maps.get(0).get("database client connections")); } public String list2String(){ diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java index 7106804aaf..b955787c69 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java @@ -39,16 +39,20 @@ public class OSUtilsTest { @Test public void testOSMetric(){ - double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); - Assert.assertTrue(availablePhysicalMemorySize > 0.0f); - double totalMemorySize = OSUtils.totalMemorySize(); - Assert.assertTrue(totalMemorySize > 0.0f); - double loadAverage = OSUtils.loadAverage(); - logger.info("loadAverage {}", loadAverage); - double memoryUsage = OSUtils.memoryUsage(); - Assert.assertTrue(memoryUsage > 0.0f); - double cpuUsage = OSUtils.cpuUsage(); - Assert.assertTrue(cpuUsage > 0.0f); + if (!OSUtils.isWindows()) { + double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); + Assert.assertTrue(availablePhysicalMemorySize > 0.0f); + double totalMemorySize = OSUtils.totalMemorySize(); + Assert.assertTrue(totalMemorySize > 0.0f); + double loadAverage = OSUtils.loadAverage(); + logger.info("loadAverage {}", loadAverage); + double memoryUsage = OSUtils.memoryUsage(); + Assert.assertTrue(memoryUsage > 0.0f); + double cpuUsage = OSUtils.cpuUsage(); + Assert.assertTrue(cpuUsage > 0.0f); + } else { + // TODO window ut + } } @Test diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java index 8bb64b03c8..abdc15cc6e 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import org.apache.commons.lang.time.DateUtils; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DataType; @@ -91,13 +91,13 @@ public class ParameterUtilsTest { globalParamList.add(property); String result2 = ParameterUtils.curingGlobalParams(null,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,scheduleTime); - Assert.assertEquals(result2, JSONObject.toJSONString(globalParamList)); + Assert.assertEquals(result2, JSON.toJSONString(globalParamList)); String result3 = ParameterUtils.curingGlobalParams(globalParamMap,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,null); - Assert.assertEquals(result3, JSONObject.toJSONString(globalParamList)); + Assert.assertEquals(result3, JSON.toJSONString(globalParamList)); String result4 = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime); - Assert.assertEquals(result4, JSONObject.toJSONString(globalParamList)); + Assert.assertEquals(result4, JSON.toJSONString(globalParamList)); //test var $ startsWith globalParamMap.put("bizDate","${system.biz.date}"); diff --git a/dolphinscheduler-service/src/test/java/utils/PreconditionsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java similarity index 98% rename from dolphinscheduler-service/src/test/java/utils/PreconditionsTest.java rename to dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java index a1b85f1b12..47b24bb93c 100644 --- a/dolphinscheduler-service/src/test/java/utils/PreconditionsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java @@ -14,9 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package utils; +package org.apache.dolphinscheduler.common.utils; -import org.apache.dolphinscheduler.common.utils.Preconditions; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java index 907a09e458..7885806b96 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java @@ -19,15 +19,12 @@ package org.apache.dolphinscheduler.common.utils; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.Mockito; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileNotFoundException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java index 947e7310db..eca22def30 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java @@ -19,8 +19,6 @@ package org.apache.dolphinscheduler.common.utils; import org.junit.Assert; import org.junit.Test; -import java.util.ArrayList; - public class StringUtilsTest { @Test public void testIsNotEmpty() { diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java index db4a86bc26..b316b17469 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java @@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.common.utils; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32Test.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32Test.java new file mode 100644 index 0000000000..ce04346743 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32Test.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils.process; + +import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +@RunWith(PowerMockRunner.class) +@PrepareForTest(OSUtils.class) +public class ProcessBuilderForWin32Test { + + private static final Logger logger = LoggerFactory.getLogger(ProcessBuilderForWin32Test.class); + + @Before + public void before() { + PowerMockito.mockStatic(OSUtils.class); + PowerMockito.when(OSUtils.isWindows()).thenReturn(true); + } + + @Test + public void testCreateProcessBuilderForWin32() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + Assert.assertNotNull(builder); + + builder = new ProcessBuilderForWin32("net"); + Assert.assertNotNull(builder); + + builder = new ProcessBuilderForWin32(Collections.singletonList("net")); + Assert.assertNotNull(builder); + + builder = new ProcessBuilderForWin32((List) null); + Assert.assertNotNull(builder); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testBuildUser() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + builder.user("test", StringUtils.EMPTY); + Assert.assertNotNull(builder); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testBuildCommand() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + builder.command(Collections.singletonList("net")); + Assert.assertNotEquals(0, builder.command().size()); + + builder = new ProcessBuilderForWin32(); + builder.command("net"); + Assert.assertNotEquals(0, builder.command().size()); + + builder = new ProcessBuilderForWin32(); + builder.command((List) null); + Assert.assertNotEquals(0, builder.command().size()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testEnvironment() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + Assert.assertNotNull(builder.environment()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + builder.environment(new String[]{ "a=123" }); + Assert.assertNotEquals(0, builder.environment().size()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testDirectory() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + builder.directory(new File("/tmp")); + Assert.assertNotNull(builder.directory()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testStream() { + try { + InputStream in = ProcessBuilderForWin32.NullInputStream.INSTANCE; + Assert.assertNotNull(in); + Assert.assertEquals(-1, in.read()); + Assert.assertEquals(0, in.available()); + + OutputStream out = ProcessBuilderForWin32.NullOutputStream.INSTANCE; + Assert.assertNotNull(out); + out.write(new byte[] {1}); + } catch (Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testRedirect() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + + builder.redirectInput(new File("/tmp")); + Assert.assertNotNull(builder.redirectInput()); + Assert.assertNotNull(builder.redirectInput().file()); + + builder.redirectOutput(new File("/tmp")); + Assert.assertNotNull(builder.redirectOutput()); + Assert.assertNotNull(builder.redirectOutput().file()); + + builder.redirectError(new File("/tmp")); + Assert.assertNotNull(builder.redirectError()); + Assert.assertNotNull(builder.redirectError().file()); + + builder.redirectInput(builder.redirectOutput()); + builder.redirectOutput(builder.redirectInput()); + builder.redirectError(builder.redirectInput()); + + Assert.assertNotNull(ProcessBuilderForWin32.Redirect.PIPE.type()); + Assert.assertNotNull(ProcessBuilderForWin32.Redirect.PIPE.toString()); + Assert.assertNotNull(ProcessBuilderForWin32.Redirect.INHERIT.type()); + Assert.assertNotNull(ProcessBuilderForWin32.Redirect.INHERIT.toString()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testRedirectErrorStream() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + builder.redirectErrorStream(true); + Assert.assertTrue(builder.redirectErrorStream()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void runCmdViaUser() { + try { + ProcessBuilderForWin32 builder = new ProcessBuilderForWin32(); + builder.user("test123", StringUtils.EMPTY); + + List commands = new ArrayList<>(); + commands.add("cmd.exe"); + commands.add("/c"); + commands.add("net user"); + builder.command(commands); + + Process process = builder.start(); + BufferedReader inReader = new BufferedReader(new InputStreamReader(process.getInputStream(), Charset.forName("GBK"))); + String line; + StringBuilder sb = new StringBuilder(); + while ((line = inReader.readLine()) != null) { + sb.append(line); + } + logger.info("net user: {}", sb.toString()); + Assert.assertNotEquals(StringUtils.EMPTY, sb.toString()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32Test.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32Test.java new file mode 100644 index 0000000000..00c54c0164 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32Test.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils.process; + +import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.Map; +import java.util.Set; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({OSUtils.class, ProcessEnvironmentForWin32.class}) +public class ProcessEnvironmentForWin32Test { + + private static final Logger logger = LoggerFactory.getLogger(ProcessBuilderForWin32Test.class); + + @Before + public void before() { + try { + PowerMockito.mockStatic(OSUtils.class); + PowerMockito.when(OSUtils.isWindows()).thenReturn(true); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testPutAndGet() { + try { + ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0); + processEnvironmentForWin32.put("a", "123"); + Assert.assertEquals("123", processEnvironmentForWin32.get("a")); + Assert.assertTrue(processEnvironmentForWin32.containsKey("a")); + Assert.assertTrue(processEnvironmentForWin32.containsValue("123")); + Assert.assertEquals("123", processEnvironmentForWin32.remove("a")); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + + try { + ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0); + processEnvironmentForWin32.put("b=", "123"); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + + try { + ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0); + processEnvironmentForWin32.put("b", "\u0000"); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + + try { + ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0); + processEnvironmentForWin32.get(null); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testEntrySet() { + try { + ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0); + processEnvironmentForWin32.clear(); + processEnvironmentForWin32.put("a", "123"); + Assert.assertEquals(0, processEnvironmentForWin32.entrySet().size()); + Assert.assertTrue(processEnvironmentForWin32.entrySet().isEmpty()); + for (Map.Entry entry : processEnvironmentForWin32.entrySet()) { + Assert.assertNotNull(entry); + Assert.assertNotNull(entry.getKey()); + Assert.assertNotNull(entry.getValue()); + Assert.assertNotNull(entry.setValue("123")); + } + + processEnvironmentForWin32.clear(); + Set keys = processEnvironmentForWin32.keySet(); + Assert.assertEquals(0, keys.size()); + Assert.assertTrue(keys.isEmpty()); + + processEnvironmentForWin32.clear(); + Collection values = processEnvironmentForWin32.values(); + Assert.assertEquals(0, keys.size()); + Assert.assertTrue(keys.isEmpty()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + @Test + public void testToEnvironmentBlock() { + try { + ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0); + Assert.assertNotNull(processEnvironmentForWin32.toEnvironmentBlock()); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32Test.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32Test.java new file mode 100644 index 0000000000..3f8bcbfb66 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32Test.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils.process; + +import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sun.security.action.GetPropertyAction; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({OSUtils.class, GetPropertyAction.class}) +public class ProcessImplForWin32Test { + + private static final Logger logger = LoggerFactory.getLogger(ProcessBuilderForWin32Test.class); + + @Before + public void before() { + PowerMockito.mockStatic(OSUtils.class); + PowerMockito.mockStatic(GetPropertyAction.class); + PowerMockito.when(OSUtils.isWindows()).thenReturn(true); + } + + @Test + public void testStart() { + try { + Process process = ProcessImplForWin32.start( + "test123", StringUtils.EMPTY, new String[]{"net"}, + null, null, null, false); + Assert.assertNotNull(process); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + + try { + Process process = ProcessImplForWin32.start( + "test123", StringUtils.EMPTY, new String[]{"net"}, + null, null, new ProcessBuilderForWin32.Redirect[]{ + ProcessBuilderForWin32.Redirect.PIPE, + ProcessBuilderForWin32.Redirect.PIPE, + ProcessBuilderForWin32.Redirect.PIPE + }, false); + Assert.assertNotNull(process); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + +} diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml index 20d19410e2..3aea888f94 100644 --- a/dolphinscheduler-dao/pom.xml +++ b/dolphinscheduler-dao/pom.xml @@ -25,7 +25,7 @@ dolphinscheduler-dao ${project.artifactId} - http://maven.apache.org + UTF-8 @@ -44,6 +44,12 @@ com.baomidou mybatis-plus-boot-starter ${mybatis-plus.version} + + + org.apache.logging.log4j + log4j-to-slf4j + + org.postgresql @@ -71,6 +77,14 @@ log4j-api org.apache.logging.log4j + + org.springframework.boot + spring-boot-starter-tomcat + + + org.apache.logging.log4j + log4j-to-slf4j + @@ -78,7 +92,10 @@ mysql mysql-connector-java - + + com.h2database + h2 + com.alibaba druid diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java index 19dbf46a6b..1a8c09e611 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java @@ -144,9 +144,11 @@ public class AlertDao extends AbstractBaseDao { * @param taskId taskId * @param taskName taskName */ - public void sendTaskTimeoutAlert(int alertgroupId,String receivers,String receiversCc,int taskId,String taskName){ + public void sendTaskTimeoutAlert(int alertgroupId,String receivers,String receiversCc, int processInstanceId, + String processInstanceName, int taskId,String taskName){ Alert alert = new Alert(); - String content = String.format("[{'id':'%d','name':'%s','event':'timeout','warnLevel':'middle'}]",taskId,taskName); + String content = String.format("[{'process instance id':'%d','task name':'%s','task id':'%d','task name':'%s'," + + "'event':'timeout','warnLevel':'middle'}]", processInstanceId, processInstanceName, taskId, taskName); alert.setTitle("Task Timeout Warn"); alert.setShowType(ShowType.TABLE); alert.setContent(content); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java index 51f60666d1..5ea5966238 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java @@ -61,7 +61,7 @@ public class MonitorDBDao { return new PostgrePerformance().getMonitorRecord(conn); } }catch (Exception e) { - logger.error("SQLException " + e); + logger.error("SQLException: {}", e.getMessage(), e); }finally { try { if (conn != null) { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java index f7ba1054a5..f89002ebdc 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java @@ -50,25 +50,28 @@ public class TaskRecordDao { static { try { conf = new PropertiesConfiguration(Constants.APPLICATION_PROPERTIES); - }catch (ConfigurationException e){ - logger.error("load configuration exception",e); + } catch (ConfigurationException e) { + logger.error("load configuration exception", e); System.exit(1); } } /** - * get task record flag + * get task record flag + * * @return whether startup taskrecord */ - public static boolean getTaskRecordFlag(){ - return conf.getBoolean(Constants.TASK_RECORD_FLAG); + public static boolean getTaskRecordFlag() { + return conf.getBoolean(Constants.TASK_RECORD_FLAG); } + /** * create connection + * * @return connection */ private static Connection getConn() { - if(!getTaskRecordFlag()){ + if (!getTaskRecordFlag()) { return null; } String driver = "com.mysql.jdbc.Driver"; @@ -90,101 +93,96 @@ public class TaskRecordDao { /** * generate where sql string + * * @param filterMap filterMap * @return sql string */ private static String getWhereString(Map filterMap) { - if(filterMap.size() ==0){ + if (filterMap.size() == 0) { return ""; } String result = " where 1=1 "; Object taskName = filterMap.get("taskName"); - if(taskName != null && StringUtils.isNotEmpty(taskName.toString())){ + if (taskName != null && StringUtils.isNotEmpty(taskName.toString())) { result += " and PROC_NAME like concat('%', '" + taskName.toString() + "', '%') "; } Object taskDate = filterMap.get("taskDate"); - if(taskDate != null && StringUtils.isNotEmpty(taskDate.toString())){ + if (taskDate != null && StringUtils.isNotEmpty(taskDate.toString())) { result += " and PROC_DATE='" + taskDate.toString() + "'"; } Object state = filterMap.get("state"); - if(state != null && StringUtils.isNotEmpty(state.toString())){ + if (state != null && StringUtils.isNotEmpty(state.toString())) { result += " and NOTE='" + state.toString() + "'"; } Object sourceTable = filterMap.get("sourceTable"); - if(sourceTable!= null && StringUtils.isNotEmpty(sourceTable.toString())){ - result += " and SOURCE_TAB like concat('%', '" + sourceTable.toString()+ "', '%')"; + if (sourceTable != null && StringUtils.isNotEmpty(sourceTable.toString())) { + result += " and SOURCE_TAB like concat('%', '" + sourceTable.toString() + "', '%')"; } Object targetTable = filterMap.get("targetTable"); - if(sourceTable!= null && StringUtils.isNotEmpty(targetTable.toString())){ - result += " and TARGET_TAB like concat('%', '"+ targetTable.toString()+"', '%') " ; + if (sourceTable != null && StringUtils.isNotEmpty(targetTable.toString())) { + result += " and TARGET_TAB like concat('%', '" + targetTable.toString() + "', '%') "; } Object start = filterMap.get("startTime"); - if(start != null && StringUtils.isNotEmpty(start.toString())){ + if (start != null && StringUtils.isNotEmpty(start.toString())) { result += " and STARTDATE>='" + start.toString() + "'"; } Object end = filterMap.get("endTime"); - if(end != null && StringUtils.isNotEmpty(end.toString())){ - result += " and ENDDATE>='" + end.toString()+ "'"; + if (end != null && StringUtils.isNotEmpty(end.toString())) { + result += " and ENDDATE>='" + end.toString() + "'"; } return result; } /** * count task record + * * @param filterMap filterMap - * @param table table + * @param table table * @return task record count */ - public static int countTaskRecord(Map filterMap, String table){ + public static int countTaskRecord(Map filterMap, String table) { int count = 0; Connection conn = null; PreparedStatement pstmt = null; + ResultSet rs = null; try { conn = getConn(); - if(conn == null){ + if (conn == null) { return count; } String sql = String.format("select count(1) as count from %s", table); sql += getWhereString(filterMap); pstmt = conn.prepareStatement(sql); - ResultSet rs = pstmt.executeQuery(); - while(rs.next()){ + rs = pstmt.executeQuery(); + while (rs.next()) { count = rs.getInt("count"); break; } } catch (SQLException e) { logger.error("Exception ", e); - }finally { - try { - if(pstmt != null) { - pstmt.close(); - } - if(conn != null){ - conn.close(); - } - } catch (SQLException e) { - logger.error("Exception ", e); - } + } finally { + closeResource(rs, pstmt, conn); } return count; } /** * query task record by filter map paging + * * @param filterMap filterMap - * @param table table + * @param table table * @return task record list */ - public static List queryAllTaskRecord(Map filterMap , String table) { + public static List queryAllTaskRecord(Map filterMap, String table) { String sql = String.format("select * from %s", table); sql += getWhereString(filterMap); @@ -194,9 +192,9 @@ public class TaskRecordDao { sql += String.format(" order by STARTDATE desc limit %d,%d", offset, pageSize); List recordList = new ArrayList<>(); - try{ + try { recordList = getQueryResult(sql); - }catch (Exception e){ + } catch (Exception e) { logger.error("Exception ", e); } return recordList; @@ -204,6 +202,7 @@ public class TaskRecordDao { /** * convert result set to task record + * * @param resultSet resultSet * @return task record * @throws SQLException if error throws SQLException @@ -232,6 +231,7 @@ public class TaskRecordDao { /** * query task list by select sql + * * @param selectSql select sql * @return task record list */ @@ -239,65 +239,81 @@ public class TaskRecordDao { List recordList = new ArrayList<>(); Connection conn = null; PreparedStatement pstmt = null; + ResultSet rs = null; try { conn = getConn(); - if(conn == null){ + if (conn == null) { return recordList; } pstmt = conn.prepareStatement(selectSql); - ResultSet rs = pstmt.executeQuery(); + rs = pstmt.executeQuery(); - while(rs.next()){ + while (rs.next()) { TaskRecord taskRecord = convertToTaskRecord(rs); recordList.add(taskRecord); } } catch (SQLException e) { logger.error("Exception ", e); - }finally { - try { - if(pstmt != null) { - pstmt.close(); - } - if(conn != null){ - conn.close(); - } - } catch (SQLException e) { - logger.error("Exception ", e); - } + } finally { + closeResource(rs, pstmt, conn); } return recordList; } /** * according to procname and procdate query task record + * * @param procName procName * @param procDate procDate * @return task record status */ - public static TaskRecordStatus getTaskRecordState(String procName,String procDate){ + public static TaskRecordStatus getTaskRecordState(String procName, String procDate) { String sql = String.format("SELECT * FROM eamp_hive_log_hd WHERE PROC_NAME='%s' and PROC_DATE like '%s'" - ,procName,procDate + "%"); + , procName, procDate + "%"); List taskRecordList = getQueryResult(sql); // contains no record and sql exception - if (CollectionUtils.isEmpty(taskRecordList)){ + if (CollectionUtils.isEmpty(taskRecordList)) { // exception return TaskRecordStatus.EXCEPTION; - }else if (taskRecordList.size() > 1){ + } else if (taskRecordList.size() > 1) { return TaskRecordStatus.EXCEPTION; - }else { + } else { TaskRecord taskRecord = taskRecordList.get(0); - if (taskRecord == null){ + if (taskRecord == null) { return TaskRecordStatus.EXCEPTION; } Long targetRowCount = taskRecord.getTargetRowCount(); - if (targetRowCount <= 0){ + if (targetRowCount <= 0) { return TaskRecordStatus.FAILURE; - }else { + } else { return TaskRecordStatus.SUCCESS; } } } + private static void closeResource(ResultSet rs, PreparedStatement pstmt, Connection conn) { + if (rs != null) { + try { + rs.close(); + } catch (SQLException e) { + logger.error("Exception ", e); + } + } + if (pstmt != null) { + try { + pstmt.close(); + } catch (SQLException e) { + logger.error("Exception ", e); + } + } + if (conn != null) { + try { + conn.close(); + } catch (SQLException e) { + logger.error("Exception ", e); + } + } + } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java index 0a8f527536..4dfb8817a8 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java @@ -39,7 +39,7 @@ public class HiveDataSource extends BaseDataSource { @Override public String getJdbcUrl() { String jdbcUrl = getAddress(); - if (jdbcUrl.lastIndexOf("/") != (jdbcUrl.length() - 1)) { + if (jdbcUrl.lastIndexOf('/') != (jdbcUrl.length() - 1)) { jdbcUrl += "/"; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java index 6e2fbe3dd8..969c17b54d 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java @@ -39,7 +39,7 @@ public class MySQLDataSource extends BaseDataSource { @Override public String getJdbcUrl() { String address = getAddress(); - if (address.lastIndexOf("/") != (address.length() - 1)) { + if (address.lastIndexOf('/') != (address.length() - 1)) { address += "/"; } String jdbcUrl = address + getDatabase(); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java index 176cba2587..c01dfa4dc0 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java @@ -40,7 +40,7 @@ public class PostgreDataSource extends BaseDataSource { @Override public String getJdbcUrl() { String jdbcUrl = getAddress(); - if (jdbcUrl.lastIndexOf("/") != (jdbcUrl.length() - 1)) { + if (jdbcUrl.lastIndexOf('/') != (jdbcUrl.length() - 1)) { jdbcUrl += "/"; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java index 6e7ea7d64f..f59d11f3fe 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java @@ -16,10 +16,10 @@ */ package org.apache.dolphinscheduler.dao.entity; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.process.Property; -import com.alibaba.fastjson.JSONObject; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; @@ -29,7 +29,6 @@ import com.baomidou.mybatisplus.core.toolkit.StringUtils; import java.util.Date; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.stream.Collectors; @@ -267,7 +266,7 @@ public class ProcessDefinition { } public void setGlobalParams(String globalParams) { - this.globalParamList = JSONObject.parseArray(globalParams, Property.class); + this.globalParamList = JSON.parseArray(globalParams, Property.class); this.globalParams = globalParams; } @@ -276,7 +275,7 @@ public class ProcessDefinition { } public void setGlobalParamList(List globalParamList) { - this.globalParams = JSONObject.toJSONString(globalParamList); + this.globalParams = JSON.toJSONString(globalParamList); this.globalParamList = globalParamList; } @@ -284,7 +283,7 @@ public class ProcessDefinition { List propList; if (globalParamMap == null && StringUtils.isNotEmpty(globalParams)) { - propList = JSONObject.parseArray(globalParams, Property.class); + propList = JSON.parseArray(globalParams, Property.class); globalParamMap = propList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java index fb5d0cda42..fd9653a36c 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java @@ -506,7 +506,7 @@ public class ProcessInstance { * check this process is start complement data * @return whether complement data */ - public Boolean isComplementData(){ + public boolean isComplementData(){ if(!StringUtils.isNotEmpty(this.historyCmd)){ return false; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java index c692575e3a..702897fba9 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java @@ -373,7 +373,7 @@ public class TaskInstance { } - public Boolean isSubProcess(){ + public boolean isSubProcess(){ return TaskType.SUB_PROCESS.getDescp().equals(this.taskType); } @@ -442,7 +442,7 @@ public class TaskInstance { this.executorName = executorName; } - public Boolean isTaskComplete() { + public boolean isTaskComplete() { return this.getState().typeIsPause() || this.getState().typeIsSuccess() diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java index 9093034ae8..3518676337 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.dao.entity; -import com.baomidou.mybatisplus.annotation.TableField; import org.apache.dolphinscheduler.common.enums.UdfType; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java index 5e5277b997..a20a3acb95 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.dao.upgrade; import org.apache.dolphinscheduler.common.utils.ConnectionUtils; -import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,15 +29,7 @@ import java.sql.SQLException; */ public class MysqlUpgradeDao extends UpgradeDao { - public static final Logger logger = LoggerFactory.getLogger(UpgradeDao.class); - - /** - * init - */ - @Override - protected void init() { - - } + public static final Logger logger = LoggerFactory.getLogger(MysqlUpgradeDao.class); /** * mysql upgrade dao holder @@ -70,12 +61,7 @@ public class MysqlUpgradeDao extends UpgradeDao { try { conn = dataSource.getConnection(); rs = conn.getMetaData().getTables(null, null, tableName, null); - if (rs.next()) { - return true; - } else { - return false; - } - + return rs.next(); } catch (SQLException e) { logger.error(e.getMessage(),e); throw new RuntimeException(e.getMessage(),e); @@ -97,11 +83,7 @@ public class MysqlUpgradeDao extends UpgradeDao { try { conn = dataSource.getConnection(); ResultSet rs = conn.getMetaData().getColumns(null,null,tableName,columnName); - if (rs.next()) { - return true; - } else { - return false; - } + return rs.next(); } catch (SQLException e) { logger.error(e.getMessage(),e); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java index c0dc905848..c58521de1d 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.dao.upgrade; import org.apache.dolphinscheduler.common.utils.ConnectionUtils; -import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,16 +30,8 @@ import java.sql.SQLException; */ public class PostgresqlUpgradeDao extends UpgradeDao { - public static final Logger logger = LoggerFactory.getLogger(UpgradeDao.class); - private static final String schema = getSchema(); - - /** - * init - */ - @Override - protected void init() { - - } + public static final Logger logger = LoggerFactory.getLogger(PostgresqlUpgradeDao.class); + private static final String SCHEMA = getSchema(); /** * postgresql upgrade dao holder @@ -59,16 +50,6 @@ public class PostgresqlUpgradeDao extends UpgradeDao { return PostgresqlUpgradeDaoHolder.INSTANCE; } - - /** - * init schema - * @param initSqlPath initSqlPath - */ - @Override - public void initSchema(String initSqlPath) { - super.initSchema(initSqlPath); - } - /** * getSchema * @return schema @@ -108,13 +89,9 @@ public class PostgresqlUpgradeDao extends UpgradeDao { try { conn = dataSource.getConnection(); - rs = conn.getMetaData().getTables(null, schema, tableName, null); - if (rs.next()) { - return true; - } else { - return false; - } + rs = conn.getMetaData().getTables(null, SCHEMA, tableName, null); + return rs.next(); } catch (SQLException e) { logger.error(e.getMessage(),e); throw new RuntimeException(e.getMessage(),e); @@ -136,13 +113,8 @@ public class PostgresqlUpgradeDao extends UpgradeDao { ResultSet rs = null; try { conn = dataSource.getConnection(); - rs = conn.getMetaData().getColumns(null,schema,tableName,columnName); - if (rs.next()) { - return true; - } else { - return false; - } - + rs = conn.getMetaData().getColumns(null, SCHEMA,tableName,columnName); + return rs.next(); } catch (SQLException e) { logger.error(e.getMessage(),e); throw new RuntimeException(e.getMessage(),e); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java index 40d12ab36f..1e1ee78036 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java @@ -27,7 +27,6 @@ import java.util.Date; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.Flag; -import org.apache.dolphinscheduler.dao.MonitorDBDao; import org.apache.dolphinscheduler.dao.entity.MonitorRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +36,7 @@ import org.slf4j.LoggerFactory; */ public class MysqlPerformance extends BaseDBPerformance{ - private static Logger logger = LoggerFactory.getLogger(MonitorDBDao.class); + private static Logger logger = LoggerFactory.getLogger(MysqlPerformance.class); /** diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgrePerformance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgrePerformance.java index 031fd00681..b1cdf6f179 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgrePerformance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgrePerformance.java @@ -24,7 +24,6 @@ import java.util.Date; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.Flag; -import org.apache.dolphinscheduler.dao.MonitorDBDao; import org.apache.dolphinscheduler.dao.entity.MonitorRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,7 +33,7 @@ import org.slf4j.LoggerFactory; */ public class PostgrePerformance extends BaseDBPerformance { - private static Logger logger = LoggerFactory.getLogger(MonitorDBDao.class); + private static Logger logger = LoggerFactory.getLogger(PostgrePerformance.class); /** * get monitor record diff --git a/dolphinscheduler-dao/src/main/resources/application.properties b/dolphinscheduler-dao/src/main/resources/application.properties index 34bb9f916b..06b0ee94d5 100644 --- a/dolphinscheduler-dao/src/main/resources/application.properties +++ b/dolphinscheduler-dao/src/main/resources/application.properties @@ -23,6 +23,10 @@ spring.datasource.url=jdbc:postgresql://localhost:5432/dolphinscheduler # mysql #spring.datasource.driver-class-name=com.mysql.jdbc.Driver #spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 +# h2 +#spring.datasource.driver-class-name=org.h2.Driver +#spring.datasource.url=jdbc:h2:file:../sql/h2;AUTO_SERVER=TRUE + spring.datasource.username=test spring.datasource.password=test diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java index 3653e6a53d..5fb7dfc09f 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java @@ -67,7 +67,7 @@ public class ErrorCommandMapperTest { //update errorCommand.setUpdateTime(new Date()); int update = errorCommandMapper.updateById(errorCommand); - Assert.assertEquals(update, 1); + Assert.assertEquals(1,update); errorCommandMapper.deleteById(errorCommand.getId()); } @@ -79,7 +79,7 @@ public class ErrorCommandMapperTest { ErrorCommand errorCommand = insertOne(); int delete = errorCommandMapper.deleteById(errorCommand.getId()); - Assert.assertEquals(delete, 1); + Assert.assertEquals(1,delete); } /** diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java index 1b3db55608..775d10656a 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java @@ -77,7 +77,7 @@ public class ProcessDefinitionMapperTest { //update processDefinition.setUpdateTime(new Date()); int update = processDefinitionMapper.updateById(processDefinition); - Assert.assertEquals(update, 1); + Assert.assertEquals(1, update); processDefinitionMapper.deleteById(processDefinition.getId()); } @@ -88,7 +88,7 @@ public class ProcessDefinitionMapperTest { public void testDelete(){ ProcessDefinition processDefinition = insertOne(); int delete = processDefinitionMapper.deleteById(processDefinition.getId()); - Assert.assertEquals(delete, 1); + Assert.assertEquals(1, delete); } /** @@ -189,7 +189,7 @@ public class ProcessDefinitionMapperTest { List processDefinitions = processDefinitionMapper.queryDefinitionListByIdList(array); processDefinitionMapper.deleteById(processDefinition.getId()); processDefinitionMapper.deleteById(processDefinition1.getId()); - Assert.assertEquals(processDefinitions.size(), 2); + Assert.assertEquals(2, processDefinitions.size()); } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java index 102d8cd5db..6e995ef3c7 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.dao.mapper; -import org.apache.dolphinscheduler.common.enums.ExecutionStatus; -import org.apache.dolphinscheduler.dao.entity.ProjectUser; import org.apache.dolphinscheduler.dao.entity.ProjectUser; import org.junit.Assert; import org.junit.Test; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java index 62b034a081..30d2be03e0 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.dao.mapper; -import org.apache.dolphinscheduler.dao.entity.Queue; import org.apache.dolphinscheduler.dao.entity.Queue; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; diff --git a/dolphinscheduler-dist/pom.xml b/dolphinscheduler-dist/pom.xml index 80ddfec9b0..0ead2b9880 100644 --- a/dolphinscheduler-dist/pom.xml +++ b/dolphinscheduler-dist/pom.xml @@ -220,10 +220,7 @@ ${basedir}/../dolphinscheduler-alert/src/main/resources - **/*.properties - **/*.xml - **/*.json - **/*.ftl + **/*.* @@ -232,9 +229,7 @@ ${basedir}/../dolphinscheduler-common/src/main/resources - **/*.properties - **/*.xml - **/*.json + **/*.* @@ -243,10 +238,7 @@ ${basedir}/../dolphinscheduler-dao/src/main/resources - **/*.properties - **/*.xml - **/*.json - **/*.yml + **/*.* @@ -255,9 +247,7 @@ ${basedir}/../dolphinscheduler-api/src/main/resources - **/*.properties - **/*.xml - **/*.json + **/*.* @@ -266,13 +256,19 @@ ${basedir}/../dolphinscheduler-server/src/main/resources - **/*.properties - **/*.xml - **/*.json config/*.* + + + ${basedir}/../dolphinscheduler-service/src/main/resources + + + *.* + + + ${basedir}/../script @@ -342,14 +338,6 @@ - - - ${basedir}/../dolphinscheduler-ui - - - install-dolphinscheduler-ui.sh - - ${basedir}/release-docs @@ -362,7 +350,7 @@ - /opt/soft/${project.build.finalName}/dist + /opt/soft/${project.build.finalName}/ui 755 root root @@ -391,6 +379,14 @@ **/*.* + + + ${basedir}/../sql + + + soft_version + + @@ -405,7 +401,7 @@ ${basedir}/../script - **/*.* + *.sh @@ -416,6 +412,9 @@ + + + diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml index b67b033ffa..39c7c6a7c0 100644 --- a/dolphinscheduler-remote/pom.xml +++ b/dolphinscheduler-remote/pom.xml @@ -12,8 +12,6 @@ dolphinscheduler-remote dolphinscheduler-remote - - http://www.example.com UTF-8 diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java index 29b2317633..dbeb318f2d 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java @@ -145,7 +145,7 @@ public class NettyRemotingServer { try { future = serverBootstrap.bind(serverConfig.getListenPort()).sync(); } catch (Exception e) { - logger.error("NettyRemotingServer bind fail {}, exit", e); + logger.error("NettyRemotingServer bind fail {}, exit",e.getMessage(), e); throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort())); } if (future.isSuccess()) { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java index 86ba79c884..ed46e1ff51 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java @@ -16,8 +16,6 @@ */ package org.apache.dolphinscheduler.remote.command; -import com.sun.org.apache.regexp.internal.RE; - import java.io.Serializable; import java.util.concurrent.atomic.AtomicLong; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskRequestCommand.java index beec055403..93c536c347 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskRequestCommand.java @@ -1 +1 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; import java.util.List; import java.util.concurrent.atomic.AtomicLong; /** * execute task request command */ public class ExecuteTaskRequestCommand implements Serializable { /** * task id */ private String taskId; /** * attempt id */ private String attemptId; /** * application name */ private String applicationName; /** * group name */ private String groupName; /** * task name */ private String taskName; /** * connector port */ private int connectorPort; /** * description info */ private String description; /** * class name */ private String className; /** * method name */ private String methodName; /** * parameters */ private String params; /** * shard itemds */ private List shardItems; public List getShardItems() { return shardItems; } public void setShardItems(List shardItems) { this.shardItems = shardItems; } public String getParams() { return params; } public void setParams(String params) { this.params = params; } public String getTaskId() { return taskId; } public void setTaskId(String taskId) { this.taskId = taskId; } public String getApplicationName() { return applicationName; } public void setApplicationName(String applicationName) { this.applicationName = applicationName; } public String getGroupName() { return groupName; } public void setGroupName(String groupName) { this.groupName = groupName; } public String getTaskName() { return taskName; } public void setTaskName(String taskName) { this.taskName = taskName; } public int getConnectorPort() { return connectorPort; } public void setConnectorPort(int connectorPort) { this.connectorPort = connectorPort; } public String getDescription() { return description; } public void setDescription(String description) { this.description = description; } public String getClassName() { return className; } public void setClassName(String className) { this.className = className; } public String getMethodName() { return methodName; } public void setMethodName(String methodName) { this.methodName = methodName; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.EXECUTE_TASK_REQUEST); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } } \ No newline at end of file +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; import java.util.List; /** * execute task request command */ public class ExecuteTaskRequestCommand implements Serializable { /** * task id */ private String taskId; /** * attempt id */ private String attemptId; /** * application name */ private String applicationName; /** * group name */ private String groupName; /** * task name */ private String taskName; /** * connector port */ private int connectorPort; /** * description info */ private String description; /** * class name */ private String className; /** * method name */ private String methodName; /** * parameters */ private String params; /** * shard itemds */ private List shardItems; public List getShardItems() { return shardItems; } public void setShardItems(List shardItems) { this.shardItems = shardItems; } public String getParams() { return params; } public void setParams(String params) { this.params = params; } public String getTaskId() { return taskId; } public void setTaskId(String taskId) { this.taskId = taskId; } public String getApplicationName() { return applicationName; } public void setApplicationName(String applicationName) { this.applicationName = applicationName; } public String getGroupName() { return groupName; } public void setGroupName(String groupName) { this.groupName = groupName; } public String getTaskName() { return taskName; } public void setTaskName(String taskName) { this.taskName = taskName; } public int getConnectorPort() { return connectorPort; } public void setConnectorPort(int connectorPort) { this.connectorPort = connectorPort; } public String getDescription() { return description; } public void setDescription(String description) { this.description = description; } public String getClassName() { return className; } public void setClassName(String className) { this.className = className; } public String getMethodName() { return methodName; } public void setMethodName(String methodName) { this.methodName = methodName; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.EXECUTE_TASK_REQUEST); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } } \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskResponseCommand.java index 7e35fa6e75..7543fc3d0e 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskResponseCommand.java @@ -1 +1 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; import java.util.concurrent.atomic.AtomicLong; /** * execute task response command */ public class ExecuteTaskResponseCommand implements Serializable { /** * task id */ private String taskId; /** * attempt id */ private String attemptId; /** * return result */ private Object result; /** * received time */ private long receivedTime; /** * execute count */ private int executeCount; /** * execute time */ private long executeTime; public String getAttemptId() { return attemptId; } public void setAttemptId(String attemptId) { this.attemptId = attemptId; } public String getTaskId() { return taskId; } public void setTaskId(String taskId) { this.taskId = taskId; } public Object getResult() { return result; } public void setResult(Object result) { this.result = result; } public long getReceivedTime() { return receivedTime; } public void setReceivedTime(long receivedTime) { this.receivedTime = receivedTime; } public int getExecuteCount() { return executeCount; } public void setExecuteCount(int executeCount) { this.executeCount = executeCount; } public long getExecuteTime() { return executeTime; } public void setExecuteTime(long executeTime) { this.executeTime = executeTime; } public Command convert2Command(long opaque){ Command command = new Command(); command.setType(CommandType.EXECUTE_TASK_RESPONSE); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } } \ No newline at end of file +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; /** * execute task response command */ public class ExecuteTaskResponseCommand implements Serializable { /** * task id */ private String taskId; /** * attempt id */ private String attemptId; /** * return result */ private Object result; /** * received time */ private long receivedTime; /** * execute count */ private int executeCount; /** * execute time */ private long executeTime; public String getAttemptId() { return attemptId; } public void setAttemptId(String attemptId) { this.attemptId = attemptId; } public String getTaskId() { return taskId; } public void setTaskId(String taskId) { this.taskId = taskId; } public Object getResult() { return result; } public void setResult(Object result) { this.result = result; } public long getReceivedTime() { return receivedTime; } public void setReceivedTime(long receivedTime) { this.receivedTime = receivedTime; } public int getExecuteCount() { return executeCount; } public void setExecuteCount(int executeCount) { this.executeCount = executeCount; } public long getExecuteTime() { return executeTime; } public void setExecuteTime(long executeTime) { this.executeTime = executeTime; } public Command convert2Command(long opaque){ Command command = new Command(); command.setType(CommandType.EXECUTE_TASK_RESPONSE); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } } \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java index c50413e98a..f90d3fff18 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java @@ -21,7 +21,6 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import java.io.Serializable; -import java.util.concurrent.atomic.AtomicLong; /** * ping machine @@ -31,12 +30,12 @@ public class Ping implements Serializable { /** * ping body */ - protected static ByteBuf EMPTY_BODY = Unpooled.EMPTY_BUFFER; + protected static final ByteBuf EMPTY_BODY = Unpooled.EMPTY_BUFFER; /** * request command body */ - private static byte[] EMPTY_BODY_ARRAY = new byte[0]; + private static final byte[] EMPTY_BODY_ARRAY = new byte[0]; private static final ByteBuf PING_BUF; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Pong.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Pong.java index e52cef6d92..1b51373bff 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Pong.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Pong.java @@ -30,12 +30,12 @@ public class Pong implements Serializable { /** * pong body */ - protected static ByteBuf EMPTY_BODY = Unpooled.EMPTY_BUFFER; + protected static final ByteBuf EMPTY_BODY = Unpooled.EMPTY_BUFFER; /** * pong command body */ - private static byte[] EMPTY_BODY_ARRAY = new byte[0]; + private static final byte[] EMPTY_BODY_ARRAY = new byte[0]; /** * ping byte buffer diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java index 4cc32ed42a..20cf8d9102 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java @@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; -import java.util.concurrent.atomic.AtomicLong; /** * get log bytes request command diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java index 621d35a804..433c4a0b1f 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java @@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; -import java.util.concurrent.atomic.AtomicLong; /** * roll view log request command diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java index 8835348ee3..b4773d0c0d 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java @@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; -import java.util.concurrent.atomic.AtomicLong; /** * view log request command diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java index d5d0d4df83..80e561d05c 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java @@ -107,7 +107,7 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { */ @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { - logger.error("exceptionCaught : {}", cause); + logger.error("exceptionCaught : {}",cause.getMessage(), cause); nettyRemotingClient.closeChannel(ChannelUtils.toAddress(ctx.channel())); ctx.channel().close(); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java index eabd6560de..22251c35c1 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java @@ -140,7 +140,7 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { */ @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { - logger.error("exceptionCaught : {}", cause); + logger.error("exceptionCaught : {}",cause.getMessage(), cause); ctx.channel().close(); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java index 5733b17790..f4791715b8 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.remote.utils; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; /** @@ -31,7 +32,7 @@ public class Constants { /** * charset */ - public static final Charset UTF8 = Charset.forName("UTF-8"); + public static final Charset UTF8 = StandardCharsets.UTF_8; /** * cpus diff --git a/dolphinscheduler-server/pom.xml b/dolphinscheduler-server/pom.xml index 080b87ebaa..e8e84297e2 100644 --- a/dolphinscheduler-server/pom.xml +++ b/dolphinscheduler-server/pom.xml @@ -25,7 +25,7 @@ dolphinscheduler-server dolphinscheduler-server - http://maven.apache.org + jar UTF-8 @@ -86,6 +86,12 @@ org.apache.curator curator-recipes + + + org.apache.zookeeper + zookeeper + + org.apache.zookeeper diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java index 4e4404ea1c..4b21d5f4a2 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.log; import io.netty.channel.Channel; +import org.apache.dolphinscheduler.common.utils.IOUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.log.*; @@ -116,16 +117,8 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { }catch (IOException e){ logger.error("get file bytes error",e); }finally { - if (bos != null){ - try { - bos.close(); - } catch (IOException ignore) {} - } - if (in != null){ - try { - in.close(); - } catch (IOException ignore) {} - } + IOUtils.closeQuietly(bos); + IOUtils.closeQuietly(in); } return new byte[0]; } @@ -146,7 +139,7 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { } catch (IOException e) { logger.error("read file error",e); } - return Collections.EMPTY_LIST; + return Collections.emptyList(); } /** @@ -168,11 +161,7 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { }catch (IOException e){ logger.error("read file error",e); }finally { - try { - if (br != null){ - br.close(); - } - } catch (IOException ignore) {} + IOUtils.closeQuietly(br); } return ""; } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index 6b5063cba4..bf58607262 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.runner.MasterSchedulerThread; +import org.apache.dolphinscheduler.server.worker.WorkerServer; import org.apache.dolphinscheduler.server.zk.ZKMasterClient; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; @@ -37,8 +38,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.WebApplicationType; import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.FilterType; import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -46,7 +49,9 @@ import java.util.concurrent.TimeUnit; /** * master server */ -@ComponentScan("org.apache.dolphinscheduler") +@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = { + @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, classes = {WorkerServer.class}) +}) public class MasterServer implements IStoppable { /** @@ -112,7 +117,7 @@ public class MasterServer implements IStoppable { masterSchedulerService = ThreadUtils.newDaemonSingleThreadExecutor("Master-Scheduler-Thread"); - heartbeatMasterService = ThreadUtils.newDaemonThreadScheduledExecutor("Master-Main-Thread",Constants.DEFAULT_MASTER_HEARTBEAT_THREAD_NUM); + heartbeatMasterService = ThreadUtils.newThreadScheduledExecutor("Master-Main-Thread",Constants.DEFAULT_MASTER_HEARTBEAT_THREAD_NUM, false); // heartbeat thread implement Runnable heartBeatThread = heartBeatThread(); @@ -147,23 +152,17 @@ public class MasterServer implements IStoppable { } logger.error("start Quartz failed", e); } - - - /** - * register hooks, which are called before the process exits - */ - Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { - @Override - public void run() { - if (zkMasterClient.getActiveMasterNum() <= 1) { - zkMasterClient.getAlertDao().sendServerStopedAlert( - 1, OSUtils.getHost(), "Master-Server"); - } - stop("shutdownhook"); - } - })); } + @PreDestroy + public void destroy() { + // master server exit alert + if (zkMasterClient.getActiveMasterNum() <= 1) { + zkMasterClient.getAlertDao().sendServerStopedAlert( + 1, OSUtils.getHost(), "Master-Server"); + } + stop("shutdownhook"); + } /** * gracefully stop @@ -244,7 +243,7 @@ public class MasterServer implements IStoppable { */ private Runnable heartBeatThread(){ logger.info("start master heart beat thread..."); - Runnable heartBeatThread = new Runnable() { + return new Runnable() { @Override public void run() { if(Stopper.isRunning()) { @@ -258,7 +257,6 @@ public class MasterServer implements IStoppable { } } }; - return heartBeatThread; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 4b22b27bec..26457a386a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.server.master.runner; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import com.google.common.collect.Lists; import org.apache.commons.io.FileUtils; import org.apache.dolphinscheduler.common.Constants; @@ -68,7 +68,7 @@ public class MasterExecThread implements Runnable { /** * runing TaskNode */ - private final Map> activeTaskNode = new ConcurrentHashMap>(); + private final Map> activeTaskNode = new ConcurrentHashMap<>(); /** * task exec service @@ -78,7 +78,7 @@ public class MasterExecThread implements Runnable { /** * submit failure nodes */ - private Boolean taskFailedSubmit = false; + private boolean taskFailedSubmit = false; /** * recover node id list @@ -454,7 +454,7 @@ public class MasterExecThread implements Runnable { // process instance id taskInstance.setProcessInstanceId(processInstance.getId()); // task instance node json - taskInstance.setTaskJson(JSONObject.toJSONString(taskNode)); + taskInstance.setTaskJson(JSON.toJSONString(taskNode)); // task instance type taskInstance.setTaskType(taskNode.getType()); // task instance whether alert @@ -565,7 +565,7 @@ public class MasterExecThread implements Runnable { TaskInstance taskInstance = completeTaskList.get(nodeName); if(taskInstance == null){ - logger.error("task instance cannot find, please check it!", nodeName); + logger.error("task instance {} cannot find, please check it!", nodeName); return conditionTaskList; } @@ -652,7 +652,7 @@ public class MasterExecThread implements Runnable { continue; } if(task.getState().typeIsPause() || task.getState().typeIsCancel()){ - logger.info("task {} stopped, the state is {}", task.getName(), task.getState().toString()); + logger.info("task {} stopped, the state is {}", task.getName(), task.getState()); }else{ addTaskToStandByList(task); } @@ -685,11 +685,12 @@ public class MasterExecThread implements Runnable { } ExecutionStatus depTaskState = completeTaskList.get(depsNode).getState(); // conditions task would not return failed. - if(depTaskState.typeIsFailure()){ - if(!haveConditionsAfterNode(depsNode) && !dag.getNode(depsNode).isConditionsTask()){ - return DependResult.FAILED; - } + if(depTaskState.typeIsFailure() + && !haveConditionsAfterNode(depsNode) + && !dag.getNode(depsNode).isConditionsTask()){ + return DependResult.FAILED; } + if(depTaskState.typeIsPause() || depTaskState.typeIsCancel()){ return DependResult.WAITING; } @@ -737,7 +738,7 @@ public class MasterExecThread implements Runnable { * * @return Boolean whether has failed task */ - private Boolean hasFailedTask(){ + private boolean hasFailedTask(){ if(this.taskFailedSubmit){ return true; @@ -753,7 +754,7 @@ public class MasterExecThread implements Runnable { * * @return Boolean whether process instance failed */ - private Boolean processFailed(){ + private boolean processFailed(){ if(hasFailedTask()) { if(processInstance.getFailureStrategy() == FailureStrategy.END){ return true; @@ -769,9 +770,9 @@ public class MasterExecThread implements Runnable { * whether task for waiting thread * @return Boolean whether has waiting thread task */ - private Boolean hasWaitingThreadTask(){ + private boolean hasWaitingThreadTask(){ List waitingList = getCompleteTaskByState(ExecutionStatus.WAITTING_THREAD); - return waitingList.size() > 0; + return CollectionUtils.isNotEmpty(waitingList); } /** @@ -787,7 +788,7 @@ public class MasterExecThread implements Runnable { } List pauseList = getCompleteTaskByState(ExecutionStatus.PAUSE); - if(pauseList.size() > 0 + if(CollectionUtils.isNotEmpty(pauseList) || !isComplementEnd() || readyToSubmitTaskList.size() > 0){ return ExecutionStatus.PAUSE; @@ -827,7 +828,8 @@ public class MasterExecThread implements Runnable { if(state == ExecutionStatus.READY_STOP){ List stopList = getCompleteTaskByState(ExecutionStatus.STOP); List killList = getCompleteTaskByState(ExecutionStatus.KILL); - if(stopList.size() > 0 || killList.size() > 0 || !isComplementEnd()){ + if(CollectionUtils.isNotEmpty(stopList) + || CollectionUtils.isNotEmpty(killList) || !isComplementEnd()){ return ExecutionStatus.STOP; }else{ return ExecutionStatus.SUCCESS; @@ -852,7 +854,7 @@ public class MasterExecThread implements Runnable { * whether complement end * @return Boolean whether is complement end */ - private Boolean isComplementEnd() { + private boolean isComplementEnd() { if(!processInstance.isComplementData()){ return true; } @@ -877,8 +879,8 @@ public class MasterExecThread implements Runnable { logger.info( "work flow process instance [id: {}, name:{}], state change from {} to {}, cmd type: {}", processInstance.getId(), processInstance.getName(), - processInstance.getState().toString(), state.toString(), - processInstance.getCommandType().toString()); + processInstance.getState(), state, + processInstance.getCommandType()); processInstance.setState(state); ProcessInstance instance = processService.findProcessInstanceById(processInstance.getId()); instance.setState(state); @@ -894,8 +896,7 @@ public class MasterExecThread implements Runnable { * @return DependResult */ private DependResult getDependResultForTask(TaskInstance taskInstance){ - DependResult inner = isTaskDepsComplete(taskInstance.getName()); - return inner; + return isTaskDepsComplete(taskInstance.getName()); } /** @@ -920,7 +921,7 @@ public class MasterExecThread implements Runnable { * has retry task in standby * @return Boolean whether has retry task in standby */ - private Boolean hasRetryTaskInStandBy(){ + private boolean hasRetryTaskInStandBy(){ for (Map.Entry entry: readyToSubmitTaskList.entrySet()) { if(entry.getValue().getState().typeIsFailure()){ return true; @@ -958,7 +959,7 @@ public class MasterExecThread implements Runnable { continue; } logger.info("task :{}, id:{} complete, state is {} ", - task.getName(), task.getId(), task.getState().toString()); + task.getName(), task.getId(), task.getState()); // node success , post node submit if(task.getState() == ExecutionStatus.SUCCESS){ completeTaskList.put(task.getName(), task); @@ -990,7 +991,7 @@ public class MasterExecThread implements Runnable { completeTaskList.put(task.getName(), task); } // send alert - if(this.recoverToleranceFaultTaskList.size() > 0){ + if(CollectionUtils.isNotEmpty(this.recoverToleranceFaultTaskList)){ alertManager.sendAlertWorkerToleranceFault(processInstance, recoverToleranceFaultTaskList); this.recoverToleranceFaultTaskList.clear(); } @@ -1034,10 +1035,7 @@ public class MasterExecThread implements Runnable { Date now = new Date(); long runningTime = DateUtils.diffMin(now, processInstance.getStartTime()); - if(runningTime > processInstance.getTimeout()){ - return true; - } - return false; + return runningTime > processInstance.getTimeout(); } /** @@ -1062,7 +1060,7 @@ public class MasterExecThread implements Runnable { TaskInstance taskInstance = taskExecThread.getTaskInstance(); taskInstance = processService.findTaskInstanceById(taskInstance.getId()); - if(taskInstance.getState().typeIsFinished()){ + if(taskInstance != null && taskInstance.getState().typeIsFinished()){ continue; } @@ -1081,22 +1079,19 @@ public class MasterExecThread implements Runnable { * @param taskInstance task instance * @return Boolean */ - private Boolean retryTaskIntervalOverTime(TaskInstance taskInstance){ + private boolean retryTaskIntervalOverTime(TaskInstance taskInstance){ if(taskInstance.getState() != ExecutionStatus.FAILURE){ - return Boolean.TRUE; + return true; } if(taskInstance.getId() == 0 || taskInstance.getMaxRetryTimes() ==0 || taskInstance.getRetryInterval() == 0 ){ - return Boolean.TRUE; + return true; } Date now = new Date(); long failedTimeInterval = DateUtils.differSec(now, taskInstance.getEndTime()); // task retry does not over time, return false - if(taskInstance.getRetryInterval() * SEC_2_MINUTES_TIME_UNIT >= failedTimeInterval){ - return Boolean.FALSE; - } - return Boolean.TRUE; + return taskInstance.getRetryInterval() * SEC_2_MINUTES_TIME_UNIT < failedTimeInterval; } /** @@ -1189,7 +1184,7 @@ public class MasterExecThread implements Runnable { */ private List getRecoveryNodeNameList(){ List recoveryNodeNameList = new ArrayList<>(); - if(recoverNodeIdList.size() > 0) { + if(CollectionUtils.isNotEmpty(recoverNodeIdList)) { for (TaskInstance task : recoverNodeIdList) { recoveryNodeNameList.add(task.getName()); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java index 66d1a3f4c2..38b5419a92 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.server.master.runner; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy; @@ -25,7 +26,6 @@ import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import com.alibaba.fastjson.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,7 +64,7 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { /** * whether already Killed,default false */ - private Boolean alreadyKilled = false; + private boolean alreadyKilled = false; /** * submit task instance and wait complete @@ -98,7 +98,7 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { logger.info("wait task: process id: {}, task id:{}, task name:{} complete", this.taskInstance.getProcessInstanceId(), this.taskInstance.getId(), this.taskInstance.getName()); // task time out - Boolean checkTimeout = false; + boolean checkTimeout = false; TaskTimeoutParameter taskTimeoutParameter = getTaskTimeoutParameter(); if(taskTimeoutParameter.getEnable()){ TaskTimeoutStrategy strategy = taskTimeoutParameter.getStrategy(); @@ -128,7 +128,9 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { // process define ProcessDefinition processDefine = processService.findProcessDefineById(processInstance.getProcessDefinitionId()); // send warn mail - alertDao.sendTaskTimeoutAlert(processInstance.getWarningGroupId(),processDefine.getReceivers(),processDefine.getReceiversCc(),taskInstance.getId(),taskInstance.getName()); + alertDao.sendTaskTimeoutAlert(processInstance.getWarningGroupId(),processDefine.getReceivers(), + processDefine.getReceiversCc(), processInstance.getId(), processInstance.getName(), + taskInstance.getId(),taskInstance.getName()); checkTimeout = false; } } @@ -174,7 +176,7 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { */ private TaskTimeoutParameter getTaskTimeoutParameter(){ String taskJson = taskInstance.getTaskJson(); - TaskNode taskNode = JSONObject.parseObject(taskJson, TaskNode.class); + TaskNode taskNode = JSON.parseObject(taskJson, TaskNode.class); return taskNode.getTaskTimeoutParameter(); } @@ -187,7 +189,6 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { private long getRemaintime(long timeoutSeconds) { Date startTime = taskInstance.getStartTime(); long usedTime = (System.currentTimeMillis() - startTime.getTime()) / 1000; - long remainTime = timeoutSeconds - usedTime; - return remainTime; + return timeoutSeconds - usedTime; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/MonitorServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/MonitorServer.java index ac549bc386..a1f43add6e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/MonitorServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/MonitorServer.java @@ -30,7 +30,7 @@ import org.springframework.context.annotation.ComponentScan; @ComponentScan("org.apache.dolphinscheduler") public class MonitorServer implements CommandLineRunner { - private static Integer ARGS_LENGTH = 4; + private static final Integer ARGS_LENGTH = 4; private static final Logger logger = LoggerFactory.getLogger(MonitorServer.class); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java index f3441edd17..050af1eec5 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java @@ -133,6 +133,8 @@ public class AlertManager { continue; } LinkedHashMap failedTaskMap = new LinkedHashMap(); + failedTaskMap.put("process instance id", String.valueOf(processInstance.getId())); + failedTaskMap.put("process instance name", processInstance.getName()); failedTaskMap.put("task id", String.valueOf(task.getId())); failedTaskMap.put("task name", task.getName()); failedTaskMap.put("task type", task.getTaskType()); @@ -193,7 +195,7 @@ public class AlertManager { logger.info("add alert to db , alert : {}", alert.toString()); }catch (Exception e){ - logger.error("send alert failed! " + e); + logger.error("send alert failed:{} ", e.getMessage()); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java index 4c33ef8db2..12c7eb2d56 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java @@ -17,12 +17,11 @@ package org.apache.dolphinscheduler.server.utils; +import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.flink.FlinkParameters; -import org.apache.commons.lang.StringUtils; -import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; @@ -32,12 +31,7 @@ import java.util.List; * spark args utils */ public class FlinkArgsUtils { - - /** - * logger of FlinkArgsUtils - */ - private static final org.slf4j.Logger logger = LoggerFactory.getLogger(FlinkArgsUtils.class); - + private static final String LOCAL_DEPLOY_MODE = "local"; /** * build args * @param param flink parameters @@ -52,7 +46,7 @@ public class FlinkArgsUtils { deployMode = tmpDeployMode; } - if (!"local".equals(deployMode)) { + if (!LOCAL_DEPLOY_MODE.equals(deployMode)) { args.add(Constants.FLINK_RUN_MODE); //-m args.add(Constants.FLINK_YARN_CLUSTER); //yarn-cluster @@ -113,12 +107,12 @@ public class FlinkArgsUtils { String queue = param.getQueue(); if (StringUtils.isNotEmpty(others)) { - if (!others.contains(Constants.FLINK_QUEUE) && StringUtils.isNotEmpty(queue) && !deployMode.equals("local")) { + if (!others.contains(Constants.FLINK_QUEUE) && StringUtils.isNotEmpty(queue) && !deployMode.equals(LOCAL_DEPLOY_MODE)) { args.add(Constants.FLINK_QUEUE); args.add(param.getQueue()); } args.add(others); - } else if (StringUtils.isNotEmpty(queue) && !deployMode.equals("local")) { + } else if (StringUtils.isNotEmpty(queue) && !deployMode.equals(LOCAL_DEPLOY_MODE)) { args.add(Constants.FLINK_QUEUE); args.add(param.getQueue()); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ParamUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ParamUtils.java index 1d7a80daf0..3040cd198d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ParamUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ParamUtils.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.server.utils; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; import java.util.Date; @@ -70,17 +71,16 @@ public class ParamUtils { Map.Entry en = iter.next(); Property property = en.getValue(); - if (property.getValue() != null && property.getValue().length() > 0){ - if (property.getValue().startsWith("$")){ - /** - * local parameter refers to global parameter with the same name - * note: the global parameters of the process instance here are solidified parameters, - * and there are no variables in them. - */ - String val = property.getValue(); - val = ParameterUtils.convertParameterPlaceholders(val, timeParams); - property.setValue(val); - } + if (StringUtils.isNotEmpty(property.getValue()) + && property.getValue().startsWith("$")){ + /** + * local parameter refers to global parameter with the same name + * note: the global parameters of the process instance here are solidified parameters, + * and there are no variables in them. + */ + String val = property.getValue(); + val = ParameterUtils.convertParameterPlaceholders(val, timeParams); + property.setValue(val); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java index e0c00c55d9..f3f8a79489 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.server.utils; +import java.nio.charset.StandardCharsets; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.LoggerUtils; @@ -29,7 +30,6 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; -import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; @@ -297,7 +297,7 @@ public class ProcessUtils { File f = new File(commandFile); if (!f.exists()) { - FileUtils.writeStringToFile(new File(commandFile), sb.toString(), Charset.forName("UTF-8")); + FileUtils.writeStringToFile(new File(commandFile), sb.toString(), StandardCharsets.UTF_8); } String runCmd = "sh " + commandFile; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index ace93079ff..86bb7d3e07 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -29,6 +29,7 @@ import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.server.master.MasterServer; import org.apache.dolphinscheduler.server.utils.ProcessUtils; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.runner.FetchTaskThread; @@ -43,10 +44,13 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.FilterType; import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -56,7 +60,10 @@ import java.util.concurrent.TimeUnit; /** * worker server */ -@ComponentScan("org.apache.dolphinscheduler") +@SpringBootApplication +@ComponentScan(value = "org.apache.dolphinscheduler", excludeFilters = { + @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, classes = {MasterServer.class}) +}) public class WorkerServer implements IStoppable { /** @@ -104,11 +111,6 @@ public class WorkerServer implements IStoppable { */ private ExecutorService fetchTaskExecutorService; - /** - * CountDownLatch latch - */ - private CountDownLatch latch; - @Value("${server.is-combined-server:false}") private Boolean isCombinedServer; @@ -149,7 +151,7 @@ public class WorkerServer implements IStoppable { this.fetchTaskExecutorService = ThreadUtils.newDaemonSingleThreadExecutor("Worker-Fetch-Thread-Executor"); - heartbeatWorkerService = ThreadUtils.newDaemonThreadScheduledExecutor("Worker-Heartbeat-Thread-Executor", Constants.DEFAUL_WORKER_HEARTBEAT_THREAD_NUM); + heartbeatWorkerService = ThreadUtils.newThreadScheduledExecutor("Worker-Heartbeat-Thread-Executor", Constants.DEFAUL_WORKER_HEARTBEAT_THREAD_NUM, false); // heartbeat thread implement Runnable heartBeatThread = heartBeatThread(); @@ -171,29 +173,15 @@ public class WorkerServer implements IStoppable { // submit fetch task thread fetchTaskExecutorService.execute(fetchTaskThread); + } - /** - * register hooks, which are called before the process exits - */ - Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { - @Override - public void run() { - // worker server exit alert - if (zkWorkerClient.getActiveMasterNum() <= 1) { - alertDao.sendServerStopedAlert(1, OSUtils.getHost(), "Worker-Server"); - } - stop("shutdownhook"); - } - })); - - //let the main thread await - latch = new CountDownLatch(1); - if (!isCombinedServer) { - try { - latch.await(); - } catch (InterruptedException ignore) { - } + @PreDestroy + public void destroy() { + // worker server exit alert + if (zkWorkerClient.getActiveMasterNum() <= 1) { + alertDao.sendServerStopedAlert(1, OSUtils.getHost(), "Worker-Server"); } + stop("shutdownhook"); } @Override @@ -251,7 +239,6 @@ public class WorkerServer implements IStoppable { }catch (Exception e){ logger.warn("zookeeper service stopped exception:{}",e.getMessage()); } - latch.countDown(); logger.info("zookeeper service stopped"); } catch (Exception e) { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java index 5e68acf94e..d36d4de5b4 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.server.worker.runner; import ch.qos.logback.classic.LoggerContext; import ch.qos.logback.classic.sift.SiftingAppender; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; @@ -93,7 +93,7 @@ public class TaskScheduleThread implements Runnable { logger.info("script path : {}", taskInstance.getExecutePath()); // task node - TaskNode taskNode = JSONObject.parseObject(taskInstance.getTaskJson(), TaskNode.class); + TaskNode taskNode = JSON.parseObject(taskInstance.getTaskJson(), TaskNode.class); // get resource files List resourceFiles = createProjectResFiles(taskNode); @@ -176,7 +176,7 @@ public class TaskScheduleThread implements Runnable { String globalParamsStr = taskInstance.getProcessInstance().getGlobalParams(); if (globalParamsStr != null) { - List globalParamsList = JSONObject.parseArray(globalParamsStr, Property.class); + List globalParamsList = JSON.parseArray(globalParamsStr, Property.class); globalParamsMap.putAll(globalParamsList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue))); } return globalParamsMap; @@ -296,9 +296,7 @@ public class TaskScheduleThread implements Runnable { if (baseParam != null) { List projectResourceFiles = baseParam.getResourceFilesList(); - if (projectResourceFiles != null) { - projectFiles.addAll(projectResourceFiles); - } + projectFiles.addAll(projectResourceFiles); } return new ArrayList<>(projectFiles); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index bac498c150..8e0ccee16c 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -16,21 +16,29 @@ */ package org.apache.dolphinscheduler.server.worker.task; +import com.sun.jna.platform.win32.Kernel32; +import com.sun.jna.platform.win32.WinNT; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils; +import org.apache.dolphinscheduler.common.utils.LoggerUtils; +import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.utils.process.ProcessBuilderForWin32; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.server.utils.ProcessUtils; import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import java.io.*; import java.lang.reflect.Field; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -194,26 +202,49 @@ public abstract class AbstractCommandExecutor { * @throws IOException IO Exception */ private void buildProcess(String commandFile) throws IOException { + // command list + List command = new ArrayList<>(); + //init process builder - ProcessBuilder processBuilder = new ProcessBuilder(); - // setting up a working directory - processBuilder.directory(new File(taskDir)); - // merge error information to standard output stream - processBuilder.redirectErrorStream(true); - // setting up user to run commands - List command = new LinkedList<>(); - command.add("sudo"); - command.add("-u"); - command.add(tenantCode); - command.add(commandInterpreter()); - command.addAll(commandOptions()); - command.add(commandFile); - processBuilder.command(command); - - process = processBuilder.start(); + if (OSUtils.isWindows()) { + ProcessBuilderForWin32 processBuilder = new ProcessBuilderForWin32(); + // setting up a working directory + processBuilder.directory(new File(taskDir)); + processBuilder.user(tenantCode, StringUtils.EMPTY); + // merge error information to standard output stream + processBuilder.redirectErrorStream(true); + + // setting up user to run commands + command.add(commandInterpreter()); + command.add("/c"); + command.addAll(commandOptions()); + command.add(commandFile); + + // setting commands + processBuilder.command(command); + process = processBuilder.start(); + } else { + ProcessBuilder processBuilder = new ProcessBuilder(); + // setting up a working directory + processBuilder.directory(new File(taskDir)); + // merge error information to standard output stream + processBuilder.redirectErrorStream(true); + + // setting up user to run commands + command.add("sudo"); + command.add("-u"); + command.add(tenantCode); + command.add(commandInterpreter()); + command.addAll(commandOptions()); + command.add(commandFile); + + // setting commands + processBuilder.command(command); + process = processBuilder.start(); + } // print command - printCommand(processBuilder); + printCommand(command); } /** @@ -320,13 +351,13 @@ public abstract class AbstractCommandExecutor { /** * print command - * @param processBuilder process builder + * @param command command */ - private void printCommand(ProcessBuilder processBuilder) { + private void printCommand(List command) { String cmdStr; try { - cmdStr = ProcessUtils.buildCommandStr(processBuilder.command()); + cmdStr = ProcessUtils.buildCommandStr(command); logger.info("task run command:\n{}", cmdStr); } catch (IOException e) { logger.error(e.getMessage(), e); @@ -358,7 +389,11 @@ public abstract class AbstractCommandExecutor { BufferedReader inReader = null; try { - inReader = new BufferedReader(new InputStreamReader(process.getInputStream())); + if (OSUtils.isWindows()) { + inReader = new BufferedReader(new InputStreamReader(process.getInputStream(), Charset.forName("GBK"))); + } else { + inReader = new BufferedReader(new InputStreamReader(process.getInputStream())); + } String line; long lastFlushTime = System.currentTimeMillis(); @@ -406,7 +441,7 @@ public abstract class AbstractCommandExecutor { } Thread.sleep(Constants.SLEEP_TIME_MILLIS); } - } + } } catch (Exception e) { logger.error("yarn applications: {} status failed ", appIds,e); result = false; @@ -510,12 +545,15 @@ public abstract class AbstractCommandExecutor { */ private int getProcessId(Process process) { int processId = 0; - try { Field f = process.getClass().getDeclaredField(Constants.PID); f.setAccessible(true); - - processId = f.getInt(process); + if (OSUtils.isWindows()) { + WinNT.HANDLE handle = (WinNT.HANDLE) f.get(process); + processId = Kernel32.INSTANCE.GetProcessId(handle); + } else { + processId = f.getInt(process); + } } catch (Throwable e) { logger.error(e.getMessage(), e); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java index db46d0d856..5d14e6b2a7 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java @@ -17,11 +17,12 @@ package org.apache.dolphinscheduler.server.worker.task; import org.apache.commons.io.FileUtils; +import org.apache.dolphinscheduler.common.utils.OSUtils; import org.slf4j.Logger; import java.io.File; import java.io.IOException; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Date; @@ -34,10 +35,15 @@ import java.util.function.Consumer; public class ShellCommandExecutor extends AbstractCommandExecutor { /** - * sh + * For Unix-like, using sh */ public static final String SH = "sh"; + /** + * For Windows, using cmd.exe + */ + public static final String CMD = "cmd.exe"; + /** * constructor * @param logHandler log handler @@ -66,7 +72,7 @@ public class ShellCommandExecutor extends AbstractCommandExecutor { @Override protected String buildCommandFilePath() { // command file - return String.format("%s/%s.command", taskDir, taskAppId); + return String.format("%s/%s.%s", taskDir, taskAppId, OSUtils.isWindows() ? "bat" : "command"); } /** @@ -75,7 +81,7 @@ public class ShellCommandExecutor extends AbstractCommandExecutor { */ @Override protected String commandInterpreter() { - return SH; + return OSUtils.isWindows() ? CMD : SH; } /** @@ -103,21 +109,26 @@ public class ShellCommandExecutor extends AbstractCommandExecutor { logger.info("create command file:{}", commandFile); StringBuilder sb = new StringBuilder(); - sb.append("#!/bin/sh\n"); - sb.append("BASEDIR=$(cd `dirname $0`; pwd)\n"); - sb.append("cd $BASEDIR\n"); - - if (envFile != null) { - sb.append("source " + envFile + "\n"); + if (OSUtils.isWindows()) { + sb.append("@echo off\n"); + sb.append("cd /d %~dp0\n"); + if (envFile != null) { + sb.append("call ").append(envFile).append("\n"); + } + } else { + sb.append("#!/bin/sh\n"); + sb.append("BASEDIR=$(cd `dirname $0`; pwd)\n"); + sb.append("cd $BASEDIR\n"); + if (envFile != null) { + sb.append("source ").append(envFile).append("\n"); + } } - sb.append("\n\n"); sb.append(execCommand); - logger.info("command : {}",sb.toString()); + logger.info("command : {}", sb.toString()); // write data to file - FileUtils.writeStringToFile(new File(commandFile), sb.toString(), - Charset.forName("UTF-8")); + FileUtils.writeStringToFile(new File(commandFile), sb.toString(), StandardCharsets.UTF_8); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskProps.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskProps.java index 8e5644ed9c..edec419384 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskProps.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskProps.java @@ -78,7 +78,7 @@ public class TaskProps { private Map definedParams; /** - * task path + * task app id */ private String taskAppId; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java index ef941cd062..952030ea62 100755 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.server.worker.task.datax; import java.io.File; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; @@ -44,6 +44,7 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.datax.DataxParameters; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; @@ -208,7 +209,7 @@ public class DataxTask extends AbstractTask { logger.debug("datax job json : {}", root.toString()); // create datax json file - FileUtils.writeStringToFile(new File(fileName), root.toString(), Charset.forName("UTF-8")); + FileUtils.writeStringToFile(new File(fileName), root.toString(), StandardCharsets.UTF_8); return fileName; } @@ -338,7 +339,7 @@ public class DataxTask extends AbstractTask { private String buildShellCommandFile(String jobConfigFilePath) throws Exception { // generate scripts - String fileName = String.format("%s/%s_node.sh", taskDir, taskProps.getTaskAppId()); + String fileName = String.format("%s/%s_node.%s", taskDir, taskProps.getTaskAppId(), OSUtils.isWindows() ? "bat" : "sh"); Path path = new File(fileName).toPath(); if (Files.exists(path)) { @@ -370,7 +371,13 @@ public class DataxTask extends AbstractTask { // create shell command file Set perms = PosixFilePermissions.fromString(Constants.RWXR_XR_X); FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); - Files.createFile(path, attr); + + if (OSUtils.isWindows()) { + Files.createFile(path); + } else { + Files.createFile(path, attr); + } + Files.write(path, dataxCommand.getBytes(), StandardOpenOption.APPEND); return fileName; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java index c925f90b9e..85c8d2723c 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.worker.task.http; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import org.apache.commons.io.Charsets; import org.apache.dolphinscheduler.common.Constants; @@ -26,6 +27,7 @@ import org.apache.dolphinscheduler.common.process.HttpProperty; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.http.HttpParameters; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; @@ -51,6 +53,7 @@ import org.slf4j.Logger; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -98,7 +101,7 @@ public class HttpTask extends AbstractTask { @Override public void init() { logger.info("http task params {}", taskProps.getTaskParams()); - this.httpParameters = JSONObject.parseObject(taskProps.getTaskParams(), HttpParameters.class); + this.httpParameters = JSON.parseObject(taskProps.getTaskParams(), HttpParameters.class); if (!httpParameters.checkParameters()) { throw new RuntimeException("http task params is not valid"); @@ -146,12 +149,12 @@ public class HttpTask extends AbstractTask { processInstance.getCmdTypeIfComplement(), processInstance.getScheduleTime()); List httpPropertyList = new ArrayList<>(); - if(httpParameters.getHttpParams() != null && httpParameters.getHttpParams().size() > 0){ + if(CollectionUtils.isNotEmpty(httpParameters.getHttpParams() )){ for (HttpProperty httpProperty: httpParameters.getHttpParams()) { - String jsonObject = JSONObject.toJSONString(httpProperty); + String jsonObject = JSON.toJSONString(httpProperty); String params = ParameterUtils.convertParameterPlaceholders(jsonObject,ParamUtils.convert(paramsMap)); logger.info("http request params：{}",params); - httpPropertyList.add(JSONObject.parseObject(params,HttpProperty.class)); + httpPropertyList.add(JSON.parseObject(params,HttpProperty.class)); } } addRequestParams(builder,httpPropertyList); @@ -176,8 +179,7 @@ public class HttpTask extends AbstractTask { if (entity == null) { return null; } - String webPage = EntityUtils.toString(entity, StandardCharsets.UTF_8.name()); - return webPage; + return EntityUtils.toString(entity, StandardCharsets.UTF_8.name()); } /** @@ -186,8 +188,7 @@ public class HttpTask extends AbstractTask { * @return status code */ protected int getStatusCode(CloseableHttpResponse httpResponse) { - int status = httpResponse.getStatusLine().getStatusCode(); - return status; + return httpResponse.getStatusLine().getStatusCode(); } /** @@ -252,7 +253,7 @@ public class HttpTask extends AbstractTask { * @param httpPropertyList http property list */ protected void addRequestParams(RequestBuilder builder,List httpPropertyList) { - if(httpPropertyList != null && httpPropertyList.size() > 0){ + if(CollectionUtils.isNotEmpty(httpPropertyList)){ JSONObject jsonParam = new JSONObject(); for (HttpProperty property: httpPropertyList){ if(property.getHttpParametersType() != null){ @@ -276,12 +277,10 @@ public class HttpTask extends AbstractTask { * @param httpPropertyList http property list */ protected void setHeaders(HttpUriRequest request,List httpPropertyList) { - if(httpPropertyList != null && httpPropertyList.size() > 0){ - for (HttpProperty property: httpPropertyList){ - if(property.getHttpParametersType() != null) { - if (property.getHttpParametersType().equals(HttpParametersType.HEADERS)) { - request.addHeader(property.getProp(), property.getValue()); - } + if(CollectionUtils.isNotEmpty(httpPropertyList)){ + for (HttpProperty property: httpPropertyList) { + if (HttpParametersType.HEADERS.equals(property.getHttpParametersType())) { + request.addHeader(property.getProp(), property.getValue()); } } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java index b86ff9952e..7f6baad427 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/mr/MapReduceTask.java @@ -122,22 +122,19 @@ public class MapReduceTask extends AbstractYarnTask { } // main class - if(mapreduceParameters.getProgramType() !=null ){ - if(mapreduceParameters.getProgramType()!= ProgramType.PYTHON){ - if(StringUtils.isNotEmpty(mapreduceParameters.getMainClass())){ - result.add(mapreduceParameters.getMainClass()); - } - } + if(!ProgramType.PYTHON.equals(mapreduceParameters.getProgramType()) + && StringUtils.isNotEmpty(mapreduceParameters.getMainClass())){ + result.add(mapreduceParameters.getMainClass()); } // others if (StringUtils.isNotEmpty(mapreduceParameters.getOthers())) { String others = mapreduceParameters.getOthers(); - if(!others.contains(Constants.MR_QUEUE)){ - if (StringUtils.isNotEmpty(mapreduceParameters.getQueue())) { - result.add(String.format("%s %s=%s", Constants.D, Constants.MR_QUEUE, mapreduceParameters.getQueue())); - } + if (!others.contains(Constants.MR_QUEUE) + && StringUtils.isNotEmpty(mapreduceParameters.getQueue())) { + result.add(String.format("%s %s=%s", Constants.D, Constants.MR_QUEUE, mapreduceParameters.getQueue())); } + result.add(mapreduceParameters.getOthers()); }else if (StringUtils.isNotEmpty(mapreduceParameters.getQueue())) { result.add(String.format("%s %s=%s", Constants.D, Constants.MR_QUEUE, mapreduceParameters.getQueue())); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java index fb881453e9..5a8afdb7bb 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.server.worker.task.processdure; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import com.cronutils.utils.StringUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DataType; @@ -60,11 +60,6 @@ public class ProcedureTask extends AbstractTask { */ private ProcessService processService; - /** - * base datasource - */ - private BaseDataSource baseDataSource; - /** * constructor * @param taskProps task props @@ -75,7 +70,7 @@ public class ProcedureTask extends AbstractTask { logger.info("procedure task params {}", taskProps.getTaskParams()); - this.procedureParameters = JSONObject.parseObject(taskProps.getTaskParams(), ProcedureParameters.class); + this.procedureParameters = JSON.parseObject(taskProps.getTaskParams(), ProcedureParameters.class); // check parameters if (!procedureParameters.checkParameters()) { @@ -117,7 +112,7 @@ public class ProcedureTask extends AbstractTask { // load class DataSourceFactory.loadClass(dataSource.getType()); // get datasource - baseDataSource = DataSourceFactory.getDatasource(dataSource.getType(), + BaseDataSource baseDataSource = DataSourceFactory.getDatasource(dataSource.getType(), dataSource.getConnectionParams()); // get jdbc connection @@ -163,7 +158,7 @@ public class ProcedureTask extends AbstractTask { stmt.setQueryTimeout(taskProps.getTaskTimeout()); } Map outParameterMap = new HashMap<>(); - if (userDefParamsList != null && userDefParamsList.size() > 0){ + if (CollectionUtils.isNotEmpty(userDefParamsList)){ int index = 1; for (Property property : userDefParamsList){ logger.info("localParams : prop : {} , dirct : {} , type : {} , value : {}" @@ -237,31 +232,31 @@ public class ProcedureTask extends AbstractTask { private void getOutputParameter(CallableStatement stmt, int index, String prop, DataType dataType) throws SQLException { switch (dataType){ case VARCHAR: - logger.info("out prameter key : {} , value : {}",prop,stmt.getString(index)); + logger.info("out prameter varchar key : {} , value : {}",prop,stmt.getString(index)); break; case INTEGER: - logger.info("out prameter key : {} , value : {}", prop, stmt.getInt(index)); + logger.info("out prameter integer key : {} , value : {}", prop, stmt.getInt(index)); break; case LONG: - logger.info("out prameter key : {} , value : {}",prop,stmt.getLong(index)); + logger.info("out prameter long key : {} , value : {}",prop,stmt.getLong(index)); break; case FLOAT: - logger.info("out prameter key : {} , value : {}",prop,stmt.getFloat(index)); + logger.info("out prameter float key : {} , value : {}",prop,stmt.getFloat(index)); break; case DOUBLE: - logger.info("out prameter key : {} , value : {}",prop,stmt.getDouble(index)); + logger.info("out prameter double key : {} , value : {}",prop,stmt.getDouble(index)); break; case DATE: - logger.info("out prameter key : {} , value : {}",prop,stmt.getDate(index)); + logger.info("out prameter date key : {} , value : {}",prop,stmt.getDate(index)); break; case TIME: - logger.info("out prameter key : {} , value : {}",prop,stmt.getTime(index)); + logger.info("out prameter time key : {} , value : {}",prop,stmt.getTime(index)); break; case TIMESTAMP: - logger.info("out prameter key : {} , value : {}",prop,stmt.getTimestamp(index)); + logger.info("out prameter timestamp key : {} , value : {}",prop,stmt.getTimestamp(index)); break; case BOOLEAN: - logger.info("out prameter key : {} , value : {}",prop, stmt.getBoolean(index)); + logger.info("out prameter boolean key : {} , value : {}",prop, stmt.getBoolean(index)); break; default: break; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java index 5704c8052e..90661a690a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.shell.ShellParameters; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; @@ -123,7 +124,7 @@ public class ShellTask extends AbstractTask { */ private String buildCommand() throws Exception { // generate scripts - String fileName = String.format("%s/%s_node.sh", taskDir, taskProps.getTaskAppId()); + String fileName = String.format("%s/%s_node.%s", taskDir, taskProps.getTaskAppId(), OSUtils.isWindows() ? "bat" : "sh"); Path path = new File(fileName).toPath(); if (Files.exists(path)) { @@ -154,7 +155,11 @@ public class ShellTask extends AbstractTask { Set perms = PosixFilePermissions.fromString(Constants.RWXR_XR_X); FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); - Files.createFile(path, attr); + if (OSUtils.isWindows()) { + Files.createFile(path); + } else { + Files.createFile(path, attr); + } Files.write(path, shellParameters.getRawScript().getBytes(), StandardOpenOption.APPEND); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index aae11f5530..82034740fc 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.server.worker.task.sql; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.serializer.SerializerFeature; @@ -92,7 +93,7 @@ public class SqlTask extends AbstractTask { super(taskProps, logger); logger.info("sql task params {}", taskProps.getTaskParams()); - this.sqlParameters = JSONObject.parseObject(taskProps.getTaskParams(), SqlParameters.class); + this.sqlParameters = JSON.parseObject(taskProps.getTaskParams(), SqlParameters.class); if (!sqlParameters.checkParameters()) { throw new RuntimeException("sql task params is not valid"); @@ -308,16 +309,16 @@ public class SqlTask extends AbstractTask { } resultJSONArray.add(mapOfColValues); } - logger.debug("execute sql : {}", JSONObject.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); + logger.debug("execute sql : {}", JSON.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); // if there is a result set if ( !resultJSONArray.isEmpty() ) { if (StringUtils.isNotEmpty(sqlParameters.getTitle())) { sendAttachment(sqlParameters.getTitle(), - JSONObject.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); + JSON.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); }else{ sendAttachment(taskProps.getNodeName() + " query resultsets ", - JSONObject.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); + JSON.toJSONString(resultJSONArray, SerializerFeature.WriteMapNullValue)); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java index 050fef7cc7..404f10bf39 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/sources/MysqlSourceGenerator.java @@ -69,17 +69,16 @@ public class MysqlSourceGenerator implements ISourceGenerator { result.append(" --columns ").append(sourceMysqlParameter.getSrcColumns()); } - }else if(sourceMysqlParameter.getSrcQueryType() == QueryType.SQL.ordinal()){ - if(StringUtils.isNotEmpty(sourceMysqlParameter.getSrcQuerySql())){ - - String srcQuery = sourceMysqlParameter.getSrcQuerySql(); - if(srcQuery.toLowerCase().contains("where")){ - srcQuery += " AND "+"$CONDITIONS"; - }else{ - srcQuery += " WHERE $CONDITIONS"; - } - result.append(" --query \'"+srcQuery+"\'"); + }else if(sourceMysqlParameter.getSrcQueryType() == QueryType.SQL.ordinal() + && StringUtils.isNotEmpty(sourceMysqlParameter.getSrcQuerySql())){ + String srcQuery = sourceMysqlParameter.getSrcQuerySql(); + if(srcQuery.toLowerCase().contains("where")){ + srcQuery += " AND "+"$CONDITIONS"; + }else{ + srcQuery += " WHERE $CONDITIONS"; } + result.append(" --query \'"+srcQuery+"\'"); + } List mapColumnHive = sourceMysqlParameter.getMapColumnHive(); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java index 0733338812..0e33b176e5 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/MysqlTargetGenerator.java @@ -75,12 +75,11 @@ public class MysqlTargetGenerator implements ITargetGenerator { result.append(" --lines-terminated-by '").append(targetMysqlParameter.getLinesTerminated()).append("'"); } - if(targetMysqlParameter.isUpdate()){ - if(StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateKey())&& - StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateMode())){ - result.append(" --update-key ").append(targetMysqlParameter.getTargetUpdateKey()) - .append(" --update-mode ").append(targetMysqlParameter.getTargetUpdateMode()); - } + if(targetMysqlParameter.isUpdate() + && StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateKey()) + && StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateMode())){ + result.append(" --update-key ").append(targetMysqlParameter.getTargetUpdateKey()) + .append(" --update-mode ").append(targetMysqlParameter.getTargetUpdateMode()); } } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java index d2a0fb2407..af312d9601 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.server.master; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.enums.*; import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.utils.DateUtils; @@ -85,7 +85,7 @@ public class MasterExecThreadTest { Map cmdParam = new HashMap<>(); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, "2020-01-01 00:00:00"); cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, "2020-01-31 23:00:00"); - Mockito.when(processInstance.getCommandParam()).thenReturn(JSONObject.toJSONString(cmdParam)); + Mockito.when(processInstance.getCommandParam()).thenReturn(JSON.toJSONString(cmdParam)); ProcessDefinition processDefinition = new ProcessDefinition(); processDefinition.setGlobalParamMap(Collections.EMPTY_MAP); processDefinition.setGlobalParamList(Collections.EMPTY_LIST); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java index 710d2c2505..2e4861e2a2 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java @@ -87,35 +87,35 @@ public class FlinkArgsUtilsTest { } //Expected values and order - assertEquals(result.size(),20); + assertEquals(20, result.size()); - assertEquals(result.get(0),"-m"); - assertEquals(result.get(1),"yarn-cluster"); + assertEquals("-m", result.get(0)); + assertEquals("yarn-cluster", result.get(1)); - assertEquals(result.get(2),"-ys"); + assertEquals("-ys", result.get(2)); assertSame(Integer.valueOf(result.get(3)),slot); - assertEquals(result.get(4),"-ynm"); + assertEquals("-ynm",result.get(4)); assertEquals(result.get(5),appName); - assertEquals(result.get(6),"-yn"); + assertEquals("-yn", result.get(6)); assertSame(Integer.valueOf(result.get(7)),taskManager); - assertEquals(result.get(8),"-yjm"); + assertEquals("-yjm", result.get(8)); assertEquals(result.get(9),jobManagerMemory); - assertEquals(result.get(10),"-ytm"); + assertEquals("-ytm", result.get(10)); assertEquals(result.get(11),taskManagerMemory); - assertEquals(result.get(12),"-d"); + assertEquals("-d", result.get(12)); - assertEquals(result.get(13),"-c"); + assertEquals("-c", result.get(13)); assertEquals(result.get(14),mainClass); assertEquals(result.get(15),mainJar.getRes()); assertEquals(result.get(16),mainArgs); - assertEquals(result.get(17),"--qu"); + assertEquals("--qu", result.get(17)); assertEquals(result.get(18),queue); assertEquals(result.get(19),others); @@ -125,7 +125,7 @@ public class FlinkArgsUtilsTest { param1.setQueue(queue); param1.setDeployMode(mode); result = FlinkArgsUtils.buildArgs(param1); - assertEquals(result.size(),5); + assertEquals(5, result.size()); } } \ No newline at end of file diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java index 77fc398702..1e0adaad9b 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java @@ -21,7 +21,12 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + public class ProcessUtilsTest { + private static final Logger logger = LoggerFactory.getLogger(ProcessUtilsTest.class); @Test @@ -30,4 +35,16 @@ public class ProcessUtilsTest { Assert.assertNotEquals("The child process of process 1 should not be empty", pidList, ""); logger.info("Sub process list : {}", pidList); } + + @Test + public void testBuildCommandStr() { + List commands = new ArrayList<>(); + commands.add("sudo"); + try { + Assert.assertEquals(ProcessUtils.buildCommandStr(commands), "sudo"); + } catch (IOException e) { + Assert.fail(e.getMessage()); + } + } + } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index 5d4263644b..250c8a2680 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.server.worker.shell; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.TaskNode; @@ -68,7 +68,7 @@ public class ShellCommandExecutorTest { TaskInstance taskInstance = processService.findTaskInstanceById(7657); String taskJson = taskInstance.getTaskJson(); - TaskNode taskNode = JSONObject.parseObject(taskJson, TaskNode.class); + TaskNode taskNode = JSON.parseObject(taskJson, TaskNode.class); taskProps.setTaskParams(taskNode.getParams()); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java index c395eabe51..07b700239b 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/sql/SqlExecutorTest.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.server.worker.sql; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; @@ -112,7 +112,7 @@ public class SqlExecutorTest { TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId); String taskJson = taskInstance.getTaskJson(); - TaskNode taskNode = JSONObject.parseObject(taskJson, TaskNode.class); + TaskNode taskNode = JSON.parseObject(taskJson, TaskNode.class); taskProps.setTaskParams(taskNode.getParams()); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java new file mode 100644 index 0000000000..ebe90147d1 --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.server.worker.task.shell; + +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.dao.entity.DataSource; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor; +import org.apache.dolphinscheduler.server.worker.task.TaskProps; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; +import org.junit.After; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationContext; + +import java.util.Date; + +/** + * shell task test + */ +@RunWith(PowerMockRunner.class) +@PrepareForTest(OSUtils.class) +public class ShellTaskTest { + + private static final Logger logger = LoggerFactory.getLogger(ShellTaskTest.class); + + private ShellTask shellTask; + + private ProcessService processService; + + private ShellCommandExecutor shellCommandExecutor; + + private ApplicationContext applicationContext; + + @Before + public void before() throws Exception { + PowerMockito.mockStatic(OSUtils.class); + processService = PowerMockito.mock(ProcessService.class); + shellCommandExecutor = PowerMockito.mock(ShellCommandExecutor.class); + + applicationContext = PowerMockito.mock(ApplicationContext.class); + SpringApplicationContext springApplicationContext = new SpringApplicationContext(); + springApplicationContext.setApplicationContext(applicationContext); + PowerMockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService); + + TaskProps props = new TaskProps(); + props.setTaskDir("/tmp"); + props.setTaskAppId(String.valueOf(System.currentTimeMillis())); + props.setTaskInstId(1); + props.setTenantCode("1"); + props.setEnvFile(".dolphinscheduler_env.sh"); + props.setTaskStartTime(new Date()); + props.setTaskTimeout(0); + props.setTaskParams("{\"rawScript\": \" echo 'hello world!'\"}"); + shellTask = new ShellTask(props, logger); + shellTask.init(); + + PowerMockito.when(processService.findDataSourceById(1)).thenReturn(getDataSource()); + PowerMockito.when(processService.findDataSourceById(2)).thenReturn(getDataSource()); + PowerMockito.when(processService.findProcessInstanceByTaskId(1)).thenReturn(getProcessInstance()); + + String fileName = String.format("%s/%s_node.%s", props.getTaskDir(), props.getTaskAppId(), OSUtils.isWindows() ? "bat" : "sh"); + PowerMockito.when(shellCommandExecutor.run(fileName, processService)).thenReturn(0); + } + + private DataSource getDataSource() { + DataSource dataSource = new DataSource(); + dataSource.setType(DbType.MYSQL); + dataSource.setConnectionParams( + "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:mysql://127.0.0.1:3306\",\"database\":\"test\",\"jdbcUrl\":\"jdbc:mysql://127.0.0.1:3306/test\"}"); + dataSource.setUserId(1); + return dataSource; + } + + private ProcessInstance getProcessInstance() { + ProcessInstance processInstance = new ProcessInstance(); + processInstance.setCommandType(CommandType.START_PROCESS); + processInstance.setScheduleTime(new Date()); + return processInstance; + } + + @After + public void after() {} + + /** + * Method: ShellTask() + */ + @Test + public void testShellTask() + throws Exception { + TaskProps props = new TaskProps(); + props.setTaskDir("/tmp"); + props.setTaskAppId(String.valueOf(System.currentTimeMillis())); + props.setTaskInstId(1); + props.setTenantCode("1"); + ShellTask shellTaskTest = new ShellTask(props, logger); + Assert.assertNotNull(shellTaskTest); + } + + /** + * Method: init for Unix-like + */ + @Test + public void testInitForUnix() { + try { + PowerMockito.when(OSUtils.isWindows()).thenReturn(false); + shellTask.init(); + Assert.assertTrue(true); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + /** + * Method: init for Windows + */ + @Test + public void testInitForWindows() { + try { + PowerMockito.when(OSUtils.isWindows()).thenReturn(true); + shellTask.init(); + Assert.assertTrue(true); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + + /** + * Method: handle() for Unix-like + */ + @Test + public void testHandleForUnix() throws Exception { + try { + PowerMockito.when(OSUtils.isWindows()).thenReturn(false); + shellTask.handle(); + Assert.assertTrue(true); + } catch (Error | Exception e) { + if (!e.getMessage().contains("process error . exitCode is : -1") + && !System.getProperty("os.name").startsWith("Windows")) { + logger.error(e.getMessage()); + } + } + } + + /** + * Method: handle() for Windows + */ + @Test + public void testHandleForWindows() throws Exception { + try { + Assume.assumeTrue(OSUtils.isWindows()); + shellTask.handle(); + Assert.assertTrue(true); + } catch (Error | Exception e) { + if (!e.getMessage().contains("process error . exitCode is : -1")) { + logger.error(e.getMessage()); + } + } + } + + /** + * Method: cancelApplication() + */ + @Test + public void testCancelApplication() throws Exception { + try { + shellTask.cancelApplication(true); + Assert.assertTrue(true); + } catch (Error | Exception e) { + logger.error(e.getMessage()); + } + } + +} diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java index f8688e7c0c..5c7afa8155 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java @@ -16,15 +16,9 @@ */ package org.apache.dolphinscheduler.server.worker.task.sqoop; -import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; -import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceHdfsParameter; -import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceHiveParameter; -import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceMysqlParameter; -import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetHdfsParameter; -import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetHiveParameter; -import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetMysqlParameter; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.server.worker.task.TaskProps; import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.SqoopJobGenerator; @@ -80,7 +74,7 @@ public class SqoopTaskTest { @Test public void testGenerator(){ String data1 = "{\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HDFS\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"0\\\",\\\"srcQuerySql\\\":\\\"\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[]}\",\"targetParams\":\"{\\\"targetPath\\\":\\\"/ods/tmp/test/person7\\\",\\\"deleteTargetDir\\\":true,\\\"fileType\\\":\\\"--as-textfile\\\",\\\"compressionCodec\\\":\\\"\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; - SqoopParameters sqoopParameters1 = JSONObject.parseObject(data1,SqoopParameters.class); + SqoopParameters sqoopParameters1 = JSON.parseObject(data1,SqoopParameters.class); SqoopJobGenerator generator = new SqoopJobGenerator(); String script = generator.generateSqoopJob(sqoopParameters1); @@ -88,21 +82,21 @@ public class SqoopTaskTest { Assert.assertEquals(expected, script); String data2 = "{\"concurrency\":1,\"modelType\":\"export\",\"sourceType\":\"HDFS\",\"targetType\":\"MYSQL\",\"sourceParams\":\"{\\\"exportDir\\\":\\\"/ods/tmp/test/person7\\\"}\",\"targetParams\":\"{\\\"targetDatasource\\\":2,\\\"targetTable\\\":\\\"person_3\\\",\\\"targetColumns\\\":\\\"id,name,age,sex,create_time\\\",\\\"preQuery\\\":\\\"\\\",\\\"isUpdate\\\":true,\\\"targetUpdateKey\\\":\\\"id\\\",\\\"targetUpdateMode\\\":\\\"allowinsert\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; - SqoopParameters sqoopParameters2 = JSONObject.parseObject(data2,SqoopParameters.class); + SqoopParameters sqoopParameters2 = JSON.parseObject(data2,SqoopParameters.class); String script2 = generator.generateSqoopJob(sqoopParameters2); String expected2 = "sqoop export -m 1 --export-dir /ods/tmp/test/person7 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_3 --columns id,name,age,sex,create_time --fields-terminated-by '@' --lines-terminated-by '\\n' --update-key id --update-mode allowinsert"; Assert.assertEquals(expected2, script2); String data3 = "{\"concurrency\":1,\"modelType\":\"export\",\"sourceType\":\"HIVE\",\"targetType\":\"MYSQL\",\"sourceParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-17\\\"}\",\"targetParams\":\"{\\\"targetDatasource\\\":2,\\\"targetTable\\\":\\\"person_3\\\",\\\"targetColumns\\\":\\\"\\\",\\\"preQuery\\\":\\\"\\\",\\\"isUpdate\\\":false,\\\"targetUpdateKey\\\":\\\"\\\",\\\"targetUpdateMode\\\":\\\"allowinsert\\\",\\\"fieldsTerminated\\\":\\\"@\\\",\\\"linesTerminated\\\":\\\"\\\\\\\\n\\\"}\",\"localParams\":[]}"; - SqoopParameters sqoopParameters3 = JSONObject.parseObject(data3,SqoopParameters.class); + SqoopParameters sqoopParameters3 = JSON.parseObject(data3,SqoopParameters.class); String script3 = generator.generateSqoopJob(sqoopParameters3); String expected3 = "sqoop export -m 1 --hcatalog-database stg --hcatalog-table person_internal --hcatalog-partition-keys date --hcatalog-partition-values 2020-02-17 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --table person_3 --fields-terminated-by '@' --lines-terminated-by '\\n'"; Assert.assertEquals(expected3, script3); String data4 = "{\"concurrency\":1,\"modelType\":\"import\",\"sourceType\":\"MYSQL\",\"targetType\":\"HIVE\",\"sourceParams\":\"{\\\"srcDatasource\\\":2,\\\"srcTable\\\":\\\"person_2\\\",\\\"srcQueryType\\\":\\\"1\\\",\\\"srcQuerySql\\\":\\\"SELECT * FROM person_2\\\",\\\"srcColumnType\\\":\\\"0\\\",\\\"srcColumns\\\":\\\"\\\",\\\"srcConditionList\\\":[],\\\"mapColumnHive\\\":[],\\\"mapColumnJava\\\":[{\\\"prop\\\":\\\"id\\\",\\\"direct\\\":\\\"IN\\\",\\\"type\\\":\\\"VARCHAR\\\",\\\"value\\\":\\\"Integer\\\"}]}\",\"targetParams\":\"{\\\"hiveDatabase\\\":\\\"stg\\\",\\\"hiveTable\\\":\\\"person_internal_2\\\",\\\"createHiveTable\\\":true,\\\"dropDelimiter\\\":false,\\\"hiveOverWrite\\\":true,\\\"replaceDelimiter\\\":\\\"\\\",\\\"hivePartitionKey\\\":\\\"date\\\",\\\"hivePartitionValue\\\":\\\"2020-02-16\\\"}\",\"localParams\":[]}"; - SqoopParameters sqoopParameters4 = JSONObject.parseObject(data4,SqoopParameters.class); + SqoopParameters sqoopParameters4 = JSON.parseObject(data4,SqoopParameters.class); String script4 = generator.generateSqoopJob(sqoopParameters4); String expected4 = "sqoop import -m 1 --connect jdbc:mysql://192.168.0.111:3306/test --username kylo --password 123456 --query 'SELECT * FROM person_2 WHERE $CONDITIONS' --map-column-java id=Integer --hive-import --hive-table stg.person_internal_2 --create-hive-table --hive-overwrite -delete-target-dir --hive-partition-key date --hive-partition-value 2020-02-16"; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/zk/StandaloneZKServerForTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/zk/StandaloneZKServerForTest.java deleted file mode 100644 index 679862f102..0000000000 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/zk/StandaloneZKServerForTest.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.server.zk; - -import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors; -import org.apache.zookeeper.server.ServerConfig; -import org.apache.zookeeper.server.ZooKeeperServerMain; -import org.apache.zookeeper.server.quorum.QuorumPeerConfig; -import org.junit.Before; -import org.junit.Ignore; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.util.Properties; - - -/** - * just for test - */ -@Ignore -public class StandaloneZKServerForTest { - - private static final Logger logger = LoggerFactory.getLogger(StandaloneZKServerForTest.class); - - private static volatile ZooKeeperServerMain zkServer = null; - - - @Before - public void before() { - logger.info("standalone zookeeper server for test service start "); - - ThreadPoolExecutors.getInstance().execute(new Runnable() { - @Override - public void run() { - - //delete zk data dir ? - File zkFile = new File(System.getProperty("java.io.tmpdir"), "zookeeper"); - - startStandaloneServer("2000", zkFile.getAbsolutePath(), "2181", "10", "5"); - } - }); - - } - - - /** - * start zk server - * @param tickTime zookeeper ticktime - * @param dataDir zookeeper data dir - * @param clientPort zookeeper client port - * @param initLimit zookeeper init limit - * @param syncLimit zookeeper sync limit - */ - private void startStandaloneServer(String tickTime, String dataDir, String clientPort, String initLimit, String syncLimit) { - Properties props = new Properties(); - props.setProperty("tickTime", tickTime); - props.setProperty("dataDir", dataDir); - props.setProperty("clientPort", clientPort); - props.setProperty("initLimit", initLimit); - props.setProperty("syncLimit", syncLimit); - - QuorumPeerConfig quorumConfig = new QuorumPeerConfig(); - try { - quorumConfig.parseProperties(props); - - if(zkServer == null ){ - - synchronized (StandaloneZKServerForTest.class){ - if(zkServer == null ){ - zkServer = new ZooKeeperServerMain(); - final ServerConfig config = new ServerConfig(); - config.readFrom(quorumConfig); - zkServer.runFromConfig(config); - } - } - - } - - } catch (Exception e) { - logger.error("start standalone server fail！", e); - } - } - - -} \ No newline at end of file diff --git a/dolphinscheduler-service/pom.xml b/dolphinscheduler-service/pom.xml index 7d775d5497..03bb94dc29 100644 --- a/dolphinscheduler-service/pom.xml +++ b/dolphinscheduler-service/pom.xml @@ -48,6 +48,11 @@ + + org.apache.logging.log4j + log4j-core + + org.quartz-scheduler quartz-jobs diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java index 5daf535625..163cd3696f 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.service.log; -import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.remote.NettyRemotingClient; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.log.*; @@ -41,7 +40,7 @@ public class LogClientService { /** * request time out */ - private final long logRequestTimeout = 10 * 1000; + private static final long LOG_REQUEST_TIMEOUT = 10 * 1000L; /** * construct client @@ -76,7 +75,7 @@ public class LogClientService { final Address address = new Address(host, port); try { Command command = request.convert2Command(); - Command response = this.client.sendSync(address, command, logRequestTimeout); + Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); if(response != null){ RollViewLogResponseCommand rollReviewLog = FastJsonSerializer.deserialize( response.getBody(), RollViewLogResponseCommand.class); @@ -104,7 +103,7 @@ public class LogClientService { final Address address = new Address(host, port); try { Command command = request.convert2Command(); - Command response = this.client.sendSync(address, command, logRequestTimeout); + Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); if(response != null){ ViewLogResponseCommand viewLog = FastJsonSerializer.deserialize( response.getBody(), ViewLogResponseCommand.class); @@ -132,7 +131,7 @@ public class LogClientService { final Address address = new Address(host, port); try { Command command = request.convert2Command(); - Command response = this.client.sendSync(address, command, logRequestTimeout); + Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); if(response != null){ GetLogBytesResponseCommand getLog = FastJsonSerializer.deserialize( response.getBody(), GetLogBytesResponseCommand.class); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java index 027666f053..e53fae6e86 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java @@ -147,7 +147,7 @@ public class PermissionCheck { List unauthorizedList = processService.listUnauthorized(userId,needChecks,authorizationType); // if exist unauthorized resource if(CollectionUtils.isNotEmpty(unauthorizedList)){ - logger.error("user {} didn't has permission of {}: {}", user.getUserName(), authorizationType.getDescp(),unauthorizedList.toString()); + logger.error("user {} didn't has permission of {}: {}", user.getUserName(), authorizationType.getDescp(),unauthorizedList); throw new RuntimeException(String.format("user %s didn't has permission of %s %s", user.getUserName(), authorizationType.getDescp(), unauthorizedList.get(0))); } } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index b589cd4295..c848ec5197 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.service.process; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.cronutils.model.Cron; import org.apache.commons.lang.ArrayUtils; @@ -116,12 +117,12 @@ public class ProcessService { ProcessInstance processInstance = constructProcessInstance(command, host); //cannot construct process instance, return null; if(processInstance == null){ - logger.error("scan command, command parameter is error: %s", command.toString()); + logger.error("scan command, command parameter is error: {}", command); moveToErrorCommand(command, "process instance is null"); return null; } if(!checkThreadNum(command, validThreadNum)){ - logger.info("there is not enough thread for this command: {}",command.toString() ); + logger.info("there is not enough thread for this command: {}", command); return setWaitingThreadProcess(command, processInstance); } processInstance.setCommandType(command.getCommandType()); @@ -207,7 +208,7 @@ public class ProcessService { CommandType commandType = command.getCommandType(); if(cmdTypeMap.containsKey(commandType)){ - JSONObject cmdParamObj = (JSONObject) JSONObject.parse(command.getCommandParam()); + JSONObject cmdParamObj = (JSONObject) JSON.parse(command.getCommandParam()); JSONObject tempObj; int processInstanceId = cmdParamObj.getInteger(CMDPARAM_RECOVER_PROCESS_ID_STRING); @@ -215,7 +216,7 @@ public class ProcessService { // for all commands for (Command tmpCommand:commands){ if(cmdTypeMap.containsKey(tmpCommand.getCommandType())){ - tempObj = (JSONObject) JSONObject.parse(tmpCommand.getCommandParam()); + tempObj = (JSONObject) JSON.parse(tmpCommand.getCommandParam()); if(tempObj != null && processInstanceId == tempObj.getInteger(CMDPARAM_RECOVER_PROCESS_ID_STRING)){ isNeedCreate = false; break; @@ -309,7 +310,7 @@ public class ProcessService { for (TaskNode taskNode : taskNodeList){ String parameter = taskNode.getParams(); if (parameter.contains(CMDPARAM_SUB_PROCESS_DEFINE_ID)){ - SubProcessParameters subProcessParam = JSONObject.parseObject(parameter, SubProcessParameters.class); + SubProcessParameters subProcessParam = JSON.parseObject(parameter, SubProcessParameters.class); ids.add(subProcessParam.getProcessDefinitionId()); recurseFindSubProcessId(subProcessParam.getProcessDefinitionId(),ids); } @@ -991,7 +992,7 @@ public class ProcessService { return insertQueueResult; }catch (Exception e){ logger.error("submit task to queue Exception: ", e); - logger.error("task queue error : %s", JSONUtils.toJson(taskInstance)); + logger.error("task queue error : {}", JSONUtils.toJson(taskInstance)); return false; } } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java index 135bfdabc6..fa1a0bfced 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java @@ -65,14 +65,16 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator { if (splits.length != Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH){ return; } - String str = splits[0] + Constants.COMMA - + splits[1] + Constants.COMMA - + OSUtils.cpuUsage() + Constants.COMMA - + OSUtils.memoryUsage() + Constants.COMMA - + OSUtils.loadAverage() + Constants.COMMA - + splits[5] + Constants.COMMA - + DateUtils.dateToString(new Date()); - zkClient.setData().forPath(znode,str.getBytes()); + StringBuilder sb = new StringBuilder(); + sb.append(splits[0]).append(Constants.COMMA) + .append(splits[1]).append(Constants.COMMA) + .append(OSUtils.cpuUsage()).append(Constants.COMMA) + .append(OSUtils.memoryUsage()).append(Constants.COMMA) + .append(OSUtils.loadAverage()).append(Constants.COMMA) + .append(splits[5]).append(Constants.COMMA) + .append(DateUtils.dateToString(new Date())); + + zkClient.setData().forPath(znode, sb.toString().getBytes()); } catch (Exception e) { logger.error("heartbeat for zk failed", e); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProvider.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProvider.java index 9eedf7a4ca..dbe8bd6395 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProvider.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProvider.java @@ -45,4 +45,14 @@ public class DefaultEnsembleProvider implements EnsembleProvider { public void close() throws IOException { //NOP } + + @Override + public void setConnectionString(String connectionString) { + //NOP + } + + @Override + public boolean updateServerListEnabled() { + return false; + } } diff --git a/dolphinscheduler-service/src/test/java/queue/ZKServer.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java similarity index 66% rename from dolphinscheduler-service/src/test/java/queue/ZKServer.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java index 65fb95c02b..96331405d4 100644 --- a/dolphinscheduler-service/src/test/java/queue/ZKServer.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java @@ -14,11 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package queue; +package org.apache.dolphinscheduler.service.zk; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.zookeeper.server.ZooKeeperServer; import org.apache.zookeeper.server.ZooKeeperServerMain; import org.apache.zookeeper.server.quorum.QuorumPeerConfig; import org.slf4j.Logger; @@ -26,27 +24,45 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; /** - * just for test + * just speed experience version + * embedded zookeeper service */ public class ZKServer { - private static final Logger logger = LoggerFactory.getLogger(ZKServer.class); private static volatile PublicZooKeeperServerMain zkServer = null; public static final int DEFAULT_ZK_TEST_PORT = 2181; - public static final String DEFAULT_ZK_STR = "localhost:" + DEFAULT_ZK_TEST_PORT; - private static String dataDir = null; private static final AtomicBoolean isStarted = new AtomicBoolean(false); + public static void main(String[] args) { + if(!isStarted()){ + ZKServer.start(); + + /** + * register hooks, which are called before the process exits + */ + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + @Override + public void run() { + stop(); + } + })); + }else{ + logger.info("zk server aleady started"); + } + } + + /** + * start service + */ public static void start() { try { startLocalZkServer(DEFAULT_ZK_TEST_PORT); @@ -79,8 +95,8 @@ public class ZKServer { * @param port The port to listen on */ public static void startLocalZkServer(final int port) { - startLocalZkServer(port, org.apache.commons.io.FileUtils.getTempDirectoryPath() + File.separator + "test-" + System.currentTimeMillis()); - startLocalZkServer(port, org.apache.commons.io.FileUtils.getTempDirectoryPath() + "test-" + System.currentTimeMillis()); + + startLocalZkServer(port, System.getProperty("user.dir") +"/zookeeper_data", ZooKeeperServer.DEFAULT_TICK_TIME,"20"); } /** @@ -88,48 +104,28 @@ public class ZKServer { * * @param port The port to listen on * @param dataDirPath The path for the Zk data directory + * @param tickTime zk tick time + * @param maxClientCnxns zk max client connections */ - private static synchronized void startLocalZkServer(final int port, final String dataDirPath) { + private static synchronized void startLocalZkServer(final int port, final String dataDirPath,final int tickTime,String maxClientCnxns) { if (zkServer != null) { throw new RuntimeException("Zookeeper server is already started!"); } - try { - zkServer = new PublicZooKeeperServerMain(); - logger.info("Zookeeper data path : {} ", dataDirPath); - dataDir = dataDirPath; - final String[] args = new String[]{Integer.toString(port), dataDirPath}; - Thread init = new Thread(new Runnable() { - @Override - public void run() { - try { - zkServer.initializeAndRun(args); - } catch (QuorumPeerConfig.ConfigException e) { - logger.warn("Caught exception while starting ZK", e); - } catch (IOException e) { - logger.warn("Caught exception while starting ZK", e); - } - } - }, "init-zk-thread"); - init.start(); - } catch (Exception e) { - logger.warn("Caught exception while starting ZK", e); - throw new RuntimeException(e); - } - - CuratorFramework zkClient = CuratorFrameworkFactory.builder() - .connectString(DEFAULT_ZK_STR) - .retryPolicy(new ExponentialBackoffRetry(10,100)) - .sessionTimeoutMs(1000 * 30) - .connectionTimeoutMs(1000 * 30) - .build(); + zkServer = new PublicZooKeeperServerMain(); + logger.info("Zookeeper data path : {} ", dataDirPath); + dataDir = dataDirPath; + final String[] args = new String[]{Integer.toString(port), dataDirPath, Integer.toString(tickTime), maxClientCnxns}; try { - zkClient.blockUntilConnected(10, TimeUnit.SECONDS); - zkClient.close(); - } catch (InterruptedException ignore) { + logger.info("Zookeeper server started "); + isStarted.compareAndSet(false, true); + + zkServer.initializeAndRun(args); + } catch (QuorumPeerConfig.ConfigException e) { + logger.warn("Caught exception while starting ZK", e); + } catch (IOException e) { + logger.warn("Caught exception while starting ZK", e); } - isStarted.compareAndSet(false, true); - logger.info("zk server started"); } /** diff --git a/dolphinscheduler-service/src/main/resources/quartz.properties b/dolphinscheduler-service/src/main/resources/quartz.properties index 9c8930b647..b01be394c6 100644 --- a/dolphinscheduler-service/src/main/resources/quartz.properties +++ b/dolphinscheduler-service/src/main/resources/quartz.properties @@ -26,6 +26,10 @@ org.quartz.dataSource.myDs.URL = jdbc:postgresql://localhost:5432/dolphinschedul # mysql #org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver #org.quartz.dataSource.myDs.URL = jdbc:mysql://localhost:3306/dolphinscheduler?characterEncoding=utf8 +#h2 +#org.quartz.dataSource.myDs.driver=org.h2.Driver +#org.quartz.dataSource.myDs.URL=jdbc:h2:file:/Users/stone/work/myworkspace/incubator-dolphinscheduler/h2;AUTO_SERVER=TRUE + org.quartz.dataSource.myDs.user = test org.quartz.dataSource.myDs.password = test diff --git a/dolphinscheduler-service/src/test/java/cron/CronUtilsTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtilsTest.java similarity index 99% rename from dolphinscheduler-service/src/test/java/cron/CronUtilsTest.java rename to dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtilsTest.java index 6a402b5e67..b4f864c5b4 100644 --- a/dolphinscheduler-service/src/test/java/cron/CronUtilsTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtilsTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package cron; +package org.apache.dolphinscheduler.service.quartz.cron; import com.cronutils.builder.CronBuilder; import com.cronutils.model.Cron; @@ -25,7 +25,6 @@ import com.cronutils.model.field.CronFieldName; import com.cronutils.model.field.expression.*; import org.apache.dolphinscheduler.common.enums.CycleEnum; import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; diff --git a/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/BaseTaskQueueTest.java similarity index 93% rename from dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java rename to dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/BaseTaskQueueTest.java index a0cc457e22..17e2ae4056 100644 --- a/dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/BaseTaskQueueTest.java @@ -14,10 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package queue; +package org.apache.dolphinscheduler.service.queue; import org.apache.dolphinscheduler.service.queue.ITaskQueue; import org.apache.dolphinscheduler.service.queue.TaskQueueFactory; +import org.apache.dolphinscheduler.service.zk.ZKServer; import org.junit.*; /** diff --git a/dolphinscheduler-service/src/test/java/queue/TaskQueueZKImplTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskQueueZKImplTest.java similarity index 99% rename from dolphinscheduler-service/src/test/java/queue/TaskQueueZKImplTest.java rename to dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskQueueZKImplTest.java index d29c5aa610..5d464ac3c9 100644 --- a/dolphinscheduler-service/src/test/java/queue/TaskQueueZKImplTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskQueueZKImplTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package queue; +package org.apache.dolphinscheduler.service.queue; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.IpUtils; diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProviderTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProviderTest.java new file mode 100644 index 0000000000..cdec9d0547 --- /dev/null +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProviderTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.service.zk; + +import org.apache.curator.ensemble.EnsembleProvider; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.*; + +public class DefaultEnsembleProviderTest { + private static final String DEFAULT_SERVER_LIST = "localhost:2181"; + + @Test + public void startAndClose() { + EnsembleProvider ensembleProvider = new DefaultEnsembleProvider(DEFAULT_SERVER_LIST); + try { + ensembleProvider.start(); + } catch (Exception e) { + Assert.fail("EnsembleProvider start error: " + e.getMessage()); + } + try { + ensembleProvider.close(); + } catch (IOException e) { + Assert.fail("EnsembleProvider close error: " + e.getMessage()); + } + } + + @Test + public void getConnectionString() { + EnsembleProvider ensembleProvider = new DefaultEnsembleProvider(DEFAULT_SERVER_LIST); + Assert.assertEquals(DEFAULT_SERVER_LIST, ensembleProvider.getConnectionString()); + } + + @Test + public void setConnectionString() { + EnsembleProvider ensembleProvider = new DefaultEnsembleProvider(DEFAULT_SERVER_LIST); + ensembleProvider.setConnectionString("otherHost:2181"); + Assert.assertEquals(DEFAULT_SERVER_LIST, ensembleProvider.getConnectionString()); + } + + @Test + public void updateServerListEnabled() { + EnsembleProvider ensembleProvider = new DefaultEnsembleProvider(DEFAULT_SERVER_LIST); + Assert.assertFalse(ensembleProvider.updateServerListEnabled()); + } +} \ No newline at end of file diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java new file mode 100644 index 0000000000..48cde32287 --- /dev/null +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.service.zk; + +import org.junit.Ignore; +import org.junit.Test; + +import static org.junit.Assert.*; + +@Ignore +public class ZKServerTest { + + @Test + public void start() { + //ZKServer is a process, can't unit test + } + + @Test + public void isStarted() { + + } + + @Test + public void stop() { + ZKServer.stop(); + } +} \ No newline at end of file diff --git a/dolphinscheduler-ui/pom.xml b/dolphinscheduler-ui/pom.xml index 3fd9aa6650..78869ffbc4 100644 --- a/dolphinscheduler-ui/pom.xml +++ b/dolphinscheduler-ui/pom.xml @@ -89,6 +89,61 @@ + + rpmbuild + + + + com.github.eirslett + frontend-maven-plugin + ${frontend-maven-plugin.version} + + + install node and npm + + install-node-and-npm + + + ${node.version} + ${npm.version} + + + + npm install node-sass --unsafe-perm + + npm + + generate-resources + + install node-sass --unsafe-perm + + + + npm install + + npm + + generate-resources + + install + + + + npm run build:release + + npm + + + run build:release + + + + + + + + + nginx diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js index a9a51aa2b1..db8acf3073 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js @@ -26,7 +26,7 @@ import Permissions from '@/module/permissions' * @desc tooltip */ const toolOper = (dagThis) => { - let disabled =!dagThis.$store.state.dag.isDetails// Permissions.getAuth() === false ? false : !dagThis.$store.state.dag.isDetails + let disabled =!!dagThis.$store.state.dag.isDetails// Permissions.getAuth() === false ? false : !dagThis.$store.state.dag.isDetails return [ { code: 'pointer', @@ -49,13 +49,13 @@ const toolOper = (dagThis) => { { code: 'download', icon: 'ans-icon-download', - disable: !!dagThis.type, + disable: !dagThis.type, desc: `${i18n.$t('Download')}` }, { code: 'screen', icon: 'ans-icon-max', - disable: disabled, + disable: false, desc: `${i18n.$t('Full Screen')}` } ] diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue index d912a9a884..a1ccd39260 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue @@ -25,7 +25,7 @@ :key="v" v-for="(item,v) in tasksTypeList" @mousedown="_getDagId(v)"> -
+
@@ -293,7 +293,7 @@ let is = true let code = '' - if (!item.disable) { + if (item.disable) { return } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index 2500ce5772..cc1c8b6d6c 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -252,6 +252,7 @@ v-if="taskType === 'CONDITIONS'" ref="CONDITIONS" @on-dependent="_onDependent" + @on-cache-dependent="_onCacheDependent" :backfill-item="backfillItem" :pre-node="preNode"> @@ -280,7 +281,7 @@ import mDependent from './tasks/dependent' import mHttp from './tasks/http' import mDatax from './tasks/datax' - import mConditions from './tasks/CONDITIONS' + import mConditions from './tasks/conditions' import mSqoop from './tasks/sqoop' import mSubProcess from './tasks/sub_process' import mSelectInput from './_source/selectInput' @@ -438,6 +439,8 @@ }, _cacheItem () { + this.conditionResult.successNode[0] = this.successBranch + this.conditionResult.failedNode[0] = this.failedBranch this.$emit('cacheTaskInfo', { item: { type: this.taskType, @@ -446,12 +449,15 @@ params: this.params, description: this.description, runFlag: this.runFlag, + conditionResult: this.conditionResult, dependence: this.cacheDependence, maxRetryTimes: this.maxRetryTimes, retryInterval: this.retryInterval, timeout: this.timeout, taskInstancePriority: this.taskInstancePriority, - workerGroupId: this.workerGroupId + workerGroupId: this.workerGroupId, + status: this.status, + branch: this.branch }, fromThis: this }) @@ -657,7 +663,9 @@ retryInterval: this.retryInterval, timeout: this.timeout, taskInstancePriority: this.taskInstancePriority, - workerGroupId: this.workerGroupId + workerGroupId: this.workerGroupId, + successBranch: this.successBranch, + failedBranch: this.failedBranch } } }, diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/datasource.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/datasource.vue index aa067d80e4..a173139d15 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/datasource.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/datasource.vue @@ -122,8 +122,11 @@ }, // Watch the cacheParams watch: { - cacheParams (val) { - this.$emit('on-dsData', val); + datasource (val) { + this.$emit('on-dsData', { + type: this.type, + datasource: val + }); } }, created () { @@ -150,7 +153,8 @@ }) } this.$emit('on-dsData', { - type: this.type + type: this.type, + datasource: this.datasource }) }) }, diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue index 4afb8b46c5..fa7ee89e98 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue @@ -27,17 +27,17 @@ - + - + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue index 4ac04d91a6..fb3f2c295c 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue @@ -23,12 +23,8 @@ - -  - - -  - + +
@@ -44,14 +40,13 @@ @click="!isDetails && _setRelation($index)"> {{el.relation === 'AND' ? $t('and') : $t('or')}} - -  - + { this.isLoading = false }, 600) + }, + cacheDependence (val) { + this.$emit('on-cache-dependent', val) } }, beforeCreate () { @@ -158,7 +156,19 @@ }, destroyed () { }, - computed: {}, + computed: { + cacheDependence () { + return { + relation: this.relation, + dependTaskList: _.map(this.dependTaskList, v => { + return { + relation: v.relation, + dependItemList: _.map(v.dependItemList, v1 => _.omit(v1, ['depTasksList', 'state', 'dateValueList'])) + } + }) + } + } + }, components: { mListBox, mNodeStatus } } @@ -262,4 +272,4 @@ } } } - \ No newline at end of file + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue index ce918f49cf..959610f95a 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue @@ -232,6 +232,8 @@ * Processing code highlighting */ _handlerEditor () { + this._destroyEditor() + // editor editor = codemirror('code-sql-mirror', { mode: 'sql', @@ -249,9 +251,34 @@ // Monitor keyboard editor.on('keypress', this.keypress) + editor.on('changes', () => { + this._cacheParams() + }) + editor.setValue(this.sql) return editor + }, + _cacheParams () { + this.$emit('on-cache-params', { + dsType: this.dsType, + dataSource: this.rtDatasource, + dtType: this.dtType, + dataTarget: this.rtDatatarget, + sql: editor?editor.getValue():'', + targetTable: this.targetTable, + jobSpeedByte: this.jobSpeedByte * 1024, + jobSpeedRecord: this.jobSpeedRecord, + preStatements: this.preStatements, + postStatements: this.postStatements + }); + }, + _destroyEditor () { + if (editor) { + editor.toTextArea() // Uninstall + editor.off($('.code-sql-mirror'), 'keypress', this.keypress) + editor.off($('.code-sql-mirror'), 'changes', this.changes) + } } }, created () { @@ -286,7 +313,27 @@ editor.off($('.code-sql-mirror'), 'keypress', this.keypress) } }, - computed: {}, + watch: { + //Watch the cacheParams + cacheParams (val) { + this._cacheParams(); + } + }, + computed: { + cacheParams () { + return { + dsType: this.dsType, + dataSource: this.rtDatasource, + dtType: this.dtType, + dataTarget: this.rtDatatarget, + targetTable: this.targetTable, + jobSpeedByte: this.jobSpeedByte * 1024, + jobSpeedRecord: this.jobSpeedRecord, + preStatements: this.preStatements, + postStatements: this.postStatements + } + } + }, components: { mListBox, mDatasource, mLocalParams, mStatementList, mSelectInput } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue index e565b4a6bd..6f495d22a0 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue @@ -129,6 +129,8 @@ * Processing code highlighting */ _handlerEditor () { + this._destroyEditor() + // editor editor = codemirror('code-python-mirror', { mode: 'python', @@ -143,26 +145,45 @@ } } + this.changes = () => { + this._cacheParams() + } + // Monitor keyboard editor.on('keypress', this.keypress) + editor.on('changes', this.changes) + editor.setValue(this.rawScript) return editor + }, + _cacheParams () { + this.$emit('on-cache-params', { + resourceList: this.cacheResourceList, + localParams: this.localParams, + rawScript: editor ? editor.getValue() : '' + }); + }, + _destroyEditor () { + if (editor) { + editor.toTextArea() // Uninstall + editor.off($('.code-python-mirror'), 'keypress', this.keypress) + editor.off($('.code-python-mirror'), 'changes', this.changes) + } } }, watch: { //Watch the cacheParams cacheParams (val) { - this.$emit('on-cache-params', val); + this._cacheParams() } }, computed: { cacheParams () { return { resourceList: this.cacheResourceList, - localParams: this.localParams, - rawScript: editor ? editor.getValue() : '' + localParams: this.localParams } } }, @@ -193,8 +214,11 @@ }, 200) }, destroyed () { - editor.toTextArea() // Uninstall - editor.off($('.code-python-mirror'), 'keypress', this.keypress) + if (editor) { + editor.toTextArea() // Uninstall + editor.off($('.code-python-mirror'), 'keypress', this.keypress) + editor.off($('.code-python-mirror'), 'changes', this.changes) + } }, components: { mLocalParams, mListBox, mResources } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue index ad40c586b9..a4b20f3310 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue @@ -163,6 +163,8 @@ * Processing code highlighting */ _handlerEditor () { + this._destroyEditor() + // editor editor = codemirror('code-shell-mirror', { mode: 'shell', @@ -177,25 +179,45 @@ } } + this.changes = () => { + this._cacheParams() + } + // Monitor keyboard editor.on('keypress', this.keypress) + + editor.on('changes', this.changes) + editor.setValue(this.rawScript) return editor + }, + _cacheParams () { + this.$emit('on-cache-params', { + resourceList: this.cacheResourceList, + localParams: this.localParams, + rawScript: editor ? editor.getValue() : '' + }); + }, + _destroyEditor () { + if (editor) { + editor.toTextArea() // Uninstall + editor.off($('.code-sql-mirror'), 'keypress', this.keypress) + editor.off($('.code-sql-mirror'), 'changes', this.changes) + } } }, watch: { //Watch the cacheParams cacheParams (val) { - this.$emit('on-cache-params', val); + this._cacheParams() } }, computed: { cacheParams () { return { resourceList: this.cacheResourceList, - localParams: this.localParams, - rawScript: editor ? editor.getValue() : '' + localParams: this.localParams } } }, @@ -229,6 +251,7 @@ if (editor) { editor.toTextArea() // Uninstall editor.off($('.code-shell-mirror'), 'keypress', this.keypress) + editor.off($('.code-shell-mirror'), 'changes', this.changes) } }, components: { mLocalParams, mListBox, mResources, mScriptBox } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue index eb5c40992e..843c1eaf1f 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue @@ -318,6 +318,8 @@ * Processing code highlighting */ _handlerEditor () { + this._destroyEditor() + // editor editor = codemirror('code-sql-mirror', { mode: 'sql', @@ -332,9 +334,15 @@ } } + this.changes = () => { + this._cacheParams() + } + // Monitor keyboard editor.on('keypress', this.keypress) + editor.on('changes', this.changes) + editor.setValue(this.sql) return editor @@ -351,6 +359,38 @@ this.receivers = res.receivers && res.receivers.split(',') || [] this.receiversCc = res.receiversCc && res.receiversCc.split(',') || [] }) + }, + _cacheParams () { + this.$emit('on-cache-params', { + type: this.type, + datasource: this.rtDatasource, + sql: editor ? editor.getValue() : '', + udfs: this.udfs, + sqlType: this.sqlType, + title: this.title, + receivers: this.receivers.join(','), + receiversCc: this.receiversCc.join(','), + showType: (() => { + + let showType = this.showType + if (showType.length === 2 && showType[0] === 'ATTACHMENT') { + return [showType[1], showType[0]].join(',') + } else { + return showType.join(',') + } + })(), + localParams: this.localParams, + connParams: this.connParams, + preStatements: this.preStatements, + postStatements: this.postStatements + }); + }, + _destroyEditor () { + if (editor) { + editor.toTextArea() // Uninstall + editor.off($('.code-sql-mirror'), 'keypress', this.keypress) + editor.off($('.code-sql-mirror'), 'changes', this.changes) + } } }, watch: { @@ -373,7 +413,7 @@ }, //Watch the cacheParams cacheParams (val) { - this.$emit('on-cache-params', val); + this._cacheParams() } }, created () { @@ -418,6 +458,7 @@ if (editor) { editor.toTextArea() // Uninstall editor.off($('.code-sql-mirror'), 'keypress', this.keypress) + editor.off($('.code-sql-mirror'), 'changes', this.changes) } }, computed: { @@ -425,7 +466,6 @@ return { type: this.type, datasource: this.rtDatasource, - sql: editor ? editor.getValue() : '', udfs: this.udfs, sqlType: this.sqlType, title: this.title, diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sqoop.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sqoop.vue index eead745a06..6594ffcf74 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sqoop.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sqoop.vue @@ -65,7 +65,7 @@
@@ -186,8 +186,8 @@
@@ -385,7 +385,7 @@
@@ -556,7 +556,8 @@ targetType:"HDFS", sourceMysqlParams:{ - srcDatasource:-1, + srcType:"MYSQL", + srcDatasource:"", srcTable:"", srcQueryType:"1", srcQuerySql:'', @@ -588,7 +589,8 @@ }, targetMysqlParams:{ - targetDatasource:-1, + targetType:"MYSQL", + targetDatasource:"", targetTable:"", targetColumns:"", fieldsTerminated:"", @@ -680,6 +682,7 @@ * return data source */ _onSourceDsData (o) { + this.sourceMysqlParams.srcType = o.type this.sourceMysqlParams.srcDatasource = o.datasource }, @@ -687,6 +690,7 @@ * return data source */ _onTargetDsData (o) { + this.targetMysqlParams.targetType = o.type this.targetMysqlParams.targetDatasource = o.datasource }, @@ -697,7 +701,7 @@ var params = null switch(this.sourceType){ case "MYSQL": - this.sourceMysqlParams.srcQuerySql = editor.getValue() + this.sourceMysqlParams.srcQuerySql = editor ? editor.getValue() : this.sourceMysqlParams.srcQuerySql params = JSON.stringify(this.sourceMysqlParams) break; case "ORACLE": @@ -879,7 +883,9 @@ * Processing code highlighting */ _handlerEditor () { - editor = codemirror('code-sql-mirror', { + this._destroyEditor() + + editor = codemirror('code-sqoop-mirror', { mode: 'sql', readOnly: this.isDetails }) @@ -892,9 +898,15 @@ } } + this.changes = () => { + this._cacheParams() + } + // Monitor keyboard editor.on('keypress', this.keypress) + editor.on('changes', this.changes) + editor.setValue(this.sourceMysqlParams.srcQuerySql) return editor @@ -906,6 +918,27 @@ _onLocalParams (a) { this.localParams = a }, + + _cacheParams () { + this.$emit('on-cache-params', { + concurrency:this.concurrency, + modelType:this.modelType, + sourceType:this.sourceType, + targetType:this.targetType, + sourceParams:this._handleSourceParams(), + targetParams:this._handleTargetParams(), + localParams:this.localParams + }); + }, + + _destroyEditor () { + if (editor) { + editor.toTextArea() // Uninstall + editor.off($('.code-sqoop-mirror'), 'keypress', this.keypress) + editor.off($('.code-sqoop-mirror'), 'changes', this.changes) + editor = null + } + }, }, watch: { // Listening to sqlType @@ -927,11 +960,12 @@ }, //Watch the cacheParams cacheParams (val) { - this.$emit('on-cache-params', val); + this._cacheParams() } }, created () { + this._destroyEditor() let o = this.backfillItem // Non-null objects represent backfill @@ -963,11 +997,28 @@ */ if (editor) { editor.toTextArea() // Uninstall - editor.off($('.code-sql-mirror'), 'keypress', this.keypress) + editor.off($('.code-sqoop-mirror'), 'keypress', this.keypress) + editor.off($('.code-sqoop-mirror'), 'changes', this.changes) + editor = null } }, computed: { + cacheParams () { + return { + concurrency:this.concurrency, + modelType:this.modelType, + sourceType:this.sourceType, + targetType:this.targetType, + localParams:this.localParams, + sourceMysqlParams:this.sourceMysqlParams, + sourceHdfsParams:this.sourceHdfsParams, + sourceHiveParams:this.sourceHiveParams, + targetHdfsParams:this.targetHdfsParams, + targetMysqlParams:this.targetMysqlParams, + targetHiveParams:this.targetHiveParams + } + } }, components: { mListBox, mDatasource, mLocalParams} } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue index caa31ed4d8..df84f0f292 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/create/index.vue @@ -132,7 +132,10 @@ this.$message.warning(`${i18n.$t('Please enter the resource content')}`) return false } - + if (editor.doc.size>3000) { + this.$message.warning(`${i18n.$t('Resource content cannot exceed 3000 lines')}`) + return false + } return true }, /** diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue index a9aee1c9fd..a0d1d7d187 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue @@ -45,6 +45,7 @@