Browse Source

Merge pull request #2 from apache/dev

aaa
pull/2/head
Simon 5 years ago committed by GitHub
parent
commit
c583c42c2d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 16
      .github/workflows/ci_backend.yml
  2. 12
      .github/workflows/ci_e2e.yml
  3. 16
      .github/workflows/ci_frontend.yml
  4. 20
      .github/workflows/ci_ut.yml
  5. 2
      README.md
  6. 2
      ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py
  7. 19
      dockerfile/Dockerfile
  8. 40
      dockerfile/README.md
  9. 38
      dockerfile/README_zh_CN.md
  10. 27
      dockerfile/checkpoint.sh
  11. 2
      dockerfile/conf/dolphinscheduler/application.properties.tpl
  12. 12
      dockerfile/conf/dolphinscheduler/common.properties.tpl
  13. 0
      dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh
  14. 2
      dockerfile/conf/dolphinscheduler/quartz.properties.tpl
  15. 2
      dockerfile/hooks/check
  16. 4
      dockerfile/startup-init-conf.sh
  17. 8
      dockerfile/startup.sh
  18. 4
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java
  19. 1
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java
  20. 38
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java
  21. 2
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java
  22. 13
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java
  23. 54
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java
  24. 1
      dolphinscheduler-alert/src/main/resources/alert.properties
  25. 8
      dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java
  26. 2
      dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java
  27. 28
      dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java
  28. 6
      dolphinscheduler-api/pom.xml
  29. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java
  30. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java
  31. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java
  32. 11
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java
  33. 25
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java
  34. 15
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java
  35. 1
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java
  36. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java
  37. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java
  38. 1
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java
  39. 1
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java
  40. 2
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java
  41. 25
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
  42. 5
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java
  43. 1
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java
  44. 8
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java
  45. 1
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMainTest.java
  46. 2
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitorUtilsTest.java
  47. 2
      dolphinscheduler-common/pom.xml
  48. 7
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
  49. 3
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/WorkerLogFilter.java
  50. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java
  51. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java
  52. 14
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java
  53. 14
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/mr/MapreduceParameters.java
  54. 17
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java
  55. 6
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
  56. 254
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
  57. 12
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java
  58. 19
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java
  59. 5
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java
  60. 25
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java
  61. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java
  62. 16
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
  63. 38
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java
  64. 10
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java
  65. 6
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/PlaceholderUtils.java
  66. 1071
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32.java
  67. 286
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32.java
  68. 785
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java
  69. 2
      dolphinscheduler-common/src/main/resources/common.properties
  70. 40
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java
  71. 2
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java
  72. 1
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java
  73. 55
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java
  74. 17
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
  75. 3
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java
  76. 6
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/IpUtilsTest.java
  77. 22
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java
  78. 24
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java
  79. 8
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java
  80. 3
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java
  81. 3
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java
  82. 2
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java
  83. 3
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java
  84. 210
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32Test.java
  85. 124
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32Test.java
  86. 70
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32Test.java
  87. 21
      dolphinscheduler-dao/pom.xml
  88. 6
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java
  89. 2
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java
  90. 134
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java
  91. 2
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java
  92. 2
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java
  93. 2
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java
  94. 9
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java
  95. 2
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java
  96. 4
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java
  97. 1
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java
  98. 24
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java
  99. 40
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java
  100. 3
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java
  101. Some files were not shown because too many files have changed in this diff Show More

16
.github/workflows/ci_backend.yml

@ -45,7 +45,13 @@ jobs:
Compile-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- uses: actions/checkout@v2
# In the checkout@v2, it doesn't support git submodule. Execute the commands manually.
- name: checkout submodules
shell: bash
run: |
git submodule sync --recursive
git -c protocol.version=2 submodule update --init --force --recursive --depth=1
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
@ -55,7 +61,13 @@ jobs:
License-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- uses: actions/checkout@v2
# In the checkout@v2, it doesn't support git submodule. Execute the commands manually.
- name: checkout submodules
shell: bash
run: |
git submodule sync --recursive
git -c protocol.version=2 submodule update --init --force --recursive --depth=1
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:

12
.github/workflows/ci_e2e.yml

@ -15,7 +15,7 @@
# limitations under the License.
#
on: ["push", "pull_request"]
on: ["pull_request"]
env:
DOCKER_DIR: ./docker
LOG_DIR: /tmp/dolphinscheduler
@ -29,9 +29,13 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
with:
submodules: true
- uses: actions/checkout@v2
# In the checkout@v2, it doesn't support git submodule. Execute the commands manually.
- name: checkout submodules
shell: bash
run: |
git submodule sync --recursive
git -c protocol.version=2 submodule update --init --force --recursive --depth=1
- uses: actions/cache@v1
with:
path: ~/.m2/repository

16
.github/workflows/ci_frontend.yml

@ -34,7 +34,13 @@ jobs:
matrix:
os: [ubuntu-latest, macos-latest]
steps:
- uses: actions/checkout@v1
- uses: actions/checkout@v2
# In the checkout@v2, it doesn't support git submodule. Execute the commands manually.
- name: checkout submodules
shell: bash
run: |
git submodule sync --recursive
git -c protocol.version=2 submodule update --init --force --recursive --depth=1
- name: Set up Node.js
uses: actions/setup-node@v1
with:
@ -49,7 +55,13 @@ jobs:
License-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- uses: actions/checkout@v2
# In the checkout@v2, it doesn't support git submodule. Execute the commands manually.
- name: checkout submodules
shell: bash
run: |
git submodule sync --recursive
git -c protocol.version=2 submodule update --init --force --recursive --depth=1
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:

20
.github/workflows/ci_ut.yml

@ -15,7 +15,7 @@
# limitations under the License.
#
on: ["push", "pull_request"]
on: ["pull_request", "push"]
env:
DOCKER_DIR: ./docker
LOG_DIR: /tmp/dolphinscheduler
@ -29,9 +29,13 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
with:
submodules: true
- uses: actions/checkout@v2
# In the checkout@v2, it doesn't support git submodule. Execute the commands manually.
- name: checkout submodules
shell: bash
run: |
git submodule sync --recursive
git -c protocol.version=2 submodule update --init --force --recursive --depth=1
- uses: actions/cache@v1
with:
path: ~/.m2/repository
@ -48,7 +52,15 @@ jobs:
run: |
export MAVEN_OPTS='-Dmaven.repo.local=.m2/repository -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:-UseGCOverheadLimit -Xmx3g'
mvn test -B -Dmaven.test.skip=false
- name: Upload coverage report to codecov
if: github.event_name == 'pull_request'
run: |
CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash)
- name: Git fetch unshallow
run: |
git fetch --unshallow
git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
git fetch origin
- name: Run SonarCloud Analysis
run: >
mvn verify --batch-mode

2
README.md

@ -17,7 +17,7 @@ Dolphin Scheduler Official Website
### Design features:
A distributed and easy-to-expand visual DAG workflow scheduling system. Dedicated to solving the complex dependencies in data processing, making the scheduling system `out of the box` for data processing.
A distributed and easy-to-extend visual DAG workflow scheduling system. Dedicated to solving the complex dependencies in data processing, making the scheduling system `out of the box` for data processing.
Its main objectives are as follows:
- Associate the Tasks according to the dependencies of the tasks in a DAG graph, which can visualize the running state of task in real time.

2
ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py

@ -30,7 +30,7 @@ sys.setdefaultencoding('utf-8')
config = Script.get_config()
# conf_dir = "/etc/"
dolphin_home = "/opt/soft/apache-dolphinscheduler-incubating-1.2.1"
dolphin_home = "/opt/soft/dolphinscheduler"
dolphin_conf_dir = dolphin_home + "/conf"
dolphin_log_dir = dolphin_home + "/logs"
dolphin_bin_dir = dolphin_home + "/bin"

19
dockerfile/Dockerfile

@ -23,11 +23,11 @@ ENV TZ Asia/Shanghai
ENV LANG C.UTF-8
ENV DEBIAN_FRONTEND noninteractive
#1. install dos2unix shadow bash openrc python sudo vim wget iputils net-tools ssh pip kazoo.
#1. install dos2unix shadow bash openrc python sudo vim wget iputils net-tools ssh pip tini kazoo.
#If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example:
#RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories
RUN apk update && \
apk add dos2unix shadow bash openrc python sudo vim wget iputils net-tools openssh-server py2-pip && \
apk add dos2unix shadow bash openrc python sudo vim wget iputils net-tools openssh-server py2-pip tini && \
apk add --update procps && \
openrc boot && \
pip install kazoo
@ -63,19 +63,22 @@ RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \
ADD ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d
#7. add configuration and modify permissions and set soft links
ADD ./checkpoint.sh /root/checkpoint.sh
ADD ./startup-init-conf.sh /root/startup-init-conf.sh
ADD ./startup.sh /root/startup.sh
ADD ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/
ADD ./conf/dolphinscheduler/env/dolphinscheduler_env /opt/dolphinscheduler/conf/env/
RUN chmod +x /root/startup-init-conf.sh && \
ADD conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/
RUN chmod +x /root/checkpoint.sh && \
chmod +x /root/startup-init-conf.sh && \
chmod +x /root/startup.sh && \
chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env && \
chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \
chmod +x /opt/dolphinscheduler/script/*.sh && \
chmod +x /opt/dolphinscheduler/bin/*.sh && \
chmod +x /opt/zookeeper/bin/*.sh && \
dos2unix /root/checkpoint.sh && \
dos2unix /root/startup-init-conf.sh && \
dos2unix /root/startup.sh && \
dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env && \
dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \
dos2unix /opt/dolphinscheduler/script/*.sh && \
dos2unix /opt/dolphinscheduler/bin/*.sh && \
dos2unix /opt/zookeeper/bin/*.sh && \
@ -87,6 +90,6 @@ RUN chmod +x /root/startup-init-conf.sh && \
RUN rm -rf /var/cache/apk/*
#9. expose port
EXPOSE 2181 2888 3888 5432 12345 8888
EXPOSE 2181 2888 3888 5432 12345 50051 8888
ENTRYPOINT ["/root/startup.sh"]
ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"]

40
dockerfile/README.md

@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org
#### You can start a dolphinscheduler instance
```
$ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test \
-e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \
-p 8888:8888 \
dolphinscheduler all
```
@ -25,13 +25,13 @@ The default postgres user `root`, postgres password `root` and database `dolphin
The default zookeeper is created in the `startup.sh`.
#### Or via Environment Variables **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`ZOOKEEPER_QUORUM`**
#### Or via Environment Variables **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`POSTGRESQL_DATABASE`** **`ZOOKEEPER_QUORUM`**
You can specify **existing postgres service**. Example:
```
$ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
-p 8888:8888 \
dolphinscheduler all
@ -42,7 +42,7 @@ You can specify **existing zookeeper service**. Example:
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-p 8888:8888 \
dolphinscheduler all
```
@ -56,7 +56,7 @@ You can start a standalone dolphinscheduler server.
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
dolphinscheduler master-server
```
@ -66,7 +66,7 @@ dolphinscheduler master-server
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
dolphinscheduler worker-server
```
@ -75,7 +75,7 @@ dolphinscheduler worker-server
```
$ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
-p 12345:12345 \
dolphinscheduler api-server
@ -85,7 +85,7 @@ dolphinscheduler api-server
```
$ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
dolphinscheduler alert-server
```
@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \
dolphinscheduler frontend
```
**Note**: You must be specify `POSTGRESQL_HOST` `POSTGRESQL_PORT` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server.
**Note**: You must be specify `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server.
## How to build a docker image
@ -140,14 +140,36 @@ This environment variable sets the port for PostgreSQL. The default value is `54
This environment variable sets the username for PostgreSQL. The default value is `root`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`POSTGRESQL_PASSWORD`**
This environment variable sets the password for PostgreSQL. The default value is `root`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`POSTGRESQL_DATABASE`**
This environment variable sets the database for PostgreSQL. The default value is `dolphinscheduler`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`DOLPHINSCHEDULER_ENV_PATH`**
This environment variable sets the runtime environment for task. The default value is `/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh`.
**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`**
User data directory path, self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler`
**`DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH`**
Directory path for user data download. self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler/download`
**`DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH`**
Process execute directory. self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler/exec`
**`TASK_QUEUE`**
This environment variable sets the task queue for `master-server` and `worker-serverr`. The default value is `zookeeper`.

38
dockerfile/README_zh_CN.md

@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org
#### 你可以运行一个dolphinscheduler实例
```
$ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test \
-e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \
-p 8888:8888 \
dolphinscheduler all
```
@ -31,7 +31,7 @@ dolphinscheduler all
```
$ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
-p 8888:8888 \
dolphinscheduler all
@ -42,7 +42,7 @@ dolphinscheduler all
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-p 8888:8888 \
dolphinscheduler all
```
@ -56,7 +56,7 @@ dolphinscheduler all
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
dolphinscheduler master-server
```
@ -66,7 +66,7 @@ dolphinscheduler master-server
```
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
dolphinscheduler worker-server
```
@ -75,7 +75,7 @@ dolphinscheduler worker-server
```
$ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
-p 12345:12345 \
dolphinscheduler api-server
@ -85,7 +85,7 @@ dolphinscheduler api-server
```
$ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
dolphinscheduler alert-server
```
@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \
dolphinscheduler frontend
```
**注意**: 当你运行dolphinscheduler中的部分服务时,你必须指定这些环境变量 `POSTGRESQL_HOST` `POSTGRESQL_PORT` `ZOOKEEPER_QUORUM`
**注意**: 当你运行dolphinscheduler中的部分服务时,你必须指定这些环境变量 `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM`
## 如何构建一个docker镜像
@ -140,14 +140,36 @@ Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些
配置`PostgreSQL`的`USERNAME`, 默认值 `root`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`POSTGRESQL_PASSWORD`**
配置`PostgreSQL`的`PASSWORD`, 默认值 `root`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`POSTGRESQL_DATABASE`**
配置`PostgreSQL`的`DATABASE`, 默认值 `dolphinscheduler`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`DOLPHINSCHEDULER_ENV_PATH`**
任务执行时的环境变量配置文件, 默认值 `/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh`
**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`**
用户数据目录, 用户自己配置, 请确保这个目录存在并且用户读写权限, 默认值 `/tmp/dolphinscheduler`
**`DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH`**
用户数据下载目录, 用户自己配置, 请确保这个目录存在并且用户读写权限, 默认值 `/tmp/dolphinscheduler/download`
**`DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH`**
任务执行目录, 用户自己配置, 请确保这个目录存在并且用户读写权限, 默认值 `/tmp/dolphinscheduler/exec`
**`TASK_QUEUE`**
配置`master-server`和`worker-serverr`的`Zookeeper`任务队列名, 默认值 `zookeeper`

27
dockerfile/checkpoint.sh

@ -0,0 +1,27 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -e
if [ "$(ps -ef | grep java | grep -c $1)" -eq 0 ]; then
echo "[ERROR] $1 process not exits."
exit 1
else
echo "[INFO] $1 process exits."
exit 0
fi

2
dockerfile/conf/dolphinscheduler/application.properties.tpl

@ -19,7 +19,7 @@
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
# postgre
spring.datasource.driver-class-name=org.postgresql.Driver
spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/dolphinscheduler?characterEncoding=utf8
spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8
# mysql
#spring.datasource.driver-class-name=com.mysql.jdbc.Driver
#spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8

12
dockerfile/conf/dolphinscheduler/common.properties.tpl

@ -38,6 +38,12 @@ dolphinscheduler.env.path=${DOLPHINSCHEDULER_ENV_PATH}
resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties
# is development state? default "false"
development.state=true
# user data directory path, self configuration, please make sure the directory exists and have read write permissions
data.basedir.path=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH}
# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions
data.download.basedir.path=${DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH}
# process execute directory. self configuration, please make sure the directory exists and have read write permissions
process.exec.basepath=${DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH}
# resource upload startup type : HDFS,S3,NONE
res.upload.startup.type=NONE
@ -49,12 +55,6 @@ res.upload.startup.type=NONE
hdfs.root.user=hdfs
# data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended
data.store2hdfs.basepath=/dolphinscheduler
# user data directory path, self configuration, please make sure the directory exists and have read write permissions
data.basedir.path=/tmp/dolphinscheduler
# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions
data.download.basedir.path=/tmp/dolphinscheduler/download
# process execute directory. self configuration, please make sure the directory exists and have read write permissions
process.exec.basepath=/tmp/dolphinscheduler/exec
# whether kerberos starts
hadoop.security.authentication.startup.state=false
# java.security.krb5.conf path

0
dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env → dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh vendored

2
dockerfile/conf/dolphinscheduler/quartz.properties.tpl

@ -22,7 +22,7 @@
org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
# postgre
org.quartz.dataSource.myDs.driver = org.postgresql.Driver
org.quartz.dataSource.myDs.URL = jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/dolphinscheduler?characterEncoding=utf8
org.quartz.dataSource.myDs.URL = jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8
org.quartz.dataSource.myDs.user = ${POSTGRESQL_USERNAME}
org.quartz.dataSource.myDs.password = ${POSTGRESQL_PASSWORD}
org.quartz.scheduler.instanceName = DolphinScheduler

2
dockerfile/hooks/check

@ -17,7 +17,7 @@
#
echo "------ dolphinscheduler check - server - status -------"
sleep 20
server_num=$(docker top `docker container list | grep startup | awk '{print $1}'`| grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l)
server_num=$(docker top `docker container list | grep '/sbin/tini' | awk '{print $1}'`| grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l)
if [ $server_num -eq 5 ]
then
echo "Server all start successfully"

4
dockerfile/startup-init-conf.sh

@ -28,11 +28,15 @@ export POSTGRESQL_HOST=${POSTGRESQL_HOST:-"127.0.0.1"}
export POSTGRESQL_PORT=${POSTGRESQL_PORT:-"5432"}
export POSTGRESQL_USERNAME=${POSTGRESQL_USERNAME:-"root"}
export POSTGRESQL_PASSWORD=${POSTGRESQL_PASSWORD:-"root"}
export POSTGRESQL_DATABASE=${POSTGRESQL_DATABASE:-"dolphinscheduler"}
#============================================================================
# System
#============================================================================
export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"}
export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"}
export DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH:-"/tmp/dolphinscheduler/download"}
export DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH=${DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH:-"/tmp/dolphinscheduler/exec"}
#============================================================================
# Zookeeper

8
dockerfile/startup.sh

@ -164,6 +164,7 @@ case "$1" in
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log
;;
(api-server)
initZK
initPostgreSQL
initApiServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log
@ -187,6 +188,9 @@ case "$1" in
;;
esac
echo "tee begin"
exec tee ${LOGFILE}
# init directories and log files
mkdir -p ${DOLPHINSCHEDULER_LOGS} && mkdir -p /var/log/nginx/ && cat /dev/null >> ${LOGFILE}
echo "tail begin"
exec bash -c "tail -n 1 -f ${LOGFILE}"

4
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java

@ -42,8 +42,8 @@ public class EnterpriseWeChatManager {
public Map<String,Object> send(Alert alert, String token){
Map<String,Object> retMap = new HashMap<>();
retMap.put(Constants.STATUS, false);
String agentId = EnterpriseWeChatUtils.enterpriseWeChatAgentId;
String users = EnterpriseWeChatUtils.enterpriseWeChatUsers;
String agentId = EnterpriseWeChatUtils.ENTERPRISE_WE_CHAT_AGENT_ID;
String users = EnterpriseWeChatUtils.ENTERPRISE_WE_CHAT_USERS;
List<String> userList = Arrays.asList(users.split(","));
logger.info("send message {}",alert);
String msg = EnterpriseWeChatUtils.makeUserSendMsg(userList, agentId,EnterpriseWeChatUtils.markdownByAlert(alert));

1
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java

@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.alert.template.impl;
import org.apache.dolphinscheduler.alert.template.AlertTemplate;
import org.apache.dolphinscheduler.alert.utils.Constants;
import org.apache.dolphinscheduler.alert.utils.JSONUtils;
import org.apache.dolphinscheduler.alert.utils.MailUtils;
import org.apache.dolphinscheduler.common.enums.ShowType;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.slf4j.Logger;

38
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java

@ -43,24 +43,24 @@ public class EnterpriseWeChatUtils {
public static final Logger logger = LoggerFactory.getLogger(EnterpriseWeChatUtils.class);
private static final String enterpriseWeChatCorpId = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_CORP_ID);
private static final String ENTERPRISE_WE_CHAT_CORP_ID = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_CORP_ID);
private static final String enterpriseWeChatSecret = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_SECRET);
private static final String ENTERPRISE_WE_CHAT_SECRET = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_SECRET);
private static final String enterpriseWeChatTokenUrl = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TOKEN_URL);
private static String enterpriseWeChatTokenUrlReplace = enterpriseWeChatTokenUrl
.replaceAll("\\$corpId", enterpriseWeChatCorpId)
.replaceAll("\\$secret", enterpriseWeChatSecret);
private static final String ENTERPRISE_WE_CHAT_TOKEN_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TOKEN_URL);
private static final String ENTERPRISE_WE_CHAT_TOKEN_URL_REPLACE = ENTERPRISE_WE_CHAT_TOKEN_URL
.replaceAll("\\$corpId", ENTERPRISE_WE_CHAT_CORP_ID)
.replaceAll("\\$secret", ENTERPRISE_WE_CHAT_SECRET);
private static final String enterpriseWeChatPushUrl = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_PUSH_URL);
private static final String ENTERPRISE_WE_CHAT_PUSH_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_PUSH_URL);
private static final String enterpriseWeChatTeamSendMsg = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TEAM_SEND_MSG);
private static final String ENTERPRISE_WE_CHAT_TEAM_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TEAM_SEND_MSG);
private static final String enterpriseWeChatUserSendMsg = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USER_SEND_MSG);
private static final String ENTERPRISE_WE_CHAT_USER_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USER_SEND_MSG);
public static final String enterpriseWeChatAgentId = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_AGENT_ID);
public static final String ENTERPRISE_WE_CHAT_AGENT_ID = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_AGENT_ID);
public static final String enterpriseWeChatUsers = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USERS);
public static final String ENTERPRISE_WE_CHAT_USERS = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USERS);
/**
* get Enterprise WeChat is enable
@ -87,7 +87,7 @@ public class EnterpriseWeChatUtils {
CloseableHttpClient httpClient = HttpClients.createDefault();
try {
HttpGet httpGet = new HttpGet(enterpriseWeChatTokenUrlReplace);
HttpGet httpGet = new HttpGet(ENTERPRISE_WE_CHAT_TOKEN_URL_REPLACE);
CloseableHttpResponse response = httpClient.execute(httpGet);
try {
HttpEntity entity = response.getEntity();
@ -114,7 +114,7 @@ public class EnterpriseWeChatUtils {
* @return Enterprise WeChat send message
*/
public static String makeTeamSendMsg(String toParty, String agentId, String msg) {
return enterpriseWeChatTeamSendMsg.replaceAll("\\$toParty", toParty)
return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\$toParty", toParty)
.replaceAll("\\$agentId", agentId)
.replaceAll("\\$msg", msg);
}
@ -128,7 +128,7 @@ public class EnterpriseWeChatUtils {
*/
public static String makeTeamSendMsg(Collection<String> toParty, String agentId, String msg) {
String listParty = FuncUtils.mkString(toParty, "|");
return enterpriseWeChatTeamSendMsg.replaceAll("\\$toParty", listParty)
return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\$toParty", listParty)
.replaceAll("\\$agentId", agentId)
.replaceAll("\\$msg", msg);
}
@ -141,7 +141,7 @@ public class EnterpriseWeChatUtils {
* @return Enterprise WeChat send message
*/
public static String makeUserSendMsg(String toUser, String agentId, String msg) {
return enterpriseWeChatUserSendMsg.replaceAll("\\$toUser", toUser)
return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\$toUser", toUser)
.replaceAll("\\$agentId", agentId)
.replaceAll("\\$msg", msg);
}
@ -155,7 +155,7 @@ public class EnterpriseWeChatUtils {
*/
public static String makeUserSendMsg(Collection<String> toUser, String agentId, String msg) {
String listUser = FuncUtils.mkString(toUser, "|");
return enterpriseWeChatUserSendMsg.replaceAll("\\$toUser", listUser)
return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\$toUser", listUser)
.replaceAll("\\$agentId", agentId)
.replaceAll("\\$msg", msg);
}
@ -169,7 +169,7 @@ public class EnterpriseWeChatUtils {
* @throws IOException the IOException
*/
public static String sendEnterpriseWeChat(String charset, String data, String token) throws IOException {
String enterpriseWeChatPushUrlReplace = enterpriseWeChatPushUrl.replaceAll("\\$token", token);
String enterpriseWeChatPushUrlReplace = ENTERPRISE_WE_CHAT_PUSH_URL.replaceAll("\\$token", token);
CloseableHttpClient httpClient = HttpClients.createDefault();
try {
@ -184,8 +184,8 @@ public class EnterpriseWeChatUtils {
} finally {
response.close();
}
logger.info("Enterprise WeChat send [{}], param:{}, resp:{}",
enterpriseWeChatPushUrl, data, resp);
logger.info("Enterprise WeChat send [{}], param:{}, resp:{}",
ENTERPRISE_WE_CHAT_PUSH_URL, data, resp);
return resp;
} finally {
httpClient.close();

2
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java

@ -20,7 +20,7 @@ import org.apache.dolphinscheduler.common.utils.StringUtils;
public class FuncUtils {
static public String mkString(Iterable<String> list, String split) {
public static String mkString(Iterable<String> list, String split) {
if (null == list || StringUtils.isEmpty(split)){
return null;

13
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java

@ -16,12 +16,13 @@
*/
package org.apache.dolphinscheduler.alert.utils;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collections;
import java.util.List;
/**
@ -38,7 +39,7 @@ public class JSONUtils {
*/
public static String toJsonString(Object object) {
try{
return JSONObject.toJSONString(object,false);
return JSON.toJSONString(object,false);
} catch (Exception e) {
throw new RuntimeException("Json deserialization exception.", e);
}
@ -50,19 +51,19 @@ public class JSONUtils {
* @param json the json
* @param clazz c
* @param <T> the generic clazz
* @return the result list
* @return the result list or empty list
*/
public static <T> List<T> toList(String json, Class<T> clazz) {
if (StringUtils.isEmpty(json)) {
return null;
return Collections.emptyList();
}
try {
return JSONArray.parseArray(json, clazz);
return JSON.parseArray(json, clazz);
} catch (Exception e) {
logger.error("JSONArray.parseArray exception!",e);
}
return null;
return Collections.emptyList();
}
}

54
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java

@ -39,29 +39,29 @@ public class MailUtils {
public static final Logger logger = LoggerFactory.getLogger(MailUtils.class);
public static final String mailProtocol = PropertyUtils.getString(Constants.MAIL_PROTOCOL);
public static final String MAIL_PROTOCOL = PropertyUtils.getString(Constants.MAIL_PROTOCOL);
public static final String mailServerHost = PropertyUtils.getString(Constants.MAIL_SERVER_HOST);
public static final String MAIL_SERVER_HOST = PropertyUtils.getString(Constants.MAIL_SERVER_HOST);
public static final Integer mailServerPort = PropertyUtils.getInt(Constants.MAIL_SERVER_PORT);
public static final Integer MAIL_SERVER_PORT = PropertyUtils.getInt(Constants.MAIL_SERVER_PORT);
public static final String mailSender = PropertyUtils.getString(Constants.MAIL_SENDER);
public static final String MAIL_SENDER = PropertyUtils.getString(Constants.MAIL_SENDER);
public static final String mailUser = PropertyUtils.getString(Constants.MAIL_USER);
public static final String MAIL_USER = PropertyUtils.getString(Constants.MAIL_USER);
public static final String mailPasswd = PropertyUtils.getString(Constants.MAIL_PASSWD);
public static final String MAIL_PASSWD = PropertyUtils.getString(Constants.MAIL_PASSWD);
public static final Boolean mailUseStartTLS = PropertyUtils.getBoolean(Constants.MAIL_SMTP_STARTTLS_ENABLE);
public static final Boolean MAIL_USE_START_TLS = PropertyUtils.getBoolean(Constants.MAIL_SMTP_STARTTLS_ENABLE);
public static final Boolean mailUseSSL = PropertyUtils.getBoolean(Constants.MAIL_SMTP_SSL_ENABLE);
public static final Boolean MAIL_USE_SSL = PropertyUtils.getBoolean(Constants.MAIL_SMTP_SSL_ENABLE);
public static final String xlsFilePath = PropertyUtils.getString(Constants.XLS_FILE_PATH);
public static final String XLS_FILE_PATH = PropertyUtils.getString(Constants.XLS_FILE_PATH);
public static final String starttlsEnable = PropertyUtils.getString(Constants.MAIL_SMTP_STARTTLS_ENABLE);
public static final String STARTTLS_ENABLE = PropertyUtils.getString(Constants.MAIL_SMTP_STARTTLS_ENABLE);
public static final String sslEnable = PropertyUtils.getString(Constants.MAIL_SMTP_SSL_ENABLE);
public static final String SSL_ENABLE = PropertyUtils.getString(Constants.MAIL_SMTP_SSL_ENABLE);
public static final String sslTrust = PropertyUtils.getString(Constants.MAIL_SMTP_SSL_TRUST);
public static final String SSL_TRUST = PropertyUtils.getString(Constants.MAIL_SMTP_SSL_TRUST);
public static final AlertTemplate alertTemplate = AlertTemplateFactory.getMessageTemplate();
@ -105,7 +105,7 @@ public class MailUtils {
try {
Session session = getSession();
email.setMailSession(session);
email.setFrom(mailSender);
email.setFrom(MAIL_SENDER);
email.setCharset(Constants.UTF_8);
if (CollectionUtils.isNotEmpty(receivers)){
// receivers mail
@ -199,10 +199,10 @@ public class MailUtils {
// 2. creating mail: Creating a MimeMessage
MimeMessage msg = new MimeMessage(session);
// 3. set sender
msg.setFrom(new InternetAddress(mailSender));
msg.setFrom(new InternetAddress(MAIL_SENDER));
// 4. set receivers
for (String receiver : receivers) {
msg.addRecipients(MimeMessage.RecipientType.TO, InternetAddress.parse(receiver));
msg.addRecipients(Message.RecipientType.TO, InternetAddress.parse(receiver));
}
return msg;
}
@ -213,19 +213,19 @@ public class MailUtils {
*/
private static Session getSession() {
Properties props = new Properties();
props.setProperty(Constants.MAIL_HOST, mailServerHost);
props.setProperty(Constants.MAIL_PORT, String.valueOf(mailServerPort));
props.setProperty(Constants.MAIL_HOST, MAIL_SERVER_HOST);
props.setProperty(Constants.MAIL_PORT, String.valueOf(MAIL_SERVER_PORT));
props.setProperty(Constants.MAIL_SMTP_AUTH, Constants.STRING_TRUE);
props.setProperty(Constants.MAIL_TRANSPORT_PROTOCOL, mailProtocol);
props.setProperty(Constants.MAIL_SMTP_STARTTLS_ENABLE, starttlsEnable);
props.setProperty(Constants.MAIL_SMTP_SSL_ENABLE, sslEnable);
props.setProperty(Constants.MAIL_SMTP_SSL_TRUST, sslTrust);
props.setProperty(Constants.MAIL_TRANSPORT_PROTOCOL, MAIL_PROTOCOL);
props.setProperty(Constants.MAIL_SMTP_STARTTLS_ENABLE, STARTTLS_ENABLE);
props.setProperty(Constants.MAIL_SMTP_SSL_ENABLE, SSL_ENABLE);
props.setProperty(Constants.MAIL_SMTP_SSL_TRUST, SSL_TRUST);
Authenticator auth = new Authenticator() {
@Override
protected PasswordAuthentication getPasswordAuthentication() {
// mail username and password
return new PasswordAuthentication(mailUser, mailPasswd);
return new PasswordAuthentication(MAIL_USER, MAIL_PASSWD);
}
};
@ -248,12 +248,10 @@ public class MailUtils {
*/
if(CollectionUtils.isNotEmpty(receiversCc)){
for (String receiverCc : receiversCc){
msg.addRecipients(MimeMessage.RecipientType.CC, InternetAddress.parse(receiverCc));
msg.addRecipients(Message.RecipientType.CC, InternetAddress.parse(receiverCc));
}
}
// set receivers type to cc
// msg.addRecipients(MimeMessage.RecipientType.CC, InternetAddress.parse(propMap.get("${CC}")));
// set subject
msg.setSubject(title);
MimeMultipart partList = new MimeMultipart();
@ -263,8 +261,8 @@ public class MailUtils {
// set attach file
MimeBodyPart part2 = new MimeBodyPart();
// make excel file
ExcelUtils.genExcelFile(content,title,xlsFilePath);
File file = new File(xlsFilePath + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS);
ExcelUtils.genExcelFile(content,title, XLS_FILE_PATH);
File file = new File(XLS_FILE_PATH + Constants.SINGLE_SLASH + title + Constants.EXCEL_SUFFIX_XLS);
part2.attachFile(file);
part2.setFileName(MimeUtility.encodeText(title + Constants.EXCEL_SUFFIX_XLS,Constants.UTF_8,"B"));
// add components to collection
@ -334,7 +332,7 @@ public class MailUtils {
* @param e the exception
*/
private static void handleException(Collection<String> receivers, Map<String, Object> retMap, Exception e) {
logger.error("Send email to {} failed {}", receivers, e);
logger.error("Send email to {} failed", receivers, e);
retMap.put(Constants.MESSAGE, "Send email to {" + String.join(",", receivers) + "} failed," + e.toString());
}

1
dolphinscheduler-alert/src/main/resources/alert.properties

@ -28,7 +28,6 @@ mail.server.port=25
mail.sender=xxx@xxx.com
mail.user=xxx@xxx.com
mail.passwd=111111
# TLS
mail.smtp.starttls.enable=true
# SSL

8
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java

@ -53,7 +53,7 @@ public class EnterpriseWeChatUtilsTest {
String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token);
String errmsg = JSON.parseObject(resp).getString("errmsg");
Assert.assertEquals(errmsg, "ok");
Assert.assertEquals("ok",errmsg);
} catch (IOException e) {
e.printStackTrace();
}
@ -68,7 +68,7 @@ public class EnterpriseWeChatUtilsTest {
String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token);
String errmsg = JSON.parseObject(resp).getString("errmsg");
Assert.assertEquals(errmsg, "ok");
Assert.assertEquals("ok",errmsg);
} catch (IOException e) {
e.printStackTrace();
}
@ -95,7 +95,7 @@ public class EnterpriseWeChatUtilsTest {
String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token);
String errmsg = JSON.parseObject(resp).getString("errmsg");
Assert.assertEquals(errmsg, "ok");
Assert.assertEquals("ok",errmsg);
} catch (IOException e) {
e.printStackTrace();
}
@ -110,7 +110,7 @@ public class EnterpriseWeChatUtilsTest {
String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token);
String errmsg = JSON.parseObject(resp).getString("errmsg");
Assert.assertEquals(errmsg, "ok");
Assert.assertEquals("ok",errmsg);
} catch (IOException e) {
e.printStackTrace();
}

2
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java

@ -46,7 +46,7 @@ public class FuncUtilsTest {
logger.info(result);
//Expected result string
assertEquals(result, "user1|user2|user3");
assertEquals("user1|user2|user3", result);
//Null list expected return null
result = FuncUtils.mkString(null, split);

28
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java

@ -17,7 +17,6 @@
package org.apache.dolphinscheduler.alert.utils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
@ -27,8 +26,7 @@ import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.*;
public class JSONUtilsTest {
@ -74,7 +72,7 @@ public class JSONUtilsTest {
result = JSONUtils.toJsonString(null);
logger.info(result);
assertEquals(result,"null");
assertEquals("null", result);
}
@ -87,25 +85,27 @@ public class JSONUtilsTest {
//Invoke toList
List<LinkedHashMap> result = JSONUtils.toList(expected ,LinkedHashMap.class);
//Equal list size=1
assertEquals(result.size(),1);
assertEquals(1,result.size());
//Transform entity to LinkedHashMap<String, Object>
LinkedHashMap<String, Object> entity = result.get(0);
//Equal expected values
assertEquals(entity.get("mysql service name"),"mysql200");
assertEquals(entity.get("mysql address"),"192.168.xx.xx");
assertEquals(entity.get("port"),"3306");
assertEquals(entity.get("no index of number"),"80");
assertEquals(entity.get("database client connections"),"190");
assertEquals("mysql200",entity.get("mysql service name"));
assertEquals("192.168.xx.xx", entity.get("mysql address"));
assertEquals("3306", entity.get("port"));
assertEquals("80", entity.get("no index of number"));
assertEquals("190", entity.get("database client connections"));
//If param is null, then return null
//If param is null, then return empty list
result = JSONUtils.toList(null ,LinkedHashMap.class);
assertNull(result);
assertNotNull(result);
assertTrue(result.isEmpty());
//If param is incorrect, then return null and log error message
//If param is incorrect, then return empty list and log error message
result = JSONUtils.toList("}{" ,LinkedHashMap.class);
assertNull(result);
assertNotNull(result);
assertTrue(result.isEmpty());
}

6
dolphinscheduler-api/pom.xml

@ -140,6 +140,12 @@
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- hadoop -->

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java

@ -25,7 +25,9 @@ import springfox.documentation.swagger2.annotations.EnableSwagger2;
@SpringBootApplication
@ServletComponentScan
@ComponentScan("org.apache.dolphinscheduler")
@ComponentScan({"org.apache.dolphinscheduler.api",
"org.apache.dolphinscheduler.dao",
"org.apache.dolphinscheduler.service"})
public class ApiApplicationServer extends SpringBootServletInitializer {
public static void main(String[] args) {

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java

@ -103,7 +103,7 @@ public class DataAnalysisController extends BaseController{
@RequestParam(value="endDate", required=false) String endDate,
@RequestParam(value="projectId", required=false, defaultValue = "0") int projectId){
try{
logger.info("count process instance state, user:{}, start date: {}, end date:{}, project id",
logger.info("count process instance state, user:{}, start date: {}, end date:{}, project id:{}",
loginUser.getUserName(), startDate, endDate, projectId);
Map<String, Object> result = dataAnalysisService.countProcessInstanceStateByProject(loginUser, projectId, startDate, endDate);
return returnDataList(result);
@ -129,7 +129,7 @@ public class DataAnalysisController extends BaseController{
public Result countDefinitionByUser(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value="projectId", required=false, defaultValue = "0") int projectId){
try{
logger.info("count process definition , user:{}, project id",
logger.info("count process definition , user:{}, project id:{}",
loginUser.getUserName(), projectId);
Map<String, Object> result = dataAnalysisService.countDefinitionByUser(loginUser, projectId);
return returnDataList(result);

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java

@ -149,7 +149,7 @@ public class ExecutorController extends BaseController {
) {
try {
logger.info("execute command, login user: {}, project:{}, process instance id:{}, execute type:{}",
loginUser.getUserName(), projectName, processInstanceId, executeType.toString());
loginUser.getUserName(), projectName, processInstanceId, executeType);
Map<String, Object> result = execService.execute(loginUser, projectName, processInstanceId, executeType);
return returnDataList(result);
} catch (Exception e) {
@ -173,7 +173,7 @@ public class ExecutorController extends BaseController {
@ResponseStatus(HttpStatus.OK)
public Result startCheckProcessDefinition(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "processDefinitionId") int processDefinitionId) {
logger.info("login user {}, check process definition", loginUser.getUserName(), processDefinitionId);
logger.info("login user {}, check process definition {}", loginUser.getUserName(), processDefinitionId);
try {
Map<String, Object> result = execService.startCheckByProcessDefinedId(processDefinitionId);
return returnDataList(result);

11
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java

@ -27,7 +27,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.servlet.HandlerInterceptor;
import org.springframework.web.servlet.ModelAndView;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@ -90,14 +89,4 @@ public class LoginHandlerInterceptor implements HandlerInterceptor {
return true;
}
@Override
public void postHandle(HttpServletRequest request, HttpServletResponse response, Object handler, ModelAndView modelAndView) throws Exception {
}
@Override
public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) throws Exception {
}
}

25
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java

@ -16,6 +16,7 @@
*/
package org.apache.dolphinscheduler.api.service;
import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
@ -303,7 +304,7 @@ public class DataSourceService extends BaseService{
for (DataSource dataSource : dataSourceList) {
String connectionParams = dataSource.getConnectionParams();
JSONObject object = JSONObject.parseObject(connectionParams);
JSONObject object = JSON.parseObject(connectionParams);
object.put(Constants.PASSWORD, Constants.XXXXXX);
dataSource.setConnectionParams(JSONUtils.toJson(object));
@ -367,11 +368,11 @@ public class DataSourceService extends BaseService{
try {
switch (dbType) {
case POSTGRESQL:
datasource = JSONObject.parseObject(parameter, PostgreDataSource.class);
datasource = JSON.parseObject(parameter, PostgreDataSource.class);
Class.forName(Constants.ORG_POSTGRESQL_DRIVER);
break;
case MYSQL:
datasource = JSONObject.parseObject(parameter, MySQLDataSource.class);
datasource = JSON.parseObject(parameter, MySQLDataSource.class);
Class.forName(Constants.COM_MYSQL_JDBC_DRIVER);
break;
case HIVE:
@ -386,26 +387,26 @@ public class DataSourceService extends BaseService{
getString(org.apache.dolphinscheduler.common.Constants.LOGIN_USER_KEY_TAB_PATH));
}
if (dbType == DbType.HIVE){
datasource = JSONObject.parseObject(parameter, HiveDataSource.class);
datasource = JSON.parseObject(parameter, HiveDataSource.class);
}else if (dbType == DbType.SPARK){
datasource = JSONObject.parseObject(parameter, SparkDataSource.class);
datasource = JSON.parseObject(parameter, SparkDataSource.class);
}
Class.forName(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER);
break;
case CLICKHOUSE:
datasource = JSONObject.parseObject(parameter, ClickHouseDataSource.class);
datasource = JSON.parseObject(parameter, ClickHouseDataSource.class);
Class.forName(Constants.COM_CLICKHOUSE_JDBC_DRIVER);
break;
case ORACLE:
datasource = JSONObject.parseObject(parameter, OracleDataSource.class);
datasource = JSON.parseObject(parameter, OracleDataSource.class);
Class.forName(Constants.COM_ORACLE_JDBC_DRIVER);
break;
case SQLSERVER:
datasource = JSONObject.parseObject(parameter, SQLServerDataSource.class);
datasource = JSON.parseObject(parameter, SQLServerDataSource.class);
Class.forName(Constants.COM_SQLSERVER_JDBC_DRIVER);
break;
case DB2:
datasource = JSONObject.parseObject(parameter, DB2ServerDataSource.class);
datasource = JSON.parseObject(parameter, DB2ServerDataSource.class);
Class.forName(Constants.COM_DB2_JDBC_DRIVER);
break;
default:
@ -507,7 +508,7 @@ public class DataSourceService extends BaseService{
parameterMap.put(Constants.PRINCIPAL,principal);
}
if (other != null && !"".equals(other)) {
LinkedHashMap<String, String> map = JSONObject.parseObject(other, new TypeReference<LinkedHashMap<String, String>>() {
LinkedHashMap<String, String> map = JSON.parseObject(other, new TypeReference<LinkedHashMap<String, String>>() {
});
if (map.size() > 0) {
StringBuilder otherSb = new StringBuilder();
@ -523,9 +524,9 @@ public class DataSourceService extends BaseService{
}
if(logger.isDebugEnabled()){
logger.info("parameters map-----" + JSONObject.toJSONString(parameterMap));
logger.info("parameters map-----" + JSON.toJSONString(parameterMap));
}
return JSONObject.toJSONString(parameterMap);
return JSON.toJSONString(parameterMap);
}

15
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java

@ -259,10 +259,7 @@ public class ExecutorService extends BaseService{
// checkTenantExists();
Tenant tenant = processService.getTenantForProcess(processDefinition.getTenantId(),
processDefinition.getUserId());
if(tenant == null){
return false;
}
return true;
return tenant != null;
}
/**
@ -298,6 +295,7 @@ public class ExecutorService extends BaseService{
if (executionStatus.typeIsPause()|| executionStatus.typeIsCancel()) {
checkResult = true;
}
break;
default:
break;
}
@ -369,7 +367,7 @@ public class ExecutorService extends BaseService{
* @return check result code
*/
public Map<String, Object> startCheckByProcessDefinedId(int processDefineId) {
Map<String, Object> result = new HashMap<String, Object>();
Map<String, Object> result = new HashMap<>();
if (processDefineId == 0){
logger.error("process definition id is null");
@ -378,10 +376,9 @@ public class ExecutorService extends BaseService{
List<Integer> ids = new ArrayList<>();
processService.recurseFindSubProcessId(processDefineId, ids);
Integer[] idArray = ids.toArray(new Integer[ids.size()]);
if (ids.size() > 0){
List<ProcessDefinition> processDefinitionList;
processDefinitionList = processDefinitionMapper.queryDefinitionListByIdList(idArray);
if (processDefinitionList != null && processDefinitionList.size() > 0){
if (!ids.isEmpty()){
List<ProcessDefinition> processDefinitionList = processDefinitionMapper.queryDefinitionListByIdList(idArray);
if (processDefinitionList != null){
for (ProcessDefinition processDefinition : processDefinitionList){
/**
* if there is no online process, exit directly

1
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java

@ -28,7 +28,6 @@ import org.apache.dolphinscheduler.dao.entity.ZookeeperRecord;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java

@ -16,6 +16,7 @@
*/
package org.apache.dolphinscheduler.api.service;
import java.nio.charset.StandardCharsets;
import org.apache.dolphinscheduler.api.dto.gantt.GanttDto;
import org.apache.dolphinscheduler.api.dto.gantt.Task;
import org.apache.dolphinscheduler.api.enums.Status;
@ -49,7 +50,6 @@ import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.text.ParseException;
import java.util.*;
import java.util.stream.Collectors;
@ -273,7 +273,8 @@ public class ProcessInstanceService extends BaseDAGService {
return resultMap;
}
BufferedReader br = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(log.getBytes(Charset.forName("utf8"))), Charset.forName("utf8")));
BufferedReader br = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(log.getBytes(
StandardCharsets.UTF_8)), StandardCharsets.UTF_8));
String line;
while ((line = br.readLine()) != null) {
if(line.contains(DEPENDENT_SPLIT)){

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java

@ -121,7 +121,7 @@ public class ZooKeeperState {
private class SendThread extends Thread {
private String cmd;
public String ret = "";
private String ret = "";
public SendThread(String cmd) {
this.cmd = cmd;

1
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java

@ -28,7 +28,6 @@ import org.springframework.test.web.servlet.MvcResult;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;

1
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java

@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.controller;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import com.alibaba.fastjson.JSONObject;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java

@ -29,8 +29,6 @@ import org.springframework.test.web.servlet.MvcResult;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import javax.ws.rs.POST;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;

25
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java

@ -16,15 +16,14 @@
*/
package org.apache.dolphinscheduler.api.controller;
import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import com.alibaba.fastjson.JSONObject;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -56,7 +55,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString());
JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
@ -80,7 +79,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString());
JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
@ -283,7 +282,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString());
JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
@ -305,7 +304,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString());
JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
@ -326,7 +325,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString());
JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
@ -346,7 +345,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString());
JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
@ -367,7 +366,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString());
JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
@ -388,7 +387,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString());
JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
@ -408,7 +407,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString());
JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
@ -429,7 +428,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString());
JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
@ -448,7 +447,7 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSONObject.parse(mvcResult.getResponse().getContentAsString());
JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());

5
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java

@ -18,9 +18,6 @@ package org.apache.dolphinscheduler.api.controller;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.enums.FailureStrategy;
import org.apache.dolphinscheduler.common.enums.Priority;
import org.apache.dolphinscheduler.common.enums.WarningType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.junit.Assert;
import org.junit.Test;
@ -31,9 +28,7 @@ import org.springframework.test.web.servlet.MvcResult;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import static org.junit.Assert.*;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;

1
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java

@ -29,7 +29,6 @@ import org.springframework.test.web.servlet.MvcResult;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import static org.junit.Assert.*;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;

8
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java

@ -175,6 +175,14 @@ public class TenantServiceTest {
logger.info(result.toString());
List<Tenant> tenantList = (List<Tenant>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(tenantList));
Mockito.when( tenantMapper.queryByTenantCode("1")).thenReturn(getList());
Map<String, Object> successRes = tenantService.queryTenantList("1");
Assert.assertEquals(Status.SUCCESS,successRes.get(Constants.STATUS));
Mockito.when( tenantMapper.queryByTenantCode("1")).thenReturn(null);
Map<String, Object> tenantNotExistRes = tenantService.queryTenantList("1");
Assert.assertEquals(Status.TENANT_NOT_EXIST,tenantNotExistRes.get(Constants.STATUS));
}
@Test

1
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMainTest.java

@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.utils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitorUtilsTest.java

@ -28,7 +28,7 @@ public class ZookeeperMonitorUtilsTest {
@Test
public void testGetMasterLsit(){
public void testGetMasterList(){
ZookeeperMonitor zookeeperMonitor = new ZookeeperMonitor();

2
dolphinscheduler-common/pom.xml

@ -25,7 +25,7 @@
</parent>
<artifactId>dolphinscheduler-common</artifactId>
<name>dolphinscheduler-common</name>
<url>http://maven.apache.org</url>
<packaging>jar</packaging>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

7
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@ -746,7 +746,7 @@ public final class Constants {
* application regex
*/
public static final String APPLICATION_REGEX = "application_\\d+_\\d+";
public static final String PID = "pid";
public static final String PID = OSUtils.isWindows() ? "handle" : "pid";
/**
* month_begin
*/
@ -813,6 +813,11 @@ public final class Constants {
*/
public static final String KERBEROS = "kerberos";
/**
* kerberos expire time
*/
public static final String KERBEROS_EXPIRE_TIME = "kerberos.expire.time";
/**
* java.security.krb5.conf
*/

3
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/WorkerLogFilter.java

@ -20,9 +20,6 @@ import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.filter.Filter;
import ch.qos.logback.core.spi.FilterReply;
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import java.util.Arrays;
/**
* worker log filter

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java

@ -16,6 +16,7 @@
*/
package org.apache.dolphinscheduler.common.model;
import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.Priority;
import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy;
@ -23,7 +24,6 @@ import org.apache.dolphinscheduler.common.enums.TaskType;
import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
@ -294,7 +294,7 @@ public class TaskNode {
if(StringUtils.isNotEmpty(this.getTimeout())){
String formatStr = String.format("%s,%s", TaskTimeoutStrategy.WARN.name(), TaskTimeoutStrategy.FAILED.name());
String timeout = this.getTimeout().replace(formatStr,TaskTimeoutStrategy.WARNFAILED.name());
return JSONObject.parseObject(timeout,TaskTimeoutParameter.class);
return JSON.parseObject(timeout,TaskTimeoutParameter.class);
}
return new TaskTimeoutParameter(false);
}

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java

@ -335,7 +335,7 @@ public abstract class AbstractShell {
try{
entry.getValue().destroy();
} catch (Exception e) {
e.printStackTrace();
logger.error("Destroy All Processes error", e);
}
}

14
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java

@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
@ -207,12 +208,15 @@ public class FlinkParameters extends AbstractParameters {
@Override
public List<String> getResourceFilesList() {
if(resourceList !=null ) {
this.resourceList.add(mainJar);
return resourceList.stream()
.map(p -> p.getRes()).collect(Collectors.toList());
if(resourceList != null ) {
List<String> resourceFiles = resourceList.stream()
.map(ResourceInfo::getRes).collect(Collectors.toList());
if(mainJar != null) {
resourceFiles.add(mainJar.getRes());
}
return resourceFiles;
}
return null;
return Collections.emptyList();
}

14
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/mr/MapreduceParameters.java

@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
@ -125,12 +126,15 @@ public class MapreduceParameters extends AbstractParameters {
@Override
public List<String> getResourceFilesList() {
if (resourceList != null) {
this.resourceList.add(mainJar);
return resourceList.stream()
.map(p -> p.getRes()).collect(Collectors.toList());
if(resourceList != null ) {
List<String> resourceFiles = resourceList.stream()
.map(ResourceInfo::getRes).collect(Collectors.toList());
if(mainJar != null) {
resourceFiles.add(mainJar.getRes());
}
return resourceFiles;
}
return null;
return Collections.emptyList();
}
@Override

17
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java

@ -120,12 +120,24 @@ public class ThreadUtils {
/**
* Wrapper over ScheduledThreadPoolExecutor
* @param threadName
* @param corePoolSize
* @return
*/
public static ScheduledExecutorService newDaemonThreadScheduledExecutor(String threadName,int corePoolSize) {
public static ScheduledExecutorService newDaemonThreadScheduledExecutor(String threadName, int corePoolSize) {
return newThreadScheduledExecutor(threadName, corePoolSize, true);
}
/**
* Wrapper over ScheduledThreadPoolExecutor
* @param threadName
* @param corePoolSize
* @param isDaemon
* @return
*/
public static ScheduledExecutorService newThreadScheduledExecutor(String threadName, int corePoolSize, boolean isDaemon) {
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setDaemon(true)
.setDaemon(isDaemon)
.setNameFormat(threadName)
.build();
ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(corePoolSize, threadFactory);
@ -135,7 +147,6 @@ public class ThreadUtils {
return executor;
}
public static ThreadInfo getThreadInfo(Thread t) {
long tid = t.getId();
return threadBean.getThreadInfo(tid, STACK_DEPTH);

6
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java

@ -44,7 +44,7 @@ public class FileUtils {
String fileSuffix = "";
if (StringUtils.isNotEmpty(filename)) {
int lastIndex = filename.lastIndexOf(".");
int lastIndex = filename.lastIndexOf('.');
if (lastIndex > 0) {
fileSuffix = filename.substring(lastIndex + 1);
}
@ -325,10 +325,8 @@ public class FileUtils {
}
} else {
File parent = file.getParentFile();
if (parent != null) {
if (!parent.mkdirs() && !parent.isDirectory()) {
if (parent != null && !parent.mkdirs() && !parent.isDirectory()) {
throw new IOException("Directory '" + parent + "' could not be created");
}
}
}
return new FileOutputStream(file, append);

254
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java

@ -16,6 +16,9 @@
*/
package org.apache.dolphinscheduler.common.utils;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
@ -32,9 +35,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.nio.file.Files;
import java.security.PrivilegedExceptionAction;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -46,32 +52,37 @@ public class HadoopUtils implements Closeable {
private static final Logger logger = LoggerFactory.getLogger(HadoopUtils.class);
private static String hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER);
private static volatile HadoopUtils instance = new HadoopUtils();
private static volatile Configuration configuration;
private static FileSystem fs;
private static final String HADOOP_UTILS_KEY = "HADOOP_UTILS_KEY";
private static final LoadingCache<String, HadoopUtils> cache = CacheBuilder
.newBuilder()
.expireAfterWrite(PropertyUtils.getInt(Constants.KERBEROS_EXPIRE_TIME, 7), TimeUnit.DAYS)
.build(new CacheLoader<String, HadoopUtils>() {
@Override
public HadoopUtils load(String key) throws Exception {
return new HadoopUtils();
}
});
private HadoopUtils(){
if(StringUtils.isEmpty(hdfsUser)){
hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER);
}
private Configuration configuration;
private FileSystem fs;
private static String hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER);
private HadoopUtils() {
init();
initHdfsPath();
}
public static HadoopUtils getInstance(){
// if kerberos startup , renew HadoopUtils
if (CommonUtils.getKerberosStartupState()){
return new HadoopUtils();
}
return instance;
public static HadoopUtils getInstance() {
return cache.getUnchecked(HADOOP_UTILS_KEY);
}
/**
* init dolphinscheduler root path in hdfs
*/
private void initHdfsPath(){
private void initHdfsPath() {
String hdfsPath = PropertyUtils.getString(Constants.DATA_STORE_2_HDFS_BASEPATH);
Path path = new Path(hdfsPath);
@ -80,7 +91,7 @@ public class HadoopUtils implements Closeable {
fs.mkdirs(path);
}
} catch (Exception e) {
logger.error(e.getMessage(),e);
logger.error(e.getMessage(), e);
}
}
@ -89,82 +100,74 @@ public class HadoopUtils implements Closeable {
* init hadoop configuration
*/
private void init() {
if (configuration == null) {
synchronized (HadoopUtils.class) {
if (configuration == null) {
try {
configuration = new Configuration();
String resUploadStartupType = PropertyUtils.getString(Constants.RES_UPLOAD_STARTUP_TYPE);
ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType);
if (resUploadType == ResUploadType.HDFS){
if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE)){
System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF,
PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH));
configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION,"kerberos");
UserGroupInformation.setConfiguration(configuration);
UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH));
}
try {
configuration = new Configuration();
String resUploadStartupType = PropertyUtils.getString(Constants.RES_UPLOAD_STARTUP_TYPE);
ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType);
if (resUploadType == ResUploadType.HDFS) {
if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE)) {
System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF,
PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH));
configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
UserGroupInformation.setConfiguration(configuration);
UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH));
}
String defaultFS = configuration.get(Constants.FS_DEFAULTFS);
//first get key from core-site.xml hdfs-site.xml ,if null ,then try to get from properties file
// the default is the local file system
if(defaultFS.startsWith("file")){
String defaultFSProp = PropertyUtils.getString(Constants.FS_DEFAULTFS);
if(StringUtils.isNotBlank(defaultFSProp)){
Map<String, String> fsRelatedProps = PropertyUtils.getPrefixedProperties("fs.");
configuration.set(Constants.FS_DEFAULTFS,defaultFSProp);
fsRelatedProps.forEach((key, value) -> configuration.set(key, value));
}else{
logger.error("property:{} can not to be empty, please set!", Constants.FS_DEFAULTFS );
throw new RuntimeException(
String.format("property: %s can not to be empty, please set!", Constants.FS_DEFAULTFS)
);
}
}else{
logger.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", Constants.FS_DEFAULTFS, defaultFS);
}
String defaultFS = configuration.get(Constants.FS_DEFAULTFS);
//first get key from core-site.xml hdfs-site.xml ,if null ,then try to get from properties file
// the default is the local file system
if (defaultFS.startsWith("file")) {
String defaultFSProp = PropertyUtils.getString(Constants.FS_DEFAULTFS);
if (StringUtils.isNotBlank(defaultFSProp)) {
Map<String, String> fsRelatedProps = PropertyUtils.getPrefixedProperties("fs.");
configuration.set(Constants.FS_DEFAULTFS, defaultFSProp);
fsRelatedProps.forEach((key, value) -> configuration.set(key, value));
} else {
logger.error("property:{} can not to be empty, please set!", Constants.FS_DEFAULTFS);
throw new RuntimeException(
String.format("property: %s can not to be empty, please set!", Constants.FS_DEFAULTFS)
);
}
} else {
logger.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", Constants.FS_DEFAULTFS, defaultFS);
}
if (fs == null) {
if(StringUtils.isNotEmpty(hdfsUser)){
//UserGroupInformation ugi = UserGroupInformation.createProxyUser(hdfsUser,UserGroupInformation.getLoginUser());
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser);
ugi.doAs(new PrivilegedExceptionAction<Boolean>() {
@Override
public Boolean run() throws Exception {
fs = FileSystem.get(configuration);
return true;
}
});
}else{
logger.warn("hdfs.root.user is not set value!");
fs = FileSystem.get(configuration);
}
if (fs == null) {
if (StringUtils.isNotEmpty(hdfsUser)) {
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser);
ugi.doAs(new PrivilegedExceptionAction<Boolean>() {
@Override
public Boolean run() throws Exception {
fs = FileSystem.get(configuration);
return true;
}
}else if (resUploadType == ResUploadType.S3){
configuration.set(Constants.FS_DEFAULTFS, PropertyUtils.getString(Constants.FS_DEFAULTFS));
configuration.set(Constants.FS_S3A_ENDPOINT, PropertyUtils.getString(Constants.FS_S3A_ENDPOINT));
configuration.set(Constants.FS_S3A_ACCESS_KEY, PropertyUtils.getString(Constants.FS_S3A_ACCESS_KEY));
configuration.set(Constants.FS_S3A_SECRET_KEY, PropertyUtils.getString(Constants.FS_S3A_SECRET_KEY));
fs = FileSystem.get(configuration);
}
String rmHaIds = PropertyUtils.getString(Constants.YARN_RESOURCEMANAGER_HA_RM_IDS);
String appAddress = PropertyUtils.getString(Constants.YARN_APPLICATION_STATUS_ADDRESS);
if (!StringUtils.isEmpty(rmHaIds)) {
appAddress = getAppAddress(appAddress, rmHaIds);
logger.info("appAddress : {}", appAddress);
}
configuration.set(Constants.YARN_APPLICATION_STATUS_ADDRESS, appAddress);
} catch (Exception e) {
logger.error(e.getMessage(), e);
});
} else {
logger.warn("hdfs.root.user is not set value!");
fs = FileSystem.get(configuration);
}
}
} else if (resUploadType == ResUploadType.S3) {
configuration.set(Constants.FS_DEFAULTFS, PropertyUtils.getString(Constants.FS_DEFAULTFS));
configuration.set(Constants.FS_S3A_ENDPOINT, PropertyUtils.getString(Constants.FS_S3A_ENDPOINT));
configuration.set(Constants.FS_S3A_ACCESS_KEY, PropertyUtils.getString(Constants.FS_S3A_ACCESS_KEY));
configuration.set(Constants.FS_S3A_SECRET_KEY, PropertyUtils.getString(Constants.FS_S3A_SECRET_KEY));
fs = FileSystem.get(configuration);
}
String rmHaIds = PropertyUtils.getString(Constants.YARN_RESOURCEMANAGER_HA_RM_IDS);
String appAddress = PropertyUtils.getString(Constants.YARN_APPLICATION_STATUS_ADDRESS);
if (!StringUtils.isEmpty(rmHaIds)) {
appAddress = getAppAddress(appAddress, rmHaIds);
logger.info("appAddress : {}", appAddress);
}
configuration.set(Constants.YARN_APPLICATION_STATUS_ADDRESS, appAddress);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
@ -188,15 +191,15 @@ public class HadoopUtils implements Closeable {
/**
* cat file on hdfs
*
* @param hdfsFilePath hdfs file path
* @param hdfsFilePath hdfs file path
* @return byte[] byte array
* @throws IOException errors
*/
public byte[] catFile(String hdfsFilePath) throws IOException {
if(StringUtils.isBlank(hdfsFilePath)){
logger.error("hdfs file path:{} is blank",hdfsFilePath);
return null;
if (StringUtils.isBlank(hdfsFilePath)) {
logger.error("hdfs file path:{} is blank", hdfsFilePath);
return new byte[0];
}
FSDataInputStream fsDataInputStream = fs.open(new Path(hdfsFilePath));
@ -204,29 +207,28 @@ public class HadoopUtils implements Closeable {
}
/**
* cat file on hdfs
*
* @param hdfsFilePath hdfs file path
* @param skipLineNums skip line numbers
* @param limit read how many lines
* @param hdfsFilePath hdfs file path
* @param skipLineNums skip line numbers
* @param limit read how many lines
* @return content of file
* @throws IOException errors
*/
public List<String> catFile(String hdfsFilePath, int skipLineNums, int limit) throws IOException {
if (StringUtils.isBlank(hdfsFilePath)){
logger.error("hdfs file path:{} is blank",hdfsFilePath);
return null;
if (StringUtils.isBlank(hdfsFilePath)) {
logger.error("hdfs file path:{} is blank", hdfsFilePath);
return Collections.emptyList();
}
try (FSDataInputStream in = fs.open(new Path(hdfsFilePath))){
try (FSDataInputStream in = fs.open(new Path(hdfsFilePath))) {
BufferedReader br = new BufferedReader(new InputStreamReader(in));
Stream<String> stream = br.lines().skip(skipLineNums).limit(limit);
return stream.collect(Collectors.toList());
}
}
/**
@ -259,17 +261,17 @@ public class HadoopUtils implements Closeable {
/**
* the src file is on the local disk. Add it to FS at
* the given dst name.
* @param srcFile local file
* @param dstHdfsPath destination hdfs path
* @param deleteSource whether to delete the src
* @param overwrite whether to overwrite an existing file
*
* @param srcFile local file
* @param dstHdfsPath destination hdfs path
* @param deleteSource whether to delete the src
* @param overwrite whether to overwrite an existing file
* @return if success or not
* @throws IOException errors
*/
public boolean copyLocalToHdfs(String srcFile, String dstHdfsPath, boolean deleteSource, boolean overwrite) throws IOException {
Path srcPath = new Path(srcFile);
Path dstPath= new Path(dstHdfsPath);
Path dstPath = new Path(dstHdfsPath);
fs.copyFromLocalFile(deleteSource, overwrite, srcPath, dstPath);
@ -279,10 +281,10 @@ public class HadoopUtils implements Closeable {
/**
* copy hdfs file to local
*
* @param srcHdfsFilePath source hdfs file path
* @param dstFile destination file
* @param deleteSource delete source
* @param overwrite overwrite
* @param srcHdfsFilePath source hdfs file path
* @param dstFile destination file
* @param deleteSource delete source
* @param overwrite overwrite
* @return result of copy hdfs file to local
* @throws IOException errors
*/
@ -293,14 +295,14 @@ public class HadoopUtils implements Closeable {
if (dstPath.exists()) {
if (dstPath.isFile()) {
if (overwrite) {
dstPath.delete();
Files.delete(dstPath.toPath());
}
} else {
logger.error("destination file must be a file");
}
}
if(!dstPath.getParentFile().exists()){
if (!dstPath.getParentFile().exists()) {
dstPath.getParentFile().mkdirs();
}
@ -308,14 +310,13 @@ public class HadoopUtils implements Closeable {
}
/**
*
* delete a file
*
* @param hdfsFilePath the path to delete.
* @param recursive if path is a directory and set to
* true, the directory is deleted else throws an exception. In
* case of a file the recursive can be set to either true or false.
* @return true if delete is successful else false.
* @param recursive if path is a directory and set to
* true, the directory is deleted else throws an exception. In
* case of a file the recursive can be set to either true or false.
* @return true if delete is successful else false.
* @throws IOException errors
*/
public boolean delete(String hdfsFilePath, boolean recursive) throws IOException {
@ -340,7 +341,7 @@ public class HadoopUtils implements Closeable {
* @return {@link FileStatus} file status
* @throws Exception errors
*/
public FileStatus[] listFileStatus(String filePath)throws Exception{
public FileStatus[] listFileStatus(String filePath) throws Exception {
try {
return fs.listStatus(new Path(filePath));
} catch (IOException e) {
@ -352,10 +353,11 @@ public class HadoopUtils implements Closeable {
/**
* Renames Path src to Path dst. Can take place on local fs
* or remote DFS.
*
* @param src path to be renamed
* @param dst new path after rename
* @throws IOException on failure
* @return true if rename is successful
* @throws IOException on failure
*/
public boolean rename(String src, String dst) throws IOException {
return fs.rename(new Path(src), new Path(dst));
@ -378,7 +380,7 @@ public class HadoopUtils implements Closeable {
String responseContent = HttpUtils.get(applicationUrl);
JSONObject jsonObject = JSONObject.parseObject(responseContent);
JSONObject jsonObject = JSON.parseObject(responseContent);
String result = jsonObject.getJSONObject("app").getString("finalStatus");
switch (result) {
@ -401,7 +403,6 @@ public class HadoopUtils implements Closeable {
}
/**
*
* @return data hdfs path
*/
public static String getHdfsDataBasePath() {
@ -428,11 +429,11 @@ public class HadoopUtils implements Closeable {
* hdfs user dir
*
* @param tenantCode tenant code
* @param userId user id
* @param userId user id
* @return hdfs resource dir
*/
public static String getHdfsUserDir(String tenantCode,int userId) {
return String.format("%s/home/%d", getHdfsTenantDir(tenantCode),userId);
public static String getHdfsUserDir(String tenantCode, int userId) {
return String.format("%s/home/%d", getHdfsTenantDir(tenantCode), userId);
}
/**
@ -480,7 +481,7 @@ public class HadoopUtils implements Closeable {
* getAppAddress
*
* @param appAddress app address
* @param rmHa resource manager ha
* @param rmHa resource manager ha
* @return app address
*/
public static String getAppAddress(String appAddress, String rmHa) {
@ -525,8 +526,6 @@ public class HadoopUtils implements Closeable {
*/
private static final class YarnHAAdminUtils extends RMAdminCLI {
private static final Logger logger = LoggerFactory.getLogger(YarnHAAdminUtils.class);
/**
* get active resourcemanager
*
@ -585,8 +584,7 @@ public class HadoopUtils implements Closeable {
JSONObject jsonObject = JSON.parseObject(retStr);
//get ResourceManager state
String state = jsonObject.getJSONObject("clusterInfo").getString("haState");
return state;
return jsonObject.getJSONObject("clusterInfo").getString("haState");
}
}

12
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java

@ -81,17 +81,15 @@ public class HttpUtils {
logger.error(e.getMessage(),e);
}
if (httpget != null && !httpget.isAborted()) {
if (!httpget.isAborted()) {
httpget.releaseConnection();
httpget.abort();
}
if (httpclient != null) {
try {
httpclient.close();
} catch (IOException e) {
logger.error(e.getMessage(),e);
}
try {
httpclient.close();
} catch (IOException e) {
logger.error(e.getMessage(),e);
}
}
return responseContent;

19
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java

@ -19,26 +19,17 @@
package org.apache.dolphinscheduler.common.utils;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
public class IOUtils {
public static void closeQuietly(InputStream fis){
if(fis != null){
public static void closeQuietly(Closeable closeable){
if(closeable != null){
try {
fis.close();
} catch (IOException ignore) {
}
}
}
public static void closeQuietly(InputStreamReader reader){
if(reader != null){
try {
reader.close();
closeable.close();
} catch (IOException ignore) {
// nothing need to do
}
}
}

5
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java

@ -17,16 +17,11 @@
package org.apache.dolphinscheduler.common.utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* http utils
*/
public class IpUtils {
private static final Logger logger = LoggerFactory.getLogger(IpUtils.class);
public static final String DOT = ".";
/**

25
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java

@ -16,6 +16,7 @@
*/
package org.apache.dolphinscheduler.common.utils;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.TypeReference;
@ -41,12 +42,6 @@ public class JSONUtils {
*/
private static final ObjectMapper objectMapper = new ObjectMapper();
/**
* init
*/
private static final JSONUtils instance = new JSONUtils();
private JSONUtils() {
//Feature that determines whether encountering of unknown properties, false means not analyzer unknown properties
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false).setTimeZone(TimeZone.getDefault());
@ -59,7 +54,7 @@ public class JSONUtils {
*/
public static String toJson(Object object) {
try{
return JSONObject.toJSONString(object,false);
return JSON.toJSONString(object,false);
} catch (Exception e) {
logger.error("object to json exception!",e);
}
@ -89,7 +84,7 @@ public class JSONUtils {
}
try {
return JSONObject.parseObject(json, clazz);
return JSON.parseObject(json, clazz);
} catch (Exception e) {
logger.error("parse object exception!",e);
}
@ -178,7 +173,7 @@ public class JSONUtils {
}
try {
return JSONObject.parseObject(json, new TypeReference<HashMap<String, String>>(){});
return JSON.parseObject(json, new TypeReference<HashMap<String, String>>(){});
} catch (Exception e) {
logger.error("json to map exception!",e);
}
@ -203,7 +198,7 @@ public class JSONUtils {
}
try {
return JSONObject.parseObject(json, new TypeReference<HashMap<K, V>>() {});
return JSON.parseObject(json, new TypeReference<HashMap<K, V>>() {});
} catch (Exception e) {
logger.error("json to map exception!",e);
}
@ -218,23 +213,23 @@ public class JSONUtils {
*/
public static String toJsonString(Object object) {
try{
return JSONObject.toJSONString(object,false);
return JSON.toJSONString(object,false);
} catch (Exception e) {
throw new RuntimeException("Json deserialization exception.", e);
throw new RuntimeException("Object json deserialization exception.", e);
}
}
public static JSONObject parseObject(String text) {
try{
return JSONObject.parseObject(text);
return JSON.parseObject(text);
} catch (Exception e) {
throw new RuntimeException("Json deserialization exception.", e);
throw new RuntimeException("String json deserialization exception.", e);
}
}
public static JSONArray parseArray(String text) {
try{
return JSONObject.parseArray(text);
return JSON.parseArray(text);
} catch (Exception e) {
throw new RuntimeException("Json deserialization exception.", e);
}

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java

@ -79,7 +79,7 @@ public class LoggerUtils {
*/
public static List<String> getAppIds(String log, Logger logger) {
List<String> appIds = new ArrayList<String>();
List<String> appIds = new ArrayList<>();
Matcher matcher = APPLICATION_REGEX.matcher(log);

16
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java

@ -400,8 +400,7 @@ public class OSUtils {
* @return true if mac
*/
public static boolean isMacOS() {
String os = System.getProperty("os.name");
return os.startsWith("Mac");
return getOSName().startsWith("Mac");
}
@ -409,9 +408,16 @@ public class OSUtils {
* whether is windows
* @return true if windows
*/
public static boolean isWindows() {
String os = System.getProperty("os.name");
return os.startsWith("Windows");
public static boolean isWindows() { ;
return getOSName().startsWith("Windows");
}
/**
* get current OS name
* @return current OS name
*/
public static String getOSName() {
return System.getProperty("os.name");
}
/**

38
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java

@ -16,6 +16,7 @@
*/
package org.apache.dolphinscheduler.common.utils;
import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.DataType;
@ -23,7 +24,6 @@ import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils;
import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils;
import org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils;
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.time.DateUtils;
import org.slf4j.Logger;
@ -119,10 +119,15 @@ public class ParameterUtils {
*/
public static String curingGlobalParams(Map<String,String> globalParamMap, List<Property> globalParamList,
CommandType commandType, Date scheduleTime){
Map<String, String> globalMap = new HashMap<>();
if(globalParamMap!= null){
globalMap.putAll(globalParamMap);
}
if (globalParamList == null || globalParamList.isEmpty()) {
return null;
}
Map<String, String> globalMap = new HashMap<>();
if (globalParamMap!= null){
globalMap.putAll(globalParamMap);
}
Map<String,String> allParamMap = new HashMap<>();
//If it is a complement, a complement time needs to be passed in, according to the task type
Map<String,String> timeParams = BusinessTimeUtils
@ -132,9 +137,7 @@ public class ParameterUtils {
allParamMap.putAll(timeParams);
}
if (globalMap != null) {
allParamMap.putAll(globalMap);
}
allParamMap.putAll(globalMap);
Set<Map.Entry<String, String>> entries = allParamMap.entrySet();
@ -146,22 +149,15 @@ public class ParameterUtils {
resolveMap.put(entry.getKey(),str);
}
}
globalMap.putAll(resolveMap);
if (globalMap != null){
globalMap.putAll(resolveMap);
}
if (globalParamList != null && globalParamList.size() > 0){
for (Property property : globalParamList){
String val = globalMap.get(property.getProp());
if (val != null){
property.setValue(val);
}
for (Property property : globalParamList){
String val = globalMap.get(property.getProp());
if (val != null){
property.setValue(val);
}
return JSONObject.toJSONString(globalParamList);
}
return null;
return JSON.toJSONString(globalParamList);
}

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java

@ -43,13 +43,11 @@ public class PropertyUtils {
private static final Properties properties = new Properties();
private static final PropertyUtils propertyUtils = new PropertyUtils();
private PropertyUtils(){
init();
private PropertyUtils() {
throw new IllegalStateException("PropertyUtils class");
}
private void init(){
static {
String[] propertyFiles = new String[]{COMMON_PROPERTIES_PATH};
for (String fileName : propertyFiles) {
InputStream fis = null;
@ -125,7 +123,7 @@ public class PropertyUtils {
* @param key property name
* @return property value
*/
public static Boolean getBoolean(String key) {
public static boolean getBoolean(String key) {
String value = properties.getProperty(key.trim());
if(null != value){
return Boolean.parseBoolean(value);

6
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/PlaceholderUtils.java

@ -31,12 +31,12 @@ public class PlaceholderUtils {
/**
* Prefix of the position to be replaced
*/
public static final String placeholderPrefix = "${";
public static final String PLACEHOLDER_PREFIX = "${";
/**
* The suffix of the position to be replaced
*/
public static final String placeholderSuffix = "}";
public static final String PLACEHOLDER_SUFFIX = "}";
/**
@ -68,7 +68,7 @@ public class PlaceholderUtils {
*/
public static PropertyPlaceholderHelper getPropertyPlaceholderHelper(boolean ignoreUnresolvablePlaceholders) {
return new PropertyPlaceholderHelper(placeholderPrefix, placeholderSuffix, null, ignoreUnresolvablePlaceholders);
return new PropertyPlaceholderHelper(PLACEHOLDER_PREFIX, PLACEHOLDER_SUFFIX, null, ignoreUnresolvablePlaceholders);
}
/**

1071
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32.java

File diff suppressed because it is too large Load Diff

286
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32.java

@ -0,0 +1,286 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils.process;
import com.sun.jna.platform.win32.Kernel32Util;
import java.util.*;
final class ProcessEnvironmentForWin32 extends HashMap<String,String> {
private static final long serialVersionUID = -8017839552603542824L;
private static String validateName(String name) {
// An initial `=' indicates a magic Windows variable name -- OK
if (name.indexOf('=', 1) != -1 ||
name.indexOf('\u0000') != -1)
throw new IllegalArgumentException
("Invalid environment variable name: \"" + name + "\"");
return name;
}
private static String validateValue(String value) {
if (value.indexOf('\u0000') != -1)
throw new IllegalArgumentException
("Invalid environment variable value: \"" + value + "\"");
return value;
}
private static String nonNullString(Object o) {
if (o == null)
throw new NullPointerException();
return (String) o;
}
public String put(String key, String value) {
return super.put(validateName(key), validateValue(value));
}
public String get(Object key) {
return super.get(nonNullString(key));
}
public boolean containsKey(Object key) {
return super.containsKey(nonNullString(key));
}
public boolean containsValue(Object value) {
return super.containsValue(nonNullString(value));
}
public String remove(Object key) {
return super.remove(nonNullString(key));
}
private static class CheckedEntry implements Entry<String,String> {
private final Entry<String,String> e;
public CheckedEntry(Entry<String,String> e) {this.e = e;}
public String getKey() { return e.getKey();}
public String getValue() { return e.getValue();}
public String setValue(String value) {
return e.setValue(validateValue(value));
}
public String toString() { return getKey() + "=" + getValue();}
public boolean equals(Object o) {return e.equals(o);}
public int hashCode() {return e.hashCode();}
}
private static class CheckedEntrySet extends AbstractSet<Entry<String,String>> {
private final Set<Entry<String,String>> s;
public CheckedEntrySet(Set<Entry<String,String>> s) {this.s = s;}
public int size() {return s.size();}
public boolean isEmpty() {return s.isEmpty();}
public void clear() { s.clear();}
public Iterator<Entry<String,String>> iterator() {
return new Iterator<Entry<String,String>>() {
Iterator<Entry<String,String>> i = s.iterator();
public boolean hasNext() { return i.hasNext();}
public Entry<String,String> next() {
return new CheckedEntry(i.next());
}
public void remove() { i.remove();}
};
}
private static Entry<String,String> checkedEntry(Object o) {
@SuppressWarnings("unchecked")
Entry<String,String> e = (Entry<String,String>) o;
nonNullString(e.getKey());
nonNullString(e.getValue());
return e;
}
public boolean contains(Object o) {return s.contains(checkedEntry(o));}
public boolean remove(Object o) {return s.remove(checkedEntry(o));}
}
private static class CheckedValues extends AbstractCollection<String> {
private final Collection<String> c;
public CheckedValues(Collection<String> c) {this.c = c;}
public int size() {return c.size();}
public boolean isEmpty() {return c.isEmpty();}
public void clear() { c.clear();}
public Iterator<String> iterator() {return c.iterator();}
public boolean contains(Object o) {return c.contains(nonNullString(o));}
public boolean remove(Object o) {return c.remove(nonNullString(o));}
}
private static class CheckedKeySet extends AbstractSet<String> {
private final Set<String> s;
public CheckedKeySet(Set<String> s) {this.s = s;}
public int size() {return s.size();}
public boolean isEmpty() {return s.isEmpty();}
public void clear() { s.clear();}
public Iterator<String> iterator() {return s.iterator();}
public boolean contains(Object o) {return s.contains(nonNullString(o));}
public boolean remove(Object o) {return s.remove(nonNullString(o));}
}
public Set<String> keySet() {
return new CheckedKeySet(super.keySet());
}
public Collection<String> values() {
return new CheckedValues(super.values());
}
public Set<Entry<String,String>> entrySet() {
return new CheckedEntrySet(super.entrySet());
}
private static final class NameComparator implements Comparator<String> {
public int compare(String s1, String s2) {
// We can't use String.compareToIgnoreCase since it
// canonicalizes to lower case, while Windows
// canonicalizes to upper case! For example, "_" should
// sort *after* "Z", not before.
int n1 = s1.length();
int n2 = s2.length();
int min = Math.min(n1, n2);
for (int i = 0; i < min; i++) {
char c1 = s1.charAt(i);
char c2 = s2.charAt(i);
if (c1 != c2) {
c1 = Character.toUpperCase(c1);
c2 = Character.toUpperCase(c2);
if (c1 != c2)
// No overflow because of numeric promotion
return c1 - c2;
}
}
return n1 - n2;
}
}
private static final class EntryComparator implements Comparator<Entry<String,String>> {
public int compare(Entry<String,String> e1,
Entry<String,String> e2) {
return nameComparator.compare(e1.getKey(), e2.getKey());
}
}
// Allow `=' as first char in name, e.g. =C:=C:\DIR
static final int MIN_NAME_LENGTH = 1;
private static final NameComparator nameComparator;
private static final EntryComparator entryComparator;
private static final ProcessEnvironmentForWin32 theEnvironment;
private static final Map<String,String> theUnmodifiableEnvironment;
private static final Map<String,String> theCaseInsensitiveEnvironment;
static {
nameComparator = new NameComparator();
entryComparator = new EntryComparator();
theEnvironment = new ProcessEnvironmentForWin32();
theUnmodifiableEnvironment = Collections.unmodifiableMap(theEnvironment);
theEnvironment.putAll(environmentBlock());
theCaseInsensitiveEnvironment = new TreeMap<>(nameComparator);
theCaseInsensitiveEnvironment.putAll(theEnvironment);
}
private ProcessEnvironmentForWin32() {
super();
}
private ProcessEnvironmentForWin32(int capacity) {
super(capacity);
}
// Only for use by System.getenv(String)
static String getenv(String name) {
// The original implementation used a native call to _wgetenv,
// but it turns out that _wgetenv is only consistent with
// GetEnvironmentStringsW (for non-ASCII) if `wmain' is used
// instead of `main', even in a process created using
// CREATE_UNICODE_ENVIRONMENT. Instead we perform the
// case-insensitive comparison ourselves. At least this
// guarantees that System.getenv().get(String) will be
// consistent with System.getenv(String).
return theCaseInsensitiveEnvironment.get(name);
}
// Only for use by System.getenv()
static Map<String,String> getenv() {
return theUnmodifiableEnvironment;
}
// Only for use by ProcessBuilder.environment()
@SuppressWarnings("unchecked")
static Map<String,String> environment() {
return (Map<String,String>) theEnvironment.clone();
}
// Only for use by ProcessBuilder.environment(String[] envp)
static Map<String,String> emptyEnvironment(int capacity) {
return new ProcessEnvironmentForWin32(capacity);
}
private static Map<String, String> environmentBlock() {
return Kernel32Util.getEnvironmentVariables();
}
// Only for use by ProcessImpl.start()
String toEnvironmentBlock() {
// Sort Unicode-case-insensitively by name
List<Entry<String,String>> list = new ArrayList<>(entrySet());
Collections.sort(list, entryComparator);
StringBuilder sb = new StringBuilder(size()*30);
int cmp = -1;
// Some versions of MSVCRT.DLL require SystemRoot to be set.
// So, we make sure that it is always set, even if not provided
// by the caller.
final String SYSTEMROOT = "SystemRoot";
for (Entry<String,String> e : list) {
String key = e.getKey();
String value = e.getValue();
if (cmp < 0 && (cmp = nameComparator.compare(key, SYSTEMROOT)) > 0) {
// Not set, so add it here
addToEnvIfSet(sb, SYSTEMROOT);
}
addToEnv(sb, key, value);
}
if (cmp < 0) {
// Got to end of list and still not found
addToEnvIfSet(sb, SYSTEMROOT);
}
if (sb.length() == 0) {
// Environment was empty and SystemRoot not set in parent
sb.append('\u0000');
}
// Block is double NUL terminated
sb.append('\u0000');
return sb.toString();
}
// add the environment variable to the child, if it exists in parent
private static void addToEnvIfSet(StringBuilder sb, String name) {
String s = getenv(name);
if (s != null)
addToEnv(sb, name, s);
}
private static void addToEnv(StringBuilder sb, String name, String val) {
sb.append(name).append('=').append(val).append('\u0000');
}
static String toEnvironmentBlock(Map<String,String> map) {
return map == null ? null : ((ProcessEnvironmentForWin32)map).toEnvironmentBlock();
}
}

785
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java

@ -0,0 +1,785 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils.process;
import com.sun.jna.Pointer;
import com.sun.jna.platform.win32.*;
import com.sun.jna.ptr.IntByReference;
import java.lang.reflect.Field;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import sun.security.action.GetPropertyAction;
import java.io.*;
import java.security.AccessController;
import java.security.PrivilegedAction;
import java.util.ArrayList;
import java.util.Locale;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static com.sun.jna.platform.win32.WinBase.STILL_ACTIVE;
import static java.util.Objects.requireNonNull;
public class ProcessImplForWin32 extends Process {
private static final Field FD_HANDLE;
static {
if (!OSUtils.isWindows()) {
throw new RuntimeException("ProcessImplForWin32 can be only initialized in " +
"Windows environment, but current OS is " + OSUtils.getOSName());
}
try {
FD_HANDLE = requireNonNull(FileDescriptor.class.getDeclaredField("handle"));
FD_HANDLE.setAccessible(true);
} catch (NoSuchFieldException e) {
throw new RuntimeException(e);
}
}
private static final int PIPE_SIZE = 4096 + 24;
private static final int HANDLE_STORAGE_SIZE = 6;
private static final int OFFSET_READ = 0;
private static final int OFFSET_WRITE = 1;
private static final WinNT.HANDLE JAVA_INVALID_HANDLE_VALUE = new WinNT.HANDLE(Pointer.createConstant(-1));
private static void setHandle(FileDescriptor obj, long handle) {
try {
FD_HANDLE.set(obj, handle);
} catch (IllegalAccessException e) {
throw new RuntimeException(e);
}
}
private static long getHandle(FileDescriptor obj) {
try {
return (Long) FD_HANDLE.get(obj);
} catch (IllegalAccessException e) {
throw new RuntimeException(e);
}
}
/**
* Open a file for writing. If {@code append} is {@code true} then the file
* is opened for atomic append directly and a FileOutputStream constructed
* with the resulting handle. This is because a FileOutputStream created
* to append to a file does not open the file in a manner that guarantees
* that writes by the child process will be atomic.
*/
private static FileOutputStream newFileOutputStream(File f, boolean append)
throws IOException
{
if (append) {
String path = f.getPath();
SecurityManager sm = System.getSecurityManager();
if (sm != null)
sm.checkWrite(path);
long handle = openForAtomicAppend(path);
final FileDescriptor fd = new FileDescriptor();
setHandle(fd, handle);
return AccessController.doPrivileged(
new PrivilegedAction<FileOutputStream>() {
public FileOutputStream run() {
return new FileOutputStream(fd);
}
}
);
} else {
return new FileOutputStream(f);
}
}
// System-dependent portion of ProcessBuilderForWindows.start()
static Process start(String username,
String password,
String cmdarray[],
java.util.Map<String,String> environment,
String dir,
ProcessBuilderForWin32.Redirect[] redirects,
boolean redirectErrorStream)
throws IOException
{
String envblock = ProcessEnvironmentForWin32.toEnvironmentBlock(environment);
FileInputStream f0 = null;
FileOutputStream f1 = null;
FileOutputStream f2 = null;
try {
long[] stdHandles;
if (redirects == null) {
stdHandles = new long[] { -1L, -1L, -1L };
} else {
stdHandles = new long[3];
if (redirects[0] == ProcessBuilderForWin32.Redirect.PIPE)
stdHandles[0] = -1L;
else if (redirects[0] == ProcessBuilderForWin32.Redirect.INHERIT)
stdHandles[0] = getHandle(FileDescriptor.in);
else {
f0 = new FileInputStream(redirects[0].file());
stdHandles[0] = getHandle(f0.getFD());
}
if (redirects[1] == ProcessBuilderForWin32.Redirect.PIPE)
stdHandles[1] = -1L;
else if (redirects[1] == ProcessBuilderForWin32.Redirect.INHERIT)
stdHandles[1] = getHandle(FileDescriptor.out);
else {
f1 = newFileOutputStream(redirects[1].file(),
redirects[1].append());
stdHandles[1] = getHandle(f1.getFD());
}
if (redirects[2] == ProcessBuilderForWin32.Redirect.PIPE)
stdHandles[2] = -1L;
else if (redirects[2] == ProcessBuilderForWin32.Redirect.INHERIT)
stdHandles[2] = getHandle(FileDescriptor.err);
else {
f2 = newFileOutputStream(redirects[2].file(),
redirects[2].append());
stdHandles[2] = getHandle(f2.getFD());
}
}
return new ProcessImplForWin32(username, password, cmdarray, envblock, dir, stdHandles, redirectErrorStream);
} finally {
// In theory, close() can throw IOException
// (although it is rather unlikely to happen here)
try { if (f0 != null) f0.close(); }
finally {
try { if (f1 != null) f1.close(); }
finally { if (f2 != null) f2.close(); }
}
}
}
private static class LazyPattern {
// Escape-support version:
// "(\")((?:\\\\\\1|.)+?)\\1|([^\\s\"]+)";
private static final Pattern PATTERN =
Pattern.compile("[^\\s\"]+|\"[^\"]*\"");
};
/* Parses the command string parameter into the executable name and
* program arguments.
*
* The command string is broken into tokens. The token separator is a space
* or quota character. The space inside quotation is not a token separator.
* There are no escape sequences.
*/
private static String[] getTokensFromCommand(String command) {
ArrayList<String> matchList = new ArrayList<>(8);
Matcher regexMatcher = ProcessImplForWin32.LazyPattern.PATTERN.matcher(command);
while (regexMatcher.find())
matchList.add(regexMatcher.group());
return matchList.toArray(new String[matchList.size()]);
}
private static final int VERIFICATION_CMD_BAT = 0;
private static final int VERIFICATION_WIN32 = 1;
private static final int VERIFICATION_WIN32_SAFE = 2; // inside quotes not allowed
private static final int VERIFICATION_LEGACY = 3;
// See Command shell overview for documentation of special characters.
// https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-xp/bb490954(v=technet.10)
private static final char ESCAPE_VERIFICATION[][] = {
// We guarantee the only command file execution for implicit [cmd.exe] run.
// http://technet.microsoft.com/en-us/library/bb490954.aspx
{' ', '\t', '<', '>', '&', '|', '^'},
{' ', '\t', '<', '>'},
{' ', '\t', '<', '>'},
{' ', '\t'}
};
private static String createCommandLine(int verificationType,
final String executablePath,
final String cmd[])
{
StringBuilder cmdbuf = new StringBuilder(80);
cmdbuf.append(executablePath);
for (int i = 1; i < cmd.length; ++i) {
cmdbuf.append(' ');
String s = cmd[i];
if (needsEscaping(verificationType, s)) {
cmdbuf.append('"');
if (verificationType == VERIFICATION_WIN32_SAFE) {
// Insert the argument, adding '\' to quote any interior quotes
int length = s.length();
for (int j = 0; j < length; j++) {
char c = s.charAt(j);
if (c == DOUBLEQUOTE) {
int count = countLeadingBackslash(verificationType, s, j);
while (count-- > 0) {
cmdbuf.append(BACKSLASH); // double the number of backslashes
}
cmdbuf.append(BACKSLASH); // backslash to quote the quote
}
cmdbuf.append(c);
}
} else {
cmdbuf.append(s);
}
// The code protects the [java.exe] and console command line
// parser, that interprets the [\"] combination as an escape
// sequence for the ["] char.
// http://msdn.microsoft.com/en-us/library/17w5ykft.aspx
//
// If the argument is an FS path, doubling of the tail [\]
// char is not a problem for non-console applications.
//
// The [\"] sequence is not an escape sequence for the [cmd.exe]
// command line parser. The case of the [""] tail escape
// sequence could not be realized due to the argument validation
// procedure.
int count = countLeadingBackslash(verificationType, s, s.length());
while (count-- > 0) {
cmdbuf.append(BACKSLASH); // double the number of backslashes
}
cmdbuf.append('"');
} else {
cmdbuf.append(s);
}
}
return cmdbuf.toString();
}
/**
* Return the argument without quotes (1st and last) if present, else the arg.
* @param str a string
* @return the string without 1st and last quotes
*/
private static String unQuote(String str) {
int len = str.length();
return (len >= 2 && str.charAt(0) == DOUBLEQUOTE && str.charAt(len - 1) == DOUBLEQUOTE)
? str.substring(1, len - 1)
: str;
}
private static boolean needsEscaping(int verificationType, String arg) {
// Switch off MS heuristic for internal ["].
// Please, use the explicit [cmd.exe] call
// if you need the internal ["].
// Example: "cmd.exe", "/C", "Extended_MS_Syntax"
// For [.exe] or [.com] file the unpaired/internal ["]
// in the argument is not a problem.
String unquotedArg = unQuote(arg);
boolean argIsQuoted = !arg.equals(unquotedArg);
boolean embeddedQuote = unquotedArg.indexOf(DOUBLEQUOTE) >= 0;
switch (verificationType) {
case VERIFICATION_CMD_BAT:
if (embeddedQuote) {
throw new IllegalArgumentException("Argument has embedded quote, " +
"use the explicit CMD.EXE call.");
}
break; // break determine whether to quote
case VERIFICATION_WIN32_SAFE:
if (argIsQuoted && embeddedQuote) {
throw new IllegalArgumentException("Malformed argument has embedded quote: "
+ unquotedArg);
}
break;
default:
break;
}
if (!argIsQuoted) {
char testEscape[] = ESCAPE_VERIFICATION[verificationType];
for (int i = 0; i < testEscape.length; ++i) {
if (arg.indexOf(testEscape[i]) >= 0) {
return true;
}
}
}
return false;
}
private static String getExecutablePath(String path)
throws IOException
{
String name = unQuote(path);
if (name.indexOf(DOUBLEQUOTE) >= 0) {
throw new IllegalArgumentException("Executable name has embedded quote, " +
"split the arguments: " + name);
}
// Win32 CreateProcess requires path to be normalized
File fileToRun = new File(name);
// From the [CreateProcess] function documentation:
//
// "If the file name does not contain an extension, .exe is appended.
// Therefore, if the file name extension is .com, this parameter
// must include the .com extension. If the file name ends in
// a period (.) with no extension, or if the file name contains a path,
// .exe is not appended."
//
// "If the file name !does not contain a directory path!,
// the system searches for the executable file in the following
// sequence:..."
//
// In practice ANY non-existent path is extended by [.exe] extension
// in the [CreateProcess] function with the only exception:
// the path ends by (.)
return fileToRun.getPath();
}
/**
* An executable is any program that is an EXE or does not have an extension
* and the Windows createProcess will be looking for .exe.
* The comparison is case insensitive based on the name.
* @param executablePath the executable file
* @return true if the path ends in .exe or does not have an extension.
*/
private boolean isExe(String executablePath) {
File file = new File(executablePath);
String upName = file.getName().toUpperCase(Locale.ROOT);
return (upName.endsWith(".EXE") || upName.indexOf('.') < 0);
}
// Old version that can be bypassed
private boolean isShellFile(String executablePath) {
String upPath = executablePath.toUpperCase();
return (upPath.endsWith(".CMD") || upPath.endsWith(".BAT"));
}
private String quoteString(String arg) {
StringBuilder argbuf = new StringBuilder(arg.length() + 2);
return argbuf.append('"').append(arg).append('"').toString();
}
// Count backslashes before start index of string.
// .bat files don't include backslashes as part of the quote
private static int countLeadingBackslash(int verificationType,
CharSequence input, int start) {
if (verificationType == VERIFICATION_CMD_BAT)
return 0;
int j;
for (j = start - 1; j >= 0 && input.charAt(j) == BACKSLASH; j--) {
// just scanning backwards
}
return (start - 1) - j; // number of BACKSLASHES
}
private static final char DOUBLEQUOTE = '\"';
private static final char BACKSLASH = '\\';
private WinNT.HANDLE handle;
private OutputStream stdin_stream;
private InputStream stdout_stream;
private InputStream stderr_stream;
private ProcessImplForWin32(
String username,
String password,
String cmd[],
final String envblock,
final String path,
final long[] stdHandles,
final boolean redirectErrorStream)
throws IOException
{
String cmdstr;
final SecurityManager security = System.getSecurityManager();
GetPropertyAction action = new GetPropertyAction("jdk.lang.Process.allowAmbiguousCommands",
(security == null) ? "true" : "false");
final boolean allowAmbiguousCommands = !"false".equalsIgnoreCase(action.run());
if (allowAmbiguousCommands && security == null) {
// Legacy mode.
// Normalize path if possible.
String executablePath = new File(cmd[0]).getPath();
// No worry about internal, unpaired ["], and redirection/piping.
if (needsEscaping(VERIFICATION_LEGACY, executablePath) )
executablePath = quoteString(executablePath);
cmdstr = createCommandLine(
//legacy mode doesn't worry about extended verification
VERIFICATION_LEGACY,
executablePath,
cmd);
} else {
String executablePath;
try {
executablePath = getExecutablePath(cmd[0]);
} catch (IllegalArgumentException e) {
// Workaround for the calls like
// Runtime.getRuntime().exec("\"C:\\Program Files\\foo\" bar")
// No chance to avoid CMD/BAT injection, except to do the work
// right from the beginning. Otherwise we have too many corner
// cases from
// Runtime.getRuntime().exec(String[] cmd [, ...])
// calls with internal ["] and escape sequences.
// Restore original command line.
StringBuilder join = new StringBuilder();
// terminal space in command line is ok
for (String s : cmd)
join.append(s).append(' ');
// Parse the command line again.
cmd = getTokensFromCommand(join.toString());
executablePath = getExecutablePath(cmd[0]);
// Check new executable name once more
if (security != null)
security.checkExec(executablePath);
}
// Quotation protects from interpretation of the [path] argument as
// start of longer path with spaces. Quotation has no influence to
// [.exe] extension heuristic.
boolean isShell = allowAmbiguousCommands ? isShellFile(executablePath)
: !isExe(executablePath);
cmdstr = createCommandLine(
// We need the extended verification procedures
isShell ? VERIFICATION_CMD_BAT
: (allowAmbiguousCommands ? VERIFICATION_WIN32 : VERIFICATION_WIN32_SAFE),
quoteString(executablePath),
cmd);
}
handle = create(username, password, cmdstr, envblock, path, stdHandles, redirectErrorStream);
AccessController.doPrivileged(
new PrivilegedAction<Void>() {
public Void run() {
if (stdHandles[0] == -1L)
stdin_stream = ProcessBuilderForWin32.NullOutputStream.INSTANCE;
else {
FileDescriptor stdin_fd = new FileDescriptor();
setHandle(stdin_fd, stdHandles[0]);
stdin_stream = new BufferedOutputStream(
new FileOutputStream(stdin_fd));
}
if (stdHandles[1] == -1L)
stdout_stream = ProcessBuilderForWin32.NullInputStream.INSTANCE;
else {
FileDescriptor stdout_fd = new FileDescriptor();
setHandle(stdout_fd, stdHandles[1]);
stdout_stream = new BufferedInputStream(
new FileInputStream(stdout_fd));
}
if (stdHandles[2] == -1L)
stderr_stream = ProcessBuilderForWin32.NullInputStream.INSTANCE;
else {
FileDescriptor stderr_fd = new FileDescriptor();
setHandle(stderr_fd, stdHandles[2]);
stderr_stream = new FileInputStream(stderr_fd);
}
return null; }});
}
public OutputStream getOutputStream() {
return stdin_stream;
}
public InputStream getInputStream() {
return stdout_stream;
}
public InputStream getErrorStream() {
return stderr_stream;
}
protected void finalize() {
closeHandle(handle);
}
public int exitValue() {
int exitCode = getExitCodeProcess(handle);
if (exitCode == STILL_ACTIVE)
throw new IllegalThreadStateException("process has not exited");
return exitCode;
}
public int waitFor() throws InterruptedException {
waitForInterruptibly(handle);
if (Thread.interrupted())
throw new InterruptedException();
return exitValue();
}
@Override
public boolean waitFor(long timeout, TimeUnit unit)
throws InterruptedException
{
if (getExitCodeProcess(handle) != STILL_ACTIVE) return true;
if (timeout <= 0) return false;
long remainingNanos = unit.toNanos(timeout);
long deadline = System.nanoTime() + remainingNanos ;
do {
// Round up to next millisecond
long msTimeout = TimeUnit.NANOSECONDS.toMillis(remainingNanos + 999_999L);
waitForTimeoutInterruptibly(handle, msTimeout);
if (Thread.interrupted())
throw new InterruptedException();
if (getExitCodeProcess(handle) != STILL_ACTIVE) {
return true;
}
remainingNanos = deadline - System.nanoTime();
} while (remainingNanos > 0);
return (getExitCodeProcess(handle) != STILL_ACTIVE);
}
public void destroy() { terminateProcess(handle); }
public Process destroyForcibly() {
destroy();
return this;
}
public boolean isAlive() {
return isProcessAlive(handle);
}
private static boolean initHolder(WinNT.HANDLEByReference pjhandles,
WinNT.HANDLEByReference[] pipe,
int offset,
WinNT.HANDLEByReference phStd) {
if (!pjhandles.getValue().equals(JAVA_INVALID_HANDLE_VALUE)) {
phStd.setValue(pjhandles.getValue());
pjhandles.setValue(JAVA_INVALID_HANDLE_VALUE);
} else {
if (!Kernel32.INSTANCE.CreatePipe(pipe[0], pipe[1], null, PIPE_SIZE)) {
throw new Win32Exception(Kernel32.INSTANCE.GetLastError());
} else {
WinNT.HANDLE thisProcessEnd = offset == OFFSET_READ ? pipe[1].getValue() : pipe[0].getValue();
phStd.setValue(pipe[offset].getValue());
pjhandles.setValue(thisProcessEnd);
}
}
Kernel32.INSTANCE.SetHandleInformation(phStd.getValue(), Kernel32.HANDLE_FLAG_INHERIT, Kernel32.HANDLE_FLAG_INHERIT);
return true;
}
private static void releaseHolder(boolean complete, WinNT.HANDLEByReference[] pipe, int offset) {
closeHandle(pipe[offset].getValue());
if (complete) {
closeHandle(pipe[offset == OFFSET_READ ? OFFSET_WRITE : OFFSET_READ].getValue());
}
}
private static void prepareIOEHandleState(WinNT.HANDLE[] stdIOE, Boolean[] inherit) {
for(int i = 0; i < HANDLE_STORAGE_SIZE; ++i) {
WinNT.HANDLE hstd = stdIOE[i];
if (!Kernel32.INVALID_HANDLE_VALUE.equals(hstd)) {
inherit[i] = Boolean.TRUE;
Kernel32.INSTANCE.SetHandleInformation(hstd, Kernel32.HANDLE_FLAG_INHERIT, 0);
}
}
}
private static void restoreIOEHandleState(WinNT.HANDLE[] stdIOE, Boolean[] inherit) {
for (int i = HANDLE_STORAGE_SIZE - 1; i >= 0; --i) {
if (!Kernel32.INVALID_HANDLE_VALUE.equals(stdIOE[i])) {
Kernel32.INSTANCE.SetHandleInformation(stdIOE[i], Kernel32.HANDLE_FLAG_INHERIT, inherit[i] ? Kernel32.HANDLE_FLAG_INHERIT : 0);
}
}
}
private static WinNT.HANDLE processCreate(String username,
String password,
String cmd,
final String envblock,
final String path,
final WinNT.HANDLEByReference[] stdHandles,
final boolean redirectErrorStream) {
WinNT.HANDLE ret = new WinNT.HANDLE(Pointer.createConstant(0));
WinNT.HANDLE[] stdIOE = new WinNT.HANDLE[] {
Kernel32.INVALID_HANDLE_VALUE, Kernel32.INVALID_HANDLE_VALUE, Kernel32.INVALID_HANDLE_VALUE,
stdHandles[0].getValue(), stdHandles[1].getValue(), stdHandles[2].getValue()
};
stdIOE[0] = Kernel32.INSTANCE.GetStdHandle(Kernel32.STD_INPUT_HANDLE);
stdIOE[1] = Kernel32.INSTANCE.GetStdHandle(Kernel32.STD_OUTPUT_HANDLE);
stdIOE[2] = Kernel32.INSTANCE.GetStdHandle(Kernel32.STD_ERROR_HANDLE);
Boolean[] inherit = new Boolean[] {
Boolean.FALSE, Boolean.FALSE, Boolean.FALSE,
Boolean.FALSE, Boolean.FALSE, Boolean.FALSE
};
prepareIOEHandleState(stdIOE, inherit);
// input
WinNT.HANDLEByReference hStdInput = new WinNT.HANDLEByReference();
WinNT.HANDLEByReference[] pipeIn = new WinNT.HANDLEByReference[] {
new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE), new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE) };
// output
WinNT.HANDLEByReference hStdOutput = new WinNT.HANDLEByReference();
WinNT.HANDLEByReference[] pipeOut = new WinNT.HANDLEByReference[] {
new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE), new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE) };
// error
WinNT.HANDLEByReference hStdError = new WinNT.HANDLEByReference();
WinNT.HANDLEByReference[] pipeError = new WinNT.HANDLEByReference[] {
new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE), new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE) };
boolean success;
if (initHolder(stdHandles[0], pipeIn, OFFSET_READ, hStdInput)) {
if (initHolder(stdHandles[1], pipeOut, OFFSET_WRITE, hStdOutput)) {
WinBase.STARTUPINFO si = new WinBase.STARTUPINFO();
si.hStdInput = hStdInput.getValue();
si.hStdOutput = hStdOutput.getValue();
if (redirectErrorStream) {
si.hStdError = si.hStdOutput;
stdHandles[2].setValue(JAVA_INVALID_HANDLE_VALUE);
success = true;
} else {
success = initHolder(stdHandles[2], pipeError, OFFSET_WRITE, hStdError);
si.hStdError = hStdError.getValue();
}
if (success) {
WTypes.LPSTR lpEnvironment = envblock == null ? new WTypes.LPSTR() : new WTypes.LPSTR(envblock);
Kernel32.PROCESS_INFORMATION pi = new WinBase.PROCESS_INFORMATION();
si.dwFlags = Kernel32.STARTF_USESTDHANDLES;
if (!Advapi32.INSTANCE.CreateProcessWithLogonW(
username
, null
, password
, Advapi32.LOGON_WITH_PROFILE
, null
, cmd
, Kernel32.CREATE_NO_WINDOW
, lpEnvironment.getPointer()
, path
, si
, pi)) {
throw new Win32Exception(Kernel32.INSTANCE.GetLastError());
} else {
closeHandle(pi.hThread);
ret = pi.hProcess;
}
}
releaseHolder(ret.getPointer().equals(Pointer.createConstant(0)), pipeError, OFFSET_WRITE);
releaseHolder(ret.getPointer().equals(Pointer.createConstant(0)), pipeOut, OFFSET_WRITE);
}
releaseHolder(ret.getPointer().equals(Pointer.createConstant(0)), pipeIn, OFFSET_READ);
}
restoreIOEHandleState(stdIOE, inherit);
return ret;
}
private static synchronized WinNT.HANDLE create(String username,
String password,
String cmd,
final String envblock,
final String path,
final long[] stdHandles,
final boolean redirectErrorStream) {
WinNT.HANDLE ret = new WinNT.HANDLE(Pointer.createConstant(0));
WinNT.HANDLEByReference[] handles = new WinNT.HANDLEByReference[stdHandles.length];
for (int i = 0; i < stdHandles.length; i++) {
handles[i] = new WinNT.HANDLEByReference(new WinNT.HANDLE(Pointer.createConstant(stdHandles[i])));
}
if (cmd != null) {
if (username != null && password != null) {
ret = processCreate(username, password, cmd, envblock, path, handles, redirectErrorStream);
}
}
for (int i = 0; i < stdHandles.length; i++) {
stdHandles[i] = handles[i].getPointer().getLong(0);
}
return ret;
}
private static int getExitCodeProcess(WinNT.HANDLE handle) {
IntByReference exitStatus = new IntByReference();
if (!Kernel32.INSTANCE.GetExitCodeProcess(handle, exitStatus)) {
throw new Win32Exception(Kernel32.INSTANCE.GetLastError());
}
return exitStatus.getValue();
}
private static void terminateProcess(WinNT.HANDLE handle) {
Kernel32.INSTANCE.TerminateProcess(handle, 1);
}
private static boolean isProcessAlive(WinNT.HANDLE handle) {
IntByReference exitStatus = new IntByReference();
Kernel32.INSTANCE.GetExitCodeProcess(handle, exitStatus);
return exitStatus.getValue() == STILL_ACTIVE;
}
private static void closeHandle(WinNT.HANDLE handle) {
Kernel32Util.closeHandle(handle);
}
/**
* Opens a file for atomic append. The file is created if it doesn't
* already exist.
*
* @param path the file to open or create
* @return the native HANDLE
*/
private static long openForAtomicAppend(String path) throws IOException {
int access = Kernel32.GENERIC_READ | Kernel32.GENERIC_WRITE;
int sharing = Kernel32.FILE_SHARE_READ | Kernel32.FILE_SHARE_WRITE;
int disposition = Kernel32.OPEN_ALWAYS;
int flagsAndAttributes = Kernel32.FILE_ATTRIBUTE_NORMAL;
if (path == null || path.isEmpty()) {
return -1;
} else {
WinNT.HANDLE handle = Kernel32.INSTANCE.CreateFile(path, access, sharing, null, disposition, flagsAndAttributes, null);
if (handle == Kernel32.INVALID_HANDLE_VALUE) {
throw new Win32Exception(Kernel32.INSTANCE.GetLastError());
}
return handle.getPointer().getLong(0);
}
}
private static void waitForInterruptibly(WinNT.HANDLE handle) {
int result = Kernel32.INSTANCE.WaitForMultipleObjects(1, new WinNT.HANDLE[]{handle}, false, Kernel32.INFINITE);
if (result == Kernel32.WAIT_FAILED) {
throw new Win32Exception(Kernel32.INSTANCE.GetLastError());
}
}
private static void waitForTimeoutInterruptibly(WinNT.HANDLE handle, long timeout) {
int result = Kernel32.INSTANCE.WaitForMultipleObjects(1, new WinNT.HANDLE[]{handle}, false, (int) timeout);
if (result == Kernel32.WAIT_FAILED) {
throw new Win32Exception(Kernel32.INSTANCE.GetLastError());
}
}
}

2
dolphinscheduler-common/src/main/resources/common.properties

@ -91,4 +91,4 @@ yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
# If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine
yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s
kerberos.expire.time=7

40
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java

@ -0,0 +1,40 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.junit.Assert;
import org.junit.Test;
/**
* Constants Test
*/
public class ConstantsTest {
/**
* Test PID via env
*/
@Test
public void testPID() {
if (OSUtils.isWindows()) {
Assert.assertEquals(Constants.PID, "handle");
} else {
Assert.assertEquals(Constants.PID, "pid");
}
}
}

2
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java

@ -27,8 +27,6 @@ import org.slf4j.Marker;
import java.util.Map;
import static org.junit.Assert.*;
public class TaskLogDiscriminatorTest {
/**

1
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java

@ -21,7 +21,6 @@ import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.classic.spi.IThrowableProxy;
import ch.qos.logback.classic.spi.LoggerContextVO;
import ch.qos.logback.core.spi.FilterReply;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.junit.Assert;
import org.junit.Test;

55
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java

@ -0,0 +1,55 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.task;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.flink.FlinkParameters;
import org.junit.Assert;
import org.junit.Test;
import java.util.LinkedList;
import java.util.List;
public class FlinkParametersTest {
@Test
public void getResourceFilesList() {
FlinkParameters flinkParameters = new FlinkParameters();
Assert.assertNotNull(flinkParameters.getResourceFilesList());
Assert.assertTrue(flinkParameters.getResourceFilesList().isEmpty());
ResourceInfo mainResource = new ResourceInfo();
mainResource.setRes("testFlinkMain-1.0.0-SNAPSHOT.jar");
flinkParameters.setMainJar(mainResource);
List<ResourceInfo> resourceInfos = new LinkedList<>();
ResourceInfo resourceInfo1 = new ResourceInfo();
resourceInfo1.setRes("testFlinkParameters1.jar");
resourceInfos.add(resourceInfo1);
flinkParameters.setResourceList(resourceInfos);
Assert.assertNotNull(flinkParameters.getResourceFilesList());
Assert.assertEquals(2, flinkParameters.getResourceFilesList().size());
ResourceInfo resourceInfo2 = new ResourceInfo();
resourceInfo2.setRes("testFlinkParameters2.jar");
resourceInfos.add(resourceInfo2);
flinkParameters.setResourceList(resourceInfos);
Assert.assertNotNull(flinkParameters.getResourceFilesList());
Assert.assertEquals(3, flinkParameters.getResourceFilesList().size());
}
}

17
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java

@ -30,29 +30,32 @@ public class FileUtilsTest {
@Test
public void suffix() {
Assert.assertEquals(FileUtils.suffix("ninfor.java"),"java");
Assert.assertEquals("java", FileUtils.suffix("ninfor.java"));
Assert.assertEquals("", FileUtils.suffix(null));
Assert.assertEquals("", FileUtils.suffix(""));
Assert.assertEquals("", FileUtils.suffix("ninfor-java"));
}
@Test
public void testGetDownloadFilename() {
PowerMockito.mockStatic(DateUtils.class);
PowerMockito.when(DateUtils.getCurrentTime(YYYYMMDDHHMMSS)).thenReturn("20190101101059");
Assert.assertEquals(FileUtils.getDownloadFilename("test"),
"/tmp/dolphinscheduler/download/20190101101059/test");
Assert.assertEquals("/tmp/dolphinscheduler/download/20190101101059/test",
FileUtils.getDownloadFilename("test"));
}
@Test
public void testGetUploadFilename() {
Assert.assertEquals(FileUtils.getUploadFilename("aaa","bbb"),
"/tmp/dolphinscheduler/aaa/resources/bbb");
Assert.assertEquals("/tmp/dolphinscheduler/aaa/resources/bbb",
FileUtils.getUploadFilename("aaa","bbb"));
}
@Test
public void testGetProcessExecDir() {
String dir = FileUtils.getProcessExecDir(1,2,3, 4);
Assert.assertEquals(dir, "/tmp/dolphinscheduler/exec/process/1/2/3/4");
Assert.assertEquals("/tmp/dolphinscheduler/exec/process/1/2/3/4", dir);
dir = FileUtils.getProcessExecDir(1,2,3);
Assert.assertEquals(dir, "/tmp/dolphinscheduler/exec/process/1/2/3");
Assert.assertEquals("/tmp/dolphinscheduler/exec/process/1/2/3", dir);
}
@Test

3
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java

@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.common.utils;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -39,7 +38,7 @@ public class HttpUtilsTest {
String result = HttpUtils.get("https://github.com/manifest.json");
Assert.assertNotNull(result);
JSONObject jsonObject = JSON.parseObject(result);
Assert.assertEquals(jsonObject.getString("name"), "GitHub");
Assert.assertEquals("GitHub", jsonObject.getString("name"));
result = HttpUtils.get("https://123.333.111.33/ccc");
Assert.assertNull(result);

6
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/IpUtilsTest.java

@ -29,13 +29,13 @@ public class IpUtilsTest {
long longNumber = IpUtils.ipToLong(ip);
long longNumber2 = IpUtils.ipToLong(ip2);
System.out.println(longNumber);
Assert.assertEquals(longNumber, 3232263681L);
Assert.assertEquals(longNumber2, 0L);
Assert.assertEquals(3232263681L, longNumber);
Assert.assertEquals(0L, longNumber2);
String ip3 = "255.255.255.255";
long longNumber3 = IpUtils.ipToLong(ip3);
System.out.println(longNumber3);
Assert.assertEquals(longNumber3, 4294967295L);
Assert.assertEquals(4294967295L, longNumber3);
}

22
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java

@ -16,10 +16,10 @@
*/
package org.apache.dolphinscheduler.common.utils;
import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.common.enums.DataType;
import org.apache.dolphinscheduler.common.enums.Direct;
import org.apache.dolphinscheduler.common.process.Property;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import org.junit.Assert;
@ -40,8 +40,8 @@ public class JSONUtilsTest {
String jsonStr = "{\"id\":\"1001\",\"name\":\"Jobs\"}";
Map<String,String> models = JSONUtils.toMap(jsonStr);
Assert.assertEquals(models.get("id"), "1001");
Assert.assertEquals(models.get("name"), "Jobs");
Assert.assertEquals("1001", models.get("id"));
Assert.assertEquals("Jobs", models.get("name"));
}
@ -53,9 +53,9 @@ public class JSONUtilsTest {
property.setType(DataType.VARCHAR);
property.setValue("sssssss");
String str = "{\"direct\":\"IN\",\"prop\":\"ds\",\"type\":\"VARCHAR\",\"value\":\"sssssss\"}";
Property property1 = JSONObject.parseObject(str, Property.class);
Property property1 = JSON.parseObject(str, Property.class);
Direct direct = property1.getDirect();
Assert.assertEquals(direct , Direct.IN);
Assert.assertEquals(Direct.IN, direct);
}
@ -66,12 +66,12 @@ public class JSONUtilsTest {
List<LinkedHashMap> maps = JSONUtils.toList(str,
LinkedHashMap.class);
Assert.assertEquals(maps.size(), 1);
Assert.assertEquals(maps.get(0).get("mysql service name"), "mysql200");
Assert.assertEquals(maps.get(0).get("mysql address"), "192.168.xx.xx");
Assert.assertEquals(maps.get(0).get("port"), "3306");
Assert.assertEquals(maps.get(0).get("no index of number"), "80");
Assert.assertEquals(maps.get(0).get("database client connections"), "190");
Assert.assertEquals(1, maps.size());
Assert.assertEquals("mysql200", maps.get(0).get("mysql service name"));
Assert.assertEquals("192.168.xx.xx", maps.get(0).get("mysql address"));
Assert.assertEquals("3306", maps.get(0).get("port"));
Assert.assertEquals("80", maps.get(0).get("no index of number"));
Assert.assertEquals("190", maps.get(0).get("database client connections"));
}
public String list2String(){

24
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java

@ -39,16 +39,20 @@ public class OSUtilsTest {
@Test
public void testOSMetric(){
double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize();
Assert.assertTrue(availablePhysicalMemorySize > 0.0f);
double totalMemorySize = OSUtils.totalMemorySize();
Assert.assertTrue(totalMemorySize > 0.0f);
double loadAverage = OSUtils.loadAverage();
logger.info("loadAverage {}", loadAverage);
double memoryUsage = OSUtils.memoryUsage();
Assert.assertTrue(memoryUsage > 0.0f);
double cpuUsage = OSUtils.cpuUsage();
Assert.assertTrue(cpuUsage > 0.0f);
if (!OSUtils.isWindows()) {
double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize();
Assert.assertTrue(availablePhysicalMemorySize > 0.0f);
double totalMemorySize = OSUtils.totalMemorySize();
Assert.assertTrue(totalMemorySize > 0.0f);
double loadAverage = OSUtils.loadAverage();
logger.info("loadAverage {}", loadAverage);
double memoryUsage = OSUtils.memoryUsage();
Assert.assertTrue(memoryUsage > 0.0f);
double cpuUsage = OSUtils.cpuUsage();
Assert.assertTrue(cpuUsage > 0.0f);
} else {
// TODO window ut
}
}
@Test

8
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java

@ -16,7 +16,7 @@
*/
package org.apache.dolphinscheduler.common.utils;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.JSON;
import org.apache.commons.lang.time.DateUtils;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.DataType;
@ -91,13 +91,13 @@ public class ParameterUtilsTest {
globalParamList.add(property);
String result2 = ParameterUtils.curingGlobalParams(null,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,scheduleTime);
Assert.assertEquals(result2, JSONObject.toJSONString(globalParamList));
Assert.assertEquals(result2, JSON.toJSONString(globalParamList));
String result3 = ParameterUtils.curingGlobalParams(globalParamMap,globalParamList,CommandType.START_CURRENT_TASK_PROCESS,null);
Assert.assertEquals(result3, JSONObject.toJSONString(globalParamList));
Assert.assertEquals(result3, JSON.toJSONString(globalParamList));
String result4 = ParameterUtils.curingGlobalParams(globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime);
Assert.assertEquals(result4, JSONObject.toJSONString(globalParamList));
Assert.assertEquals(result4, JSON.toJSONString(globalParamList));
//test var $ startsWith
globalParamMap.put("bizDate","${system.biz.date}");

3
dolphinscheduler-service/src/test/java/utils/PreconditionsTest.java → dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java

@ -14,9 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package utils;
package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.utils.Preconditions;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;

3
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java

@ -19,15 +19,12 @@ package org.apache.dolphinscheduler.common.utils;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileNotFoundException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

2
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java

@ -19,8 +19,6 @@ package org.apache.dolphinscheduler.common.utils;
import org.junit.Assert;
import org.junit.Test;
import java.util.ArrayList;
public class StringUtilsTest {
@Test
public void testIsNotEmpty() {

3
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java

@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.common.utils;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.LoggerFactory;

210
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32Test.java

@ -0,0 +1,210 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils.process;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@RunWith(PowerMockRunner.class)
@PrepareForTest(OSUtils.class)
public class ProcessBuilderForWin32Test {
private static final Logger logger = LoggerFactory.getLogger(ProcessBuilderForWin32Test.class);
@Before
public void before() {
PowerMockito.mockStatic(OSUtils.class);
PowerMockito.when(OSUtils.isWindows()).thenReturn(true);
}
@Test
public void testCreateProcessBuilderForWin32() {
try {
ProcessBuilderForWin32 builder = new ProcessBuilderForWin32();
Assert.assertNotNull(builder);
builder = new ProcessBuilderForWin32("net");
Assert.assertNotNull(builder);
builder = new ProcessBuilderForWin32(Collections.singletonList("net"));
Assert.assertNotNull(builder);
builder = new ProcessBuilderForWin32((List<String>) null);
Assert.assertNotNull(builder);
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
@Test
public void testBuildUser() {
try {
ProcessBuilderForWin32 builder = new ProcessBuilderForWin32();
builder.user("test", StringUtils.EMPTY);
Assert.assertNotNull(builder);
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
@Test
public void testBuildCommand() {
try {
ProcessBuilderForWin32 builder = new ProcessBuilderForWin32();
builder.command(Collections.singletonList("net"));
Assert.assertNotEquals(0, builder.command().size());
builder = new ProcessBuilderForWin32();
builder.command("net");
Assert.assertNotEquals(0, builder.command().size());
builder = new ProcessBuilderForWin32();
builder.command((List<String>) null);
Assert.assertNotEquals(0, builder.command().size());
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
@Test
public void testEnvironment() {
try {
ProcessBuilderForWin32 builder = new ProcessBuilderForWin32();
Assert.assertNotNull(builder.environment());
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
try {
ProcessBuilderForWin32 builder = new ProcessBuilderForWin32();
builder.environment(new String[]{ "a=123" });
Assert.assertNotEquals(0, builder.environment().size());
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
@Test
public void testDirectory() {
try {
ProcessBuilderForWin32 builder = new ProcessBuilderForWin32();
builder.directory(new File("/tmp"));
Assert.assertNotNull(builder.directory());
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
@Test
public void testStream() {
try {
InputStream in = ProcessBuilderForWin32.NullInputStream.INSTANCE;
Assert.assertNotNull(in);
Assert.assertEquals(-1, in.read());
Assert.assertEquals(0, in.available());
OutputStream out = ProcessBuilderForWin32.NullOutputStream.INSTANCE;
Assert.assertNotNull(out);
out.write(new byte[] {1});
} catch (Exception e) {
logger.error(e.getMessage());
}
}
@Test
public void testRedirect() {
try {
ProcessBuilderForWin32 builder = new ProcessBuilderForWin32();
builder.redirectInput(new File("/tmp"));
Assert.assertNotNull(builder.redirectInput());
Assert.assertNotNull(builder.redirectInput().file());
builder.redirectOutput(new File("/tmp"));
Assert.assertNotNull(builder.redirectOutput());
Assert.assertNotNull(builder.redirectOutput().file());
builder.redirectError(new File("/tmp"));
Assert.assertNotNull(builder.redirectError());
Assert.assertNotNull(builder.redirectError().file());
builder.redirectInput(builder.redirectOutput());
builder.redirectOutput(builder.redirectInput());
builder.redirectError(builder.redirectInput());
Assert.assertNotNull(ProcessBuilderForWin32.Redirect.PIPE.type());
Assert.assertNotNull(ProcessBuilderForWin32.Redirect.PIPE.toString());
Assert.assertNotNull(ProcessBuilderForWin32.Redirect.INHERIT.type());
Assert.assertNotNull(ProcessBuilderForWin32.Redirect.INHERIT.toString());
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
@Test
public void testRedirectErrorStream() {
try {
ProcessBuilderForWin32 builder = new ProcessBuilderForWin32();
builder.redirectErrorStream(true);
Assert.assertTrue(builder.redirectErrorStream());
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
@Test
public void runCmdViaUser() {
try {
ProcessBuilderForWin32 builder = new ProcessBuilderForWin32();
builder.user("test123", StringUtils.EMPTY);
List<String> commands = new ArrayList<>();
commands.add("cmd.exe");
commands.add("/c");
commands.add("net user");
builder.command(commands);
Process process = builder.start();
BufferedReader inReader = new BufferedReader(new InputStreamReader(process.getInputStream(), Charset.forName("GBK")));
String line;
StringBuilder sb = new StringBuilder();
while ((line = inReader.readLine()) != null) {
sb.append(line);
}
logger.info("net user: {}", sb.toString());
Assert.assertNotEquals(StringUtils.EMPTY, sb.toString());
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
}

124
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32Test.java

@ -0,0 +1,124 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils.process;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collection;
import java.util.Map;
import java.util.Set;
@RunWith(PowerMockRunner.class)
@PrepareForTest({OSUtils.class, ProcessEnvironmentForWin32.class})
public class ProcessEnvironmentForWin32Test {
private static final Logger logger = LoggerFactory.getLogger(ProcessBuilderForWin32Test.class);
@Before
public void before() {
try {
PowerMockito.mockStatic(OSUtils.class);
PowerMockito.when(OSUtils.isWindows()).thenReturn(true);
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
@Test
public void testPutAndGet() {
try {
ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0);
processEnvironmentForWin32.put("a", "123");
Assert.assertEquals("123", processEnvironmentForWin32.get("a"));
Assert.assertTrue(processEnvironmentForWin32.containsKey("a"));
Assert.assertTrue(processEnvironmentForWin32.containsValue("123"));
Assert.assertEquals("123", processEnvironmentForWin32.remove("a"));
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
try {
ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0);
processEnvironmentForWin32.put("b=", "123");
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
try {
ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0);
processEnvironmentForWin32.put("b", "\u0000");
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
try {
ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0);
processEnvironmentForWin32.get(null);
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
@Test
public void testEntrySet() {
try {
ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0);
processEnvironmentForWin32.clear();
processEnvironmentForWin32.put("a", "123");
Assert.assertEquals(0, processEnvironmentForWin32.entrySet().size());
Assert.assertTrue(processEnvironmentForWin32.entrySet().isEmpty());
for (Map.Entry<String, String> entry : processEnvironmentForWin32.entrySet()) {
Assert.assertNotNull(entry);
Assert.assertNotNull(entry.getKey());
Assert.assertNotNull(entry.getValue());
Assert.assertNotNull(entry.setValue("123"));
}
processEnvironmentForWin32.clear();
Set<String> keys = processEnvironmentForWin32.keySet();
Assert.assertEquals(0, keys.size());
Assert.assertTrue(keys.isEmpty());
processEnvironmentForWin32.clear();
Collection<String> values = processEnvironmentForWin32.values();
Assert.assertEquals(0, keys.size());
Assert.assertTrue(keys.isEmpty());
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
@Test
public void testToEnvironmentBlock() {
try {
ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0);
Assert.assertNotNull(processEnvironmentForWin32.toEnvironmentBlock());
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
}

70
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32Test.java

@ -0,0 +1,70 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils.process;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sun.security.action.GetPropertyAction;
@RunWith(PowerMockRunner.class)
@PrepareForTest({OSUtils.class, GetPropertyAction.class})
public class ProcessImplForWin32Test {
private static final Logger logger = LoggerFactory.getLogger(ProcessBuilderForWin32Test.class);
@Before
public void before() {
PowerMockito.mockStatic(OSUtils.class);
PowerMockito.mockStatic(GetPropertyAction.class);
PowerMockito.when(OSUtils.isWindows()).thenReturn(true);
}
@Test
public void testStart() {
try {
Process process = ProcessImplForWin32.start(
"test123", StringUtils.EMPTY, new String[]{"net"},
null, null, null, false);
Assert.assertNotNull(process);
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
try {
Process process = ProcessImplForWin32.start(
"test123", StringUtils.EMPTY, new String[]{"net"},
null, null, new ProcessBuilderForWin32.Redirect[]{
ProcessBuilderForWin32.Redirect.PIPE,
ProcessBuilderForWin32.Redirect.PIPE,
ProcessBuilderForWin32.Redirect.PIPE
}, false);
Assert.assertNotNull(process);
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
}

21
dolphinscheduler-dao/pom.xml

@ -25,7 +25,7 @@
</parent>
<artifactId>dolphinscheduler-dao</artifactId>
<name>${project.artifactId}</name>
<url>http://maven.apache.org</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
@ -44,6 +44,12 @@
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-boot-starter</artifactId>
<version>${mybatis-plus.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-to-slf4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
@ -71,6 +77,14 @@
<artifactId>log4j-api</artifactId>
<groupId>org.apache.logging.log4j</groupId>
</exclusion>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-tomcat</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-to-slf4j</artifactId>
</exclusion>
</exclusions>
</dependency>
@ -78,7 +92,10 @@
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>

6
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java

@ -144,9 +144,11 @@ public class AlertDao extends AbstractBaseDao {
* @param taskId taskId
* @param taskName taskName
*/
public void sendTaskTimeoutAlert(int alertgroupId,String receivers,String receiversCc,int taskId,String taskName){
public void sendTaskTimeoutAlert(int alertgroupId,String receivers,String receiversCc, int processInstanceId,
String processInstanceName, int taskId,String taskName){
Alert alert = new Alert();
String content = String.format("[{'id':'%d','name':'%s','event':'timeout','warnLevel':'middle'}]",taskId,taskName);
String content = String.format("[{'process instance id':'%d','task name':'%s','task id':'%d','task name':'%s'," +
"'event':'timeout','warnLevel':'middle'}]", processInstanceId, processInstanceName, taskId, taskName);
alert.setTitle("Task Timeout Warn");
alert.setShowType(ShowType.TABLE);
alert.setContent(content);

2
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/MonitorDBDao.java

@ -61,7 +61,7 @@ public class MonitorDBDao {
return new PostgrePerformance().getMonitorRecord(conn);
}
}catch (Exception e) {
logger.error("SQLException " + e);
logger.error("SQLException: {}", e.getMessage(), e);
}finally {
try {
if (conn != null) {

134
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java

@ -50,25 +50,28 @@ public class TaskRecordDao {
static {
try {
conf = new PropertiesConfiguration(Constants.APPLICATION_PROPERTIES);
}catch (ConfigurationException e){
logger.error("load configuration exception",e);
} catch (ConfigurationException e) {
logger.error("load configuration exception", e);
System.exit(1);
}
}
/**
* get task record flag
* get task record flag
*
* @return whether startup taskrecord
*/
public static boolean getTaskRecordFlag(){
return conf.getBoolean(Constants.TASK_RECORD_FLAG);
public static boolean getTaskRecordFlag() {
return conf.getBoolean(Constants.TASK_RECORD_FLAG);
}
/**
* create connection
*
* @return connection
*/
private static Connection getConn() {
if(!getTaskRecordFlag()){
if (!getTaskRecordFlag()) {
return null;
}
String driver = "com.mysql.jdbc.Driver";
@ -90,101 +93,96 @@ public class TaskRecordDao {
/**
* generate where sql string
*
* @param filterMap filterMap
* @return sql string
*/
private static String getWhereString(Map<String, String> filterMap) {
if(filterMap.size() ==0){
if (filterMap.size() == 0) {
return "";
}
String result = " where 1=1 ";
Object taskName = filterMap.get("taskName");
if(taskName != null && StringUtils.isNotEmpty(taskName.toString())){
if (taskName != null && StringUtils.isNotEmpty(taskName.toString())) {
result += " and PROC_NAME like concat('%', '" + taskName.toString() + "', '%') ";
}
Object taskDate = filterMap.get("taskDate");
if(taskDate != null && StringUtils.isNotEmpty(taskDate.toString())){
if (taskDate != null && StringUtils.isNotEmpty(taskDate.toString())) {
result += " and PROC_DATE='" + taskDate.toString() + "'";
}
Object state = filterMap.get("state");
if(state != null && StringUtils.isNotEmpty(state.toString())){
if (state != null && StringUtils.isNotEmpty(state.toString())) {
result += " and NOTE='" + state.toString() + "'";
}
Object sourceTable = filterMap.get("sourceTable");
if(sourceTable!= null && StringUtils.isNotEmpty(sourceTable.toString())){
result += " and SOURCE_TAB like concat('%', '" + sourceTable.toString()+ "', '%')";
if (sourceTable != null && StringUtils.isNotEmpty(sourceTable.toString())) {
result += " and SOURCE_TAB like concat('%', '" + sourceTable.toString() + "', '%')";
}
Object targetTable = filterMap.get("targetTable");
if(sourceTable!= null && StringUtils.isNotEmpty(targetTable.toString())){
result += " and TARGET_TAB like concat('%', '"+ targetTable.toString()+"', '%') " ;
if (sourceTable != null && StringUtils.isNotEmpty(targetTable.toString())) {
result += " and TARGET_TAB like concat('%', '" + targetTable.toString() + "', '%') ";
}
Object start = filterMap.get("startTime");
if(start != null && StringUtils.isNotEmpty(start.toString())){
if (start != null && StringUtils.isNotEmpty(start.toString())) {
result += " and STARTDATE>='" + start.toString() + "'";
}
Object end = filterMap.get("endTime");
if(end != null && StringUtils.isNotEmpty(end.toString())){
result += " and ENDDATE>='" + end.toString()+ "'";
if (end != null && StringUtils.isNotEmpty(end.toString())) {
result += " and ENDDATE>='" + end.toString() + "'";
}
return result;
}
/**
* count task record
*
* @param filterMap filterMap
* @param table table
* @param table table
* @return task record count
*/
public static int countTaskRecord(Map<String, String> filterMap, String table){
public static int countTaskRecord(Map<String, String> filterMap, String table) {
int count = 0;
Connection conn = null;
PreparedStatement pstmt = null;
ResultSet rs = null;
try {
conn = getConn();
if(conn == null){
if (conn == null) {
return count;
}
String sql = String.format("select count(1) as count from %s", table);
sql += getWhereString(filterMap);
pstmt = conn.prepareStatement(sql);
ResultSet rs = pstmt.executeQuery();
while(rs.next()){
rs = pstmt.executeQuery();
while (rs.next()) {
count = rs.getInt("count");
break;
}
} catch (SQLException e) {
logger.error("Exception ", e);
}finally {
try {
if(pstmt != null) {
pstmt.close();
}
if(conn != null){
conn.close();
}
} catch (SQLException e) {
logger.error("Exception ", e);
}
} finally {
closeResource(rs, pstmt, conn);
}
return count;
}
/**
* query task record by filter map paging
*
* @param filterMap filterMap
* @param table table
* @param table table
* @return task record list
*/
public static List<TaskRecord> queryAllTaskRecord(Map<String,String> filterMap , String table) {
public static List<TaskRecord> queryAllTaskRecord(Map<String, String> filterMap, String table) {
String sql = String.format("select * from %s", table);
sql += getWhereString(filterMap);
@ -194,9 +192,9 @@ public class TaskRecordDao {
sql += String.format(" order by STARTDATE desc limit %d,%d", offset, pageSize);
List<TaskRecord> recordList = new ArrayList<>();
try{
try {
recordList = getQueryResult(sql);
}catch (Exception e){
} catch (Exception e) {
logger.error("Exception ", e);
}
return recordList;
@ -204,6 +202,7 @@ public class TaskRecordDao {
/**
* convert result set to task record
*
* @param resultSet resultSet
* @return task record
* @throws SQLException if error throws SQLException
@ -232,6 +231,7 @@ public class TaskRecordDao {
/**
* query task list by select sql
*
* @param selectSql select sql
* @return task record list
*/
@ -239,65 +239,81 @@ public class TaskRecordDao {
List<TaskRecord> recordList = new ArrayList<>();
Connection conn = null;
PreparedStatement pstmt = null;
ResultSet rs = null;
try {
conn = getConn();
if(conn == null){
if (conn == null) {
return recordList;
}
pstmt = conn.prepareStatement(selectSql);
ResultSet rs = pstmt.executeQuery();
rs = pstmt.executeQuery();
while(rs.next()){
while (rs.next()) {
TaskRecord taskRecord = convertToTaskRecord(rs);
recordList.add(taskRecord);
}
} catch (SQLException e) {
logger.error("Exception ", e);
}finally {
try {
if(pstmt != null) {
pstmt.close();
}
if(conn != null){
conn.close();
}
} catch (SQLException e) {
logger.error("Exception ", e);
}
} finally {
closeResource(rs, pstmt, conn);
}
return recordList;
}
/**
* according to procname and procdate query task record
*
* @param procName procName
* @param procDate procDate
* @return task record status
*/
public static TaskRecordStatus getTaskRecordState(String procName,String procDate){
public static TaskRecordStatus getTaskRecordState(String procName, String procDate) {
String sql = String.format("SELECT * FROM eamp_hive_log_hd WHERE PROC_NAME='%s' and PROC_DATE like '%s'"
,procName,procDate + "%");
, procName, procDate + "%");
List<TaskRecord> taskRecordList = getQueryResult(sql);
// contains no record and sql exception
if (CollectionUtils.isEmpty(taskRecordList)){
if (CollectionUtils.isEmpty(taskRecordList)) {
// exception
return TaskRecordStatus.EXCEPTION;
}else if (taskRecordList.size() > 1){
} else if (taskRecordList.size() > 1) {
return TaskRecordStatus.EXCEPTION;
}else {
} else {
TaskRecord taskRecord = taskRecordList.get(0);
if (taskRecord == null){
if (taskRecord == null) {
return TaskRecordStatus.EXCEPTION;
}
Long targetRowCount = taskRecord.getTargetRowCount();
if (targetRowCount <= 0){
if (targetRowCount <= 0) {
return TaskRecordStatus.FAILURE;
}else {
} else {
return TaskRecordStatus.SUCCESS;
}
}
}
private static void closeResource(ResultSet rs, PreparedStatement pstmt, Connection conn) {
if (rs != null) {
try {
rs.close();
} catch (SQLException e) {
logger.error("Exception ", e);
}
}
if (pstmt != null) {
try {
pstmt.close();
} catch (SQLException e) {
logger.error("Exception ", e);
}
}
if (conn != null) {
try {
conn.close();
} catch (SQLException e) {
logger.error("Exception ", e);
}
}
}
}

2
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java

@ -39,7 +39,7 @@ public class HiveDataSource extends BaseDataSource {
@Override
public String getJdbcUrl() {
String jdbcUrl = getAddress();
if (jdbcUrl.lastIndexOf("/") != (jdbcUrl.length() - 1)) {
if (jdbcUrl.lastIndexOf('/') != (jdbcUrl.length() - 1)) {
jdbcUrl += "/";
}

2
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java

@ -39,7 +39,7 @@ public class MySQLDataSource extends BaseDataSource {
@Override
public String getJdbcUrl() {
String address = getAddress();
if (address.lastIndexOf("/") != (address.length() - 1)) {
if (address.lastIndexOf('/') != (address.length() - 1)) {
address += "/";
}
String jdbcUrl = address + getDatabase();

2
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java

@ -40,7 +40,7 @@ public class PostgreDataSource extends BaseDataSource {
@Override
public String getJdbcUrl() {
String jdbcUrl = getAddress();
if (jdbcUrl.lastIndexOf("/") != (jdbcUrl.length() - 1)) {
if (jdbcUrl.lastIndexOf('/') != (jdbcUrl.length() - 1)) {
jdbcUrl += "/";
}

9
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java

@ -16,10 +16,10 @@
*/
package org.apache.dolphinscheduler.dao.entity;
import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.enums.ReleaseState;
import org.apache.dolphinscheduler.common.process.Property;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
@ -29,7 +29,6 @@ import com.baomidou.mybatisplus.core.toolkit.StringUtils;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
@ -267,7 +266,7 @@ public class ProcessDefinition {
}
public void setGlobalParams(String globalParams) {
this.globalParamList = JSONObject.parseArray(globalParams, Property.class);
this.globalParamList = JSON.parseArray(globalParams, Property.class);
this.globalParams = globalParams;
}
@ -276,7 +275,7 @@ public class ProcessDefinition {
}
public void setGlobalParamList(List<Property> globalParamList) {
this.globalParams = JSONObject.toJSONString(globalParamList);
this.globalParams = JSON.toJSONString(globalParamList);
this.globalParamList = globalParamList;
}
@ -284,7 +283,7 @@ public class ProcessDefinition {
List<Property> propList;
if (globalParamMap == null && StringUtils.isNotEmpty(globalParams)) {
propList = JSONObject.parseArray(globalParams, Property.class);
propList = JSON.parseArray(globalParams, Property.class);
globalParamMap = propList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue));
}

2
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java

@ -506,7 +506,7 @@ public class ProcessInstance {
* check this process is start complement data
* @return whether complement data
*/
public Boolean isComplementData(){
public boolean isComplementData(){
if(!StringUtils.isNotEmpty(this.historyCmd)){
return false;
}

4
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java

@ -373,7 +373,7 @@ public class TaskInstance {
}
public Boolean isSubProcess(){
public boolean isSubProcess(){
return TaskType.SUB_PROCESS.getDescp().equals(this.taskType);
}
@ -442,7 +442,7 @@ public class TaskInstance {
this.executorName = executorName;
}
public Boolean isTaskComplete() {
public boolean isTaskComplete() {
return this.getState().typeIsPause()
|| this.getState().typeIsSuccess()

1
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java

@ -16,7 +16,6 @@
*/
package org.apache.dolphinscheduler.dao.entity;
import com.baomidou.mybatisplus.annotation.TableField;
import org.apache.dolphinscheduler.common.enums.UdfType;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;

24
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java

@ -17,7 +17,6 @@
package org.apache.dolphinscheduler.dao.upgrade;
import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -30,15 +29,7 @@ import java.sql.SQLException;
*/
public class MysqlUpgradeDao extends UpgradeDao {
public static final Logger logger = LoggerFactory.getLogger(UpgradeDao.class);
/**
* init
*/
@Override
protected void init() {
}
public static final Logger logger = LoggerFactory.getLogger(MysqlUpgradeDao.class);
/**
* mysql upgrade dao holder
@ -70,12 +61,7 @@ public class MysqlUpgradeDao extends UpgradeDao {
try {
conn = dataSource.getConnection();
rs = conn.getMetaData().getTables(null, null, tableName, null);
if (rs.next()) {
return true;
} else {
return false;
}
return rs.next();
} catch (SQLException e) {
logger.error(e.getMessage(),e);
throw new RuntimeException(e.getMessage(),e);
@ -97,11 +83,7 @@ public class MysqlUpgradeDao extends UpgradeDao {
try {
conn = dataSource.getConnection();
ResultSet rs = conn.getMetaData().getColumns(null,null,tableName,columnName);
if (rs.next()) {
return true;
} else {
return false;
}
return rs.next();
} catch (SQLException e) {
logger.error(e.getMessage(),e);

40
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java

@ -17,7 +17,6 @@
package org.apache.dolphinscheduler.dao.upgrade;
import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -31,16 +30,8 @@ import java.sql.SQLException;
*/
public class PostgresqlUpgradeDao extends UpgradeDao {
public static final Logger logger = LoggerFactory.getLogger(UpgradeDao.class);
private static final String schema = getSchema();
/**
* init
*/
@Override
protected void init() {
}
public static final Logger logger = LoggerFactory.getLogger(PostgresqlUpgradeDao.class);
private static final String SCHEMA = getSchema();
/**
* postgresql upgrade dao holder
@ -59,16 +50,6 @@ public class PostgresqlUpgradeDao extends UpgradeDao {
return PostgresqlUpgradeDaoHolder.INSTANCE;
}
/**
* init schema
* @param initSqlPath initSqlPath
*/
@Override
public void initSchema(String initSqlPath) {
super.initSchema(initSqlPath);
}
/**
* getSchema
* @return schema
@ -108,13 +89,9 @@ public class PostgresqlUpgradeDao extends UpgradeDao {
try {
conn = dataSource.getConnection();
rs = conn.getMetaData().getTables(null, schema, tableName, null);
if (rs.next()) {
return true;
} else {
return false;
}
rs = conn.getMetaData().getTables(null, SCHEMA, tableName, null);
return rs.next();
} catch (SQLException e) {
logger.error(e.getMessage(),e);
throw new RuntimeException(e.getMessage(),e);
@ -136,13 +113,8 @@ public class PostgresqlUpgradeDao extends UpgradeDao {
ResultSet rs = null;
try {
conn = dataSource.getConnection();
rs = conn.getMetaData().getColumns(null,schema,tableName,columnName);
if (rs.next()) {
return true;
} else {
return false;
}
rs = conn.getMetaData().getColumns(null, SCHEMA,tableName,columnName);
return rs.next();
} catch (SQLException e) {
logger.error(e.getMessage(),e);
throw new RuntimeException(e.getMessage(),e);

3
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/MysqlPerformance.java

@ -27,7 +27,6 @@ import java.util.Date;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.dao.MonitorDBDao;
import org.apache.dolphinscheduler.dao.entity.MonitorRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -37,7 +36,7 @@ import org.slf4j.LoggerFactory;
*/
public class MysqlPerformance extends BaseDBPerformance{
private static Logger logger = LoggerFactory.getLogger(MonitorDBDao.class);
private static Logger logger = LoggerFactory.getLogger(MysqlPerformance.class);
/**

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save