Browse Source

Merge remote-tracking branch 'upstream/dev' into dev

pull/3/MERGE
dailidong 5 years ago
parent
commit
34db3ca930
  1. 12
      .github/workflows/ci_backend.yml
  2. 8
      .github/workflows/ci_e2e.yml
  3. 12
      .github/workflows/ci_frontend.yml
  4. 8
      .github/workflows/ci_ut.yml
  5. 13
      dockerfile/Dockerfile
  6. 40
      dockerfile/README.md
  7. 38
      dockerfile/README_zh_CN.md
  8. 27
      dockerfile/checkpoint.sh
  9. 2
      dockerfile/conf/dolphinscheduler/application.properties.tpl
  10. 12
      dockerfile/conf/dolphinscheduler/common.properties.tpl
  11. 0
      dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh
  12. 2
      dockerfile/conf/dolphinscheduler/quartz.properties.tpl
  13. 4
      dockerfile/startup-init-conf.sh
  14. 4
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java
  15. 1
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java
  16. 36
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java
  17. 8
      dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java
  18. 1
      dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java
  19. 6
      dolphinscheduler-api/pom.xml
  20. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java
  21. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java
  22. 15
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java
  23. 1
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java
  24. 1
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java
  25. 1
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java
  26. 2
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java
  27. 2
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
  28. 5
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java
  29. 1
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java
  30. 8
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java
  31. 1
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMainTest.java
  32. 2
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitorUtilsTest.java
  33. 2
      dolphinscheduler-common/pom.xml
  34. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
  35. 3
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/WorkerLogFilter.java
  36. 16
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
  37. 14
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java
  38. 1065
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32.java
  39. 286
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32.java
  40. 785
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java
  41. 40
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java
  42. 2
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java
  43. 1
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java
  44. 1
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java
  45. 4
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java
  46. 3
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java
  47. 3
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java
  48. 2
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java
  49. 3
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java
  50. 210
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32Test.java
  51. 124
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32Test.java
  52. 70
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32Test.java
  53. 2
      dolphinscheduler-dao/pom.xml
  54. 6
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java
  55. 1
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java
  56. 1
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java
  57. 1
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java
  58. 1
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java
  59. 4
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java
  60. 2
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java
  61. 1
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java
  62. 2
      dolphinscheduler-remote/pom.xml
  63. 2
      dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java
  64. 2
      dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java
  65. 2
      dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskRequestCommand.java
  66. 2
      dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskResponseCommand.java
  67. 1
      dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java
  68. 1
      dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java
  69. 1
      dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java
  70. 1
      dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java
  71. 2
      dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java
  72. 2
      dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java
  73. 8
      dolphinscheduler-server/pom.xml
  74. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java
  75. 4
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java
  76. 4
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java
  77. 58
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java
  78. 29
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java
  79. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskProps.java
  80. 9
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java
  81. 7
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java
  82. 17
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java
  83. 199
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java
  84. 6
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java
  85. 1
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java
  86. 6
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
  87. 18
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java
  88. 10
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProvider.java
  89. 3
      dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtilsTest.java
  90. 2
      dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/BaseTaskQueueTest.java
  91. 2
      dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskQueueZKImplTest.java
  92. 65
      dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProviderTest.java
  93. 2
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue
  94. 10
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue
  95. 13
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue
  96. 49
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue
  97. 30
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue
  98. 29
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue
  99. 44
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue
  100. 13
      pom.xml
  101. Some files were not shown because too many files have changed in this diff Show More

12
.github/workflows/ci_backend.yml

@ -46,6 +46,12 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
# In the checkout@v2, it doesn't support git submodule. Execute the commands manually.
- name: checkout submodules
shell: bash
run: |
git submodule sync --recursive
git -c protocol.version=2 submodule update --init --force --recursive --depth=1
- name: Set up JDK 1.8 - name: Set up JDK 1.8
uses: actions/setup-java@v1 uses: actions/setup-java@v1
with: with:
@ -56,6 +62,12 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
# In the checkout@v2, it doesn't support git submodule. Execute the commands manually.
- name: checkout submodules
shell: bash
run: |
git submodule sync --recursive
git -c protocol.version=2 submodule update --init --force --recursive --depth=1
- name: Set up JDK 1.8 - name: Set up JDK 1.8
uses: actions/setup-java@v1 uses: actions/setup-java@v1
with: with:

8
.github/workflows/ci_e2e.yml

@ -15,7 +15,7 @@
# limitations under the License. # limitations under the License.
# #
on: ["push", "pull_request"] on: ["pull_request"]
env: env:
DOCKER_DIR: ./docker DOCKER_DIR: ./docker
LOG_DIR: /tmp/dolphinscheduler LOG_DIR: /tmp/dolphinscheduler
@ -30,6 +30,12 @@ jobs:
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
# In the checkout@v2, it doesn't support git submodule. Execute the commands manually.
- name: checkout submodules
shell: bash
run: |
git submodule sync --recursive
git -c protocol.version=2 submodule update --init --force --recursive --depth=1
- uses: actions/cache@v1 - uses: actions/cache@v1
with: with:
path: ~/.m2/repository path: ~/.m2/repository

12
.github/workflows/ci_frontend.yml

@ -35,6 +35,12 @@ jobs:
os: [ubuntu-latest, macos-latest] os: [ubuntu-latest, macos-latest]
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
# In the checkout@v2, it doesn't support git submodule. Execute the commands manually.
- name: checkout submodules
shell: bash
run: |
git submodule sync --recursive
git -c protocol.version=2 submodule update --init --force --recursive --depth=1
- name: Set up Node.js - name: Set up Node.js
uses: actions/setup-node@v1 uses: actions/setup-node@v1
with: with:
@ -50,6 +56,12 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
# In the checkout@v2, it doesn't support git submodule. Execute the commands manually.
- name: checkout submodules
shell: bash
run: |
git submodule sync --recursive
git -c protocol.version=2 submodule update --init --force --recursive --depth=1
- name: Set up JDK 1.8 - name: Set up JDK 1.8
uses: actions/setup-java@v1 uses: actions/setup-java@v1
with: with:

8
.github/workflows/ci_ut.yml

@ -15,7 +15,7 @@
# limitations under the License. # limitations under the License.
# #
on: ["push", "pull_request"] on: ["pull_request"]
env: env:
DOCKER_DIR: ./docker DOCKER_DIR: ./docker
LOG_DIR: /tmp/dolphinscheduler LOG_DIR: /tmp/dolphinscheduler
@ -30,6 +30,12 @@ jobs:
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
# In the checkout@v2, it doesn't support git submodule. Execute the commands manually.
- name: checkout submodules
shell: bash
run: |
git submodule sync --recursive
git -c protocol.version=2 submodule update --init --force --recursive --depth=1
- uses: actions/cache@v1 - uses: actions/cache@v1
with: with:
path: ~/.m2/repository path: ~/.m2/repository

13
dockerfile/Dockerfile

@ -63,19 +63,22 @@ RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \
ADD ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d ADD ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d
#7. add configuration and modify permissions and set soft links #7. add configuration and modify permissions and set soft links
ADD ./checkpoint.sh /root/checkpoint.sh
ADD ./startup-init-conf.sh /root/startup-init-conf.sh ADD ./startup-init-conf.sh /root/startup-init-conf.sh
ADD ./startup.sh /root/startup.sh ADD ./startup.sh /root/startup.sh
ADD ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/ ADD ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/
ADD ./conf/dolphinscheduler/env/dolphinscheduler_env /opt/dolphinscheduler/conf/env/ ADD conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/
RUN chmod +x /root/startup-init-conf.sh && \ RUN chmod +x /root/checkpoint.sh && \
chmod +x /root/startup-init-conf.sh && \
chmod +x /root/startup.sh && \ chmod +x /root/startup.sh && \
chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env && \ chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \
chmod +x /opt/dolphinscheduler/script/*.sh && \ chmod +x /opt/dolphinscheduler/script/*.sh && \
chmod +x /opt/dolphinscheduler/bin/*.sh && \ chmod +x /opt/dolphinscheduler/bin/*.sh && \
chmod +x /opt/zookeeper/bin/*.sh && \ chmod +x /opt/zookeeper/bin/*.sh && \
dos2unix /root/checkpoint.sh && \
dos2unix /root/startup-init-conf.sh && \ dos2unix /root/startup-init-conf.sh && \
dos2unix /root/startup.sh && \ dos2unix /root/startup.sh && \
dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env && \ dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \
dos2unix /opt/dolphinscheduler/script/*.sh && \ dos2unix /opt/dolphinscheduler/script/*.sh && \
dos2unix /opt/dolphinscheduler/bin/*.sh && \ dos2unix /opt/dolphinscheduler/bin/*.sh && \
dos2unix /opt/zookeeper/bin/*.sh && \ dos2unix /opt/zookeeper/bin/*.sh && \
@ -87,6 +90,6 @@ RUN chmod +x /root/startup-init-conf.sh && \
RUN rm -rf /var/cache/apk/* RUN rm -rf /var/cache/apk/*
#9. expose port #9. expose port
EXPOSE 2181 2888 3888 5432 12345 8888 EXPOSE 2181 2888 3888 5432 12345 50051 8888
ENTRYPOINT ["/root/startup.sh"] ENTRYPOINT ["/root/startup.sh"]

40
dockerfile/README.md

@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org
#### You can start a dolphinscheduler instance #### You can start a dolphinscheduler instance
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test \ -e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \
-p 8888:8888 \ -p 8888:8888 \
dolphinscheduler all dolphinscheduler all
``` ```
@ -25,13 +25,13 @@ The default postgres user `root`, postgres password `root` and database `dolphin
The default zookeeper is created in the `startup.sh`. The default zookeeper is created in the `startup.sh`.
#### Or via Environment Variables **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`ZOOKEEPER_QUORUM`** #### Or via Environment Variables **`POSTGRESQL_HOST`** **`POSTGRESQL_PORT`** **`POSTGRESQL_DATABASE`** **`ZOOKEEPER_QUORUM`**
You can specify **existing postgres service**. Example: You can specify **existing postgres service**. Example:
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ -e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
-p 8888:8888 \ -p 8888:8888 \
dolphinscheduler all dolphinscheduler all
@ -42,7 +42,7 @@ You can specify **existing zookeeper service**. Example:
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181" -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-p 8888:8888 \ -p 8888:8888 \
dolphinscheduler all dolphinscheduler all
``` ```
@ -56,7 +56,7 @@ You can start a standalone dolphinscheduler server.
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181" -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ -e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
dolphinscheduler master-server dolphinscheduler master-server
``` ```
@ -66,7 +66,7 @@ dolphinscheduler master-server
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181" -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ -e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
dolphinscheduler worker-server dolphinscheduler worker-server
``` ```
@ -75,7 +75,7 @@ dolphinscheduler worker-server
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ -e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
-p 12345:12345 \ -p 12345:12345 \
dolphinscheduler api-server dolphinscheduler api-server
@ -85,7 +85,7 @@ dolphinscheduler api-server
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ -e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
dolphinscheduler alert-server dolphinscheduler alert-server
``` ```
@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \
dolphinscheduler frontend dolphinscheduler frontend
``` ```
**Note**: You must be specify `POSTGRESQL_HOST` `POSTGRESQL_PORT` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server. **Note**: You must be specify `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server.
## How to build a docker image ## How to build a docker image
@ -140,14 +140,36 @@ This environment variable sets the port for PostgreSQL. The default value is `54
This environment variable sets the username for PostgreSQL. The default value is `root`. This environment variable sets the username for PostgreSQL. The default value is `root`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`POSTGRESQL_PASSWORD`** **`POSTGRESQL_PASSWORD`**
This environment variable sets the password for PostgreSQL. The default value is `root`. This environment variable sets the password for PostgreSQL. The default value is `root`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`POSTGRESQL_DATABASE`**
This environment variable sets the database for PostgreSQL. The default value is `dolphinscheduler`.
**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`DOLPHINSCHEDULER_ENV_PATH`** **`DOLPHINSCHEDULER_ENV_PATH`**
This environment variable sets the runtime environment for task. The default value is `/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh`. This environment variable sets the runtime environment for task. The default value is `/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh`.
**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`**
User data directory path, self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler`
**`DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH`**
Directory path for user data download. self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler/download`
**`DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH`**
Process execute directory. self configuration, please make sure the directory exists and have read write permissions. The default value is `/tmp/dolphinscheduler/exec`
**`TASK_QUEUE`** **`TASK_QUEUE`**
This environment variable sets the task queue for `master-server` and `worker-serverr`. The default value is `zookeeper`. This environment variable sets the task queue for `master-server` and `worker-serverr`. The default value is `zookeeper`.

38
dockerfile/README_zh_CN.md

@ -16,7 +16,7 @@ Official Website: https://dolphinscheduler.apache.org
#### 你可以运行一个dolphinscheduler实例 #### 你可以运行一个dolphinscheduler实例
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test \ -e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -e POSTGRESQL_DATABASE=dolphinscheduler \
-p 8888:8888 \ -p 8888:8888 \
dolphinscheduler all dolphinscheduler all
``` ```
@ -31,7 +31,7 @@ dolphinscheduler all
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ -e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
-p 8888:8888 \ -p 8888:8888 \
dolphinscheduler all dolphinscheduler all
@ -42,7 +42,7 @@ dolphinscheduler all
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181" -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-p 8888:8888 \ -p 8888:8888 \
dolphinscheduler all dolphinscheduler all
``` ```
@ -56,7 +56,7 @@ dolphinscheduler all
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181" -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ -e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
dolphinscheduler master-server dolphinscheduler master-server
``` ```
@ -66,7 +66,7 @@ dolphinscheduler master-server
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181" -e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ -e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
dolphinscheduler worker-server dolphinscheduler worker-server
``` ```
@ -75,7 +75,7 @@ dolphinscheduler worker-server
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ -e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
-p 12345:12345 \ -p 12345:12345 \
dolphinscheduler api-server dolphinscheduler api-server
@ -85,7 +85,7 @@ dolphinscheduler api-server
``` ```
$ docker run -dit --name dolphinscheduler \ $ docker run -dit --name dolphinscheduler \
-e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" \ -e POSTGRESQL_HOST="192.168.x.x" -e POSTGRESQL_PORT="5432" -e POSTGRESQL_DATABASE="dolphinscheduler" \
-e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \ -e POSTGRESQL_USERNAME="test" -e POSTGRESQL_PASSWORD="test" \
dolphinscheduler alert-server dolphinscheduler alert-server
``` ```
@ -99,7 +99,7 @@ $ docker run -dit --name dolphinscheduler \
dolphinscheduler frontend dolphinscheduler frontend
``` ```
**注意**: 当你运行dolphinscheduler中的部分服务时,你必须指定这些环境变量 `POSTGRESQL_HOST` `POSTGRESQL_PORT` `ZOOKEEPER_QUORUM` **注意**: 当你运行dolphinscheduler中的部分服务时,你必须指定这些环境变量 `POSTGRESQL_HOST` `POSTGRESQL_PORT` `POSTGRESQL_DATABASE` `POSTGRESQL_USERNAME` `POSTGRESQL_PASSWORD` `ZOOKEEPER_QUORUM`
## 如何构建一个docker镜像 ## 如何构建一个docker镜像
@ -140,14 +140,36 @@ Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些
配置`PostgreSQL`的`USERNAME`, 默认值 `root` 配置`PostgreSQL`的`USERNAME`, 默认值 `root`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`POSTGRESQL_PASSWORD`** **`POSTGRESQL_PASSWORD`**
配置`PostgreSQL`的`PASSWORD`, 默认值 `root` 配置`PostgreSQL`的`PASSWORD`, 默认值 `root`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`POSTGRESQL_DATABASE`**
配置`PostgreSQL`的`DATABASE`, 默认值 `dolphinscheduler`
**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
**`DOLPHINSCHEDULER_ENV_PATH`** **`DOLPHINSCHEDULER_ENV_PATH`**
任务执行时的环境变量配置文件, 默认值 `/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh` 任务执行时的环境变量配置文件, 默认值 `/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh`
**`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`**
用户数据目录, 用户自己配置, 请确保这个目录存在并且用户读写权限, 默认值 `/tmp/dolphinscheduler`
**`DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH`**
用户数据下载目录, 用户自己配置, 请确保这个目录存在并且用户读写权限, 默认值 `/tmp/dolphinscheduler/download`
**`DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH`**
任务执行目录, 用户自己配置, 请确保这个目录存在并且用户读写权限, 默认值 `/tmp/dolphinscheduler/exec`
**`TASK_QUEUE`** **`TASK_QUEUE`**
配置`master-server`和`worker-serverr`的`Zookeeper`任务队列名, 默认值 `zookeeper` 配置`master-server`和`worker-serverr`的`Zookeeper`任务队列名, 默认值 `zookeeper`

27
dockerfile/checkpoint.sh

@ -0,0 +1,27 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -e
if [ "$(ps -ef | grep java | grep -c $1)" -eq 0 ]; then
echo "[ERROR] $1 process not exits."
exit 1
else
echo "[INFO] $1 process exits."
exit 0
fi

2
dockerfile/conf/dolphinscheduler/application.properties.tpl

@ -19,7 +19,7 @@
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
# postgre # postgre
spring.datasource.driver-class-name=org.postgresql.Driver spring.datasource.driver-class-name=org.postgresql.Driver
spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/dolphinscheduler?characterEncoding=utf8 spring.datasource.url=jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8
# mysql # mysql
#spring.datasource.driver-class-name=com.mysql.jdbc.Driver #spring.datasource.driver-class-name=com.mysql.jdbc.Driver
#spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 #spring.datasource.url=jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8

12
dockerfile/conf/dolphinscheduler/common.properties.tpl

@ -38,6 +38,12 @@ dolphinscheduler.env.path=${DOLPHINSCHEDULER_ENV_PATH}
resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties
# is development state? default "false" # is development state? default "false"
development.state=true development.state=true
# user data directory path, self configuration, please make sure the directory exists and have read write permissions
data.basedir.path=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH}
# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions
data.download.basedir.path=${DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH}
# process execute directory. self configuration, please make sure the directory exists and have read write permissions
process.exec.basepath=${DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH}
# resource upload startup type : HDFS,S3,NONE # resource upload startup type : HDFS,S3,NONE
res.upload.startup.type=NONE res.upload.startup.type=NONE
@ -49,12 +55,6 @@ res.upload.startup.type=NONE
hdfs.root.user=hdfs hdfs.root.user=hdfs
# data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended # data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended
data.store2hdfs.basepath=/dolphinscheduler data.store2hdfs.basepath=/dolphinscheduler
# user data directory path, self configuration, please make sure the directory exists and have read write permissions
data.basedir.path=/tmp/dolphinscheduler
# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions
data.download.basedir.path=/tmp/dolphinscheduler/download
# process execute directory. self configuration, please make sure the directory exists and have read write permissions
process.exec.basepath=/tmp/dolphinscheduler/exec
# whether kerberos starts # whether kerberos starts
hadoop.security.authentication.startup.state=false hadoop.security.authentication.startup.state=false
# java.security.krb5.conf path # java.security.krb5.conf path

0
dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env → dockerfile/conf/dolphinscheduler/env/dolphinscheduler_env.sh vendored

2
dockerfile/conf/dolphinscheduler/quartz.properties.tpl

@ -22,7 +22,7 @@
org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.PostgreSQLDelegate org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
# postgre # postgre
org.quartz.dataSource.myDs.driver = org.postgresql.Driver org.quartz.dataSource.myDs.driver = org.postgresql.Driver
org.quartz.dataSource.myDs.URL = jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/dolphinscheduler?characterEncoding=utf8 org.quartz.dataSource.myDs.URL = jdbc:postgresql://${POSTGRESQL_HOST}:${POSTGRESQL_PORT}/${POSTGRESQL_DATABASE}?characterEncoding=utf8
org.quartz.dataSource.myDs.user = ${POSTGRESQL_USERNAME} org.quartz.dataSource.myDs.user = ${POSTGRESQL_USERNAME}
org.quartz.dataSource.myDs.password = ${POSTGRESQL_PASSWORD} org.quartz.dataSource.myDs.password = ${POSTGRESQL_PASSWORD}
org.quartz.scheduler.instanceName = DolphinScheduler org.quartz.scheduler.instanceName = DolphinScheduler

4
dockerfile/startup-init-conf.sh

@ -28,11 +28,15 @@ export POSTGRESQL_HOST=${POSTGRESQL_HOST:-"127.0.0.1"}
export POSTGRESQL_PORT=${POSTGRESQL_PORT:-"5432"} export POSTGRESQL_PORT=${POSTGRESQL_PORT:-"5432"}
export POSTGRESQL_USERNAME=${POSTGRESQL_USERNAME:-"root"} export POSTGRESQL_USERNAME=${POSTGRESQL_USERNAME:-"root"}
export POSTGRESQL_PASSWORD=${POSTGRESQL_PASSWORD:-"root"} export POSTGRESQL_PASSWORD=${POSTGRESQL_PASSWORD:-"root"}
export POSTGRESQL_DATABASE=${POSTGRESQL_DATABASE:-"dolphinscheduler"}
#============================================================================ #============================================================================
# System # System
#============================================================================ #============================================================================
export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"} export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"}
export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"}
export DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_DOWNLOAD_BASEDIR_PATH:-"/tmp/dolphinscheduler/download"}
export DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH=${DOLPHINSCHEDULER_PROCESS_EXEC_BASEPATH:-"/tmp/dolphinscheduler/exec"}
#============================================================================ #============================================================================
# Zookeeper # Zookeeper

4
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/manager/EnterpriseWeChatManager.java

@ -42,8 +42,8 @@ public class EnterpriseWeChatManager {
public Map<String,Object> send(Alert alert, String token){ public Map<String,Object> send(Alert alert, String token){
Map<String,Object> retMap = new HashMap<>(); Map<String,Object> retMap = new HashMap<>();
retMap.put(Constants.STATUS, false); retMap.put(Constants.STATUS, false);
String agentId = EnterpriseWeChatUtils.enterpriseWeChatAgentId; String agentId = EnterpriseWeChatUtils.ENTERPRISE_WE_CHAT_AGENT_ID;
String users = EnterpriseWeChatUtils.enterpriseWeChatUsers; String users = EnterpriseWeChatUtils.ENTERPRISE_WE_CHAT_USERS;
List<String> userList = Arrays.asList(users.split(",")); List<String> userList = Arrays.asList(users.split(","));
logger.info("send message {}",alert); logger.info("send message {}",alert);
String msg = EnterpriseWeChatUtils.makeUserSendMsg(userList, agentId,EnterpriseWeChatUtils.markdownByAlert(alert)); String msg = EnterpriseWeChatUtils.makeUserSendMsg(userList, agentId,EnterpriseWeChatUtils.markdownByAlert(alert));

1
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java

@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.alert.template.impl;
import org.apache.dolphinscheduler.alert.template.AlertTemplate; import org.apache.dolphinscheduler.alert.template.AlertTemplate;
import org.apache.dolphinscheduler.alert.utils.Constants; import org.apache.dolphinscheduler.alert.utils.Constants;
import org.apache.dolphinscheduler.alert.utils.JSONUtils; import org.apache.dolphinscheduler.alert.utils.JSONUtils;
import org.apache.dolphinscheduler.alert.utils.MailUtils;
import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.enums.ShowType;
import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;

36
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java

@ -43,24 +43,24 @@ public class EnterpriseWeChatUtils {
public static final Logger logger = LoggerFactory.getLogger(EnterpriseWeChatUtils.class); public static final Logger logger = LoggerFactory.getLogger(EnterpriseWeChatUtils.class);
private static final String enterpriseWeChatCorpId = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_CORP_ID); private static final String ENTERPRISE_WE_CHAT_CORP_ID = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_CORP_ID);
private static final String enterpriseWeChatSecret = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_SECRET); private static final String ENTERPRISE_WE_CHAT_SECRET = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_SECRET);
private static final String enterpriseWeChatTokenUrl = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TOKEN_URL); private static final String ENTERPRISE_WE_CHAT_TOKEN_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TOKEN_URL);
private static String enterpriseWeChatTokenUrlReplace = enterpriseWeChatTokenUrl private static final String ENTERPRISE_WE_CHAT_TOKEN_URL_REPLACE = ENTERPRISE_WE_CHAT_TOKEN_URL
.replaceAll("\\$corpId", enterpriseWeChatCorpId) .replaceAll("\\$corpId", ENTERPRISE_WE_CHAT_CORP_ID)
.replaceAll("\\$secret", enterpriseWeChatSecret); .replaceAll("\\$secret", ENTERPRISE_WE_CHAT_SECRET);
private static final String enterpriseWeChatPushUrl = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_PUSH_URL); private static final String ENTERPRISE_WE_CHAT_PUSH_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_PUSH_URL);
private static final String enterpriseWeChatTeamSendMsg = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TEAM_SEND_MSG); private static final String ENTERPRISE_WE_CHAT_TEAM_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TEAM_SEND_MSG);
private static final String enterpriseWeChatUserSendMsg = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USER_SEND_MSG); private static final String ENTERPRISE_WE_CHAT_USER_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USER_SEND_MSG);
public static final String enterpriseWeChatAgentId = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_AGENT_ID); public static final String ENTERPRISE_WE_CHAT_AGENT_ID = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_AGENT_ID);
public static final String enterpriseWeChatUsers = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USERS); public static final String ENTERPRISE_WE_CHAT_USERS = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USERS);
/** /**
* get Enterprise WeChat is enable * get Enterprise WeChat is enable
@ -87,7 +87,7 @@ public class EnterpriseWeChatUtils {
CloseableHttpClient httpClient = HttpClients.createDefault(); CloseableHttpClient httpClient = HttpClients.createDefault();
try { try {
HttpGet httpGet = new HttpGet(enterpriseWeChatTokenUrlReplace); HttpGet httpGet = new HttpGet(ENTERPRISE_WE_CHAT_TOKEN_URL_REPLACE);
CloseableHttpResponse response = httpClient.execute(httpGet); CloseableHttpResponse response = httpClient.execute(httpGet);
try { try {
HttpEntity entity = response.getEntity(); HttpEntity entity = response.getEntity();
@ -114,7 +114,7 @@ public class EnterpriseWeChatUtils {
* @return Enterprise WeChat send message * @return Enterprise WeChat send message
*/ */
public static String makeTeamSendMsg(String toParty, String agentId, String msg) { public static String makeTeamSendMsg(String toParty, String agentId, String msg) {
return enterpriseWeChatTeamSendMsg.replaceAll("\\$toParty", toParty) return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\$toParty", toParty)
.replaceAll("\\$agentId", agentId) .replaceAll("\\$agentId", agentId)
.replaceAll("\\$msg", msg); .replaceAll("\\$msg", msg);
} }
@ -128,7 +128,7 @@ public class EnterpriseWeChatUtils {
*/ */
public static String makeTeamSendMsg(Collection<String> toParty, String agentId, String msg) { public static String makeTeamSendMsg(Collection<String> toParty, String agentId, String msg) {
String listParty = FuncUtils.mkString(toParty, "|"); String listParty = FuncUtils.mkString(toParty, "|");
return enterpriseWeChatTeamSendMsg.replaceAll("\\$toParty", listParty) return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\$toParty", listParty)
.replaceAll("\\$agentId", agentId) .replaceAll("\\$agentId", agentId)
.replaceAll("\\$msg", msg); .replaceAll("\\$msg", msg);
} }
@ -141,7 +141,7 @@ public class EnterpriseWeChatUtils {
* @return Enterprise WeChat send message * @return Enterprise WeChat send message
*/ */
public static String makeUserSendMsg(String toUser, String agentId, String msg) { public static String makeUserSendMsg(String toUser, String agentId, String msg) {
return enterpriseWeChatUserSendMsg.replaceAll("\\$toUser", toUser) return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\$toUser", toUser)
.replaceAll("\\$agentId", agentId) .replaceAll("\\$agentId", agentId)
.replaceAll("\\$msg", msg); .replaceAll("\\$msg", msg);
} }
@ -155,7 +155,7 @@ public class EnterpriseWeChatUtils {
*/ */
public static String makeUserSendMsg(Collection<String> toUser, String agentId, String msg) { public static String makeUserSendMsg(Collection<String> toUser, String agentId, String msg) {
String listUser = FuncUtils.mkString(toUser, "|"); String listUser = FuncUtils.mkString(toUser, "|");
return enterpriseWeChatUserSendMsg.replaceAll("\\$toUser", listUser) return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\$toUser", listUser)
.replaceAll("\\$agentId", agentId) .replaceAll("\\$agentId", agentId)
.replaceAll("\\$msg", msg); .replaceAll("\\$msg", msg);
} }
@ -169,7 +169,7 @@ public class EnterpriseWeChatUtils {
* @throws IOException the IOException * @throws IOException the IOException
*/ */
public static String sendEnterpriseWeChat(String charset, String data, String token) throws IOException { public static String sendEnterpriseWeChat(String charset, String data, String token) throws IOException {
String enterpriseWeChatPushUrlReplace = enterpriseWeChatPushUrl.replaceAll("\\$token", token); String enterpriseWeChatPushUrlReplace = ENTERPRISE_WE_CHAT_PUSH_URL.replaceAll("\\$token", token);
CloseableHttpClient httpClient = HttpClients.createDefault(); CloseableHttpClient httpClient = HttpClients.createDefault();
try { try {
@ -185,7 +185,7 @@ public class EnterpriseWeChatUtils {
response.close(); response.close();
} }
logger.info("Enterprise WeChat send [{}], param:{}, resp:{}", logger.info("Enterprise WeChat send [{}], param:{}, resp:{}",
enterpriseWeChatPushUrl, data, resp); ENTERPRISE_WE_CHAT_PUSH_URL, data, resp);
return resp; return resp;
} finally { } finally {
httpClient.close(); httpClient.close();

8
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java

@ -53,7 +53,7 @@ public class EnterpriseWeChatUtilsTest {
String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token);
String errmsg = JSON.parseObject(resp).getString("errmsg"); String errmsg = JSON.parseObject(resp).getString("errmsg");
Assert.assertEquals(errmsg, "ok"); Assert.assertEquals("ok",errmsg);
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }
@ -68,7 +68,7 @@ public class EnterpriseWeChatUtilsTest {
String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token);
String errmsg = JSON.parseObject(resp).getString("errmsg"); String errmsg = JSON.parseObject(resp).getString("errmsg");
Assert.assertEquals(errmsg, "ok"); Assert.assertEquals("ok",errmsg);
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }
@ -95,7 +95,7 @@ public class EnterpriseWeChatUtilsTest {
String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token);
String errmsg = JSON.parseObject(resp).getString("errmsg"); String errmsg = JSON.parseObject(resp).getString("errmsg");
Assert.assertEquals(errmsg, "ok"); Assert.assertEquals("ok",errmsg);
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }
@ -110,7 +110,7 @@ public class EnterpriseWeChatUtilsTest {
String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token);
String errmsg = JSON.parseObject(resp).getString("errmsg"); String errmsg = JSON.parseObject(resp).getString("errmsg");
Assert.assertEquals(errmsg, "ok"); Assert.assertEquals("ok",errmsg);
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }

1
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java

@ -17,7 +17,6 @@
package org.apache.dolphinscheduler.alert.utils; package org.apache.dolphinscheduler.alert.utils;
import org.junit.After;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.slf4j.Logger; import org.slf4j.Logger;

6
dolphinscheduler-api/pom.xml

@ -140,6 +140,12 @@
<dependency> <dependency>
<groupId>org.apache.curator</groupId> <groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId> <artifactId>curator-recipes</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<!-- hadoop --> <!-- hadoop -->

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java

@ -103,7 +103,7 @@ public class DataAnalysisController extends BaseController{
@RequestParam(value="endDate", required=false) String endDate, @RequestParam(value="endDate", required=false) String endDate,
@RequestParam(value="projectId", required=false, defaultValue = "0") int projectId){ @RequestParam(value="projectId", required=false, defaultValue = "0") int projectId){
try{ try{
logger.info("count process instance state, user:{}, start date: {}, end date:{}, project id", logger.info("count process instance state, user:{}, start date: {}, end date:{}, project id:{}",
loginUser.getUserName(), startDate, endDate, projectId); loginUser.getUserName(), startDate, endDate, projectId);
Map<String, Object> result = dataAnalysisService.countProcessInstanceStateByProject(loginUser, projectId, startDate, endDate); Map<String, Object> result = dataAnalysisService.countProcessInstanceStateByProject(loginUser, projectId, startDate, endDate);
return returnDataList(result); return returnDataList(result);
@ -129,7 +129,7 @@ public class DataAnalysisController extends BaseController{
public Result countDefinitionByUser(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, public Result countDefinitionByUser(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value="projectId", required=false, defaultValue = "0") int projectId){ @RequestParam(value="projectId", required=false, defaultValue = "0") int projectId){
try{ try{
logger.info("count process definition , user:{}, project id", logger.info("count process definition , user:{}, project id:{}",
loginUser.getUserName(), projectId); loginUser.getUserName(), projectId);
Map<String, Object> result = dataAnalysisService.countDefinitionByUser(loginUser, projectId); Map<String, Object> result = dataAnalysisService.countDefinitionByUser(loginUser, projectId);
return returnDataList(result); return returnDataList(result);

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java

@ -149,7 +149,7 @@ public class ExecutorController extends BaseController {
) { ) {
try { try {
logger.info("execute command, login user: {}, project:{}, process instance id:{}, execute type:{}", logger.info("execute command, login user: {}, project:{}, process instance id:{}, execute type:{}",
loginUser.getUserName(), projectName, processInstanceId, executeType.toString()); loginUser.getUserName(), projectName, processInstanceId, executeType);
Map<String, Object> result = execService.execute(loginUser, projectName, processInstanceId, executeType); Map<String, Object> result = execService.execute(loginUser, projectName, processInstanceId, executeType);
return returnDataList(result); return returnDataList(result);
} catch (Exception e) { } catch (Exception e) {
@ -173,7 +173,7 @@ public class ExecutorController extends BaseController {
@ResponseStatus(HttpStatus.OK) @ResponseStatus(HttpStatus.OK)
public Result startCheckProcessDefinition(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, public Result startCheckProcessDefinition(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "processDefinitionId") int processDefinitionId) { @RequestParam(value = "processDefinitionId") int processDefinitionId) {
logger.info("login user {}, check process definition", loginUser.getUserName(), processDefinitionId); logger.info("login user {}, check process definition {}", loginUser.getUserName(), processDefinitionId);
try { try {
Map<String, Object> result = execService.startCheckByProcessDefinedId(processDefinitionId); Map<String, Object> result = execService.startCheckByProcessDefinedId(processDefinitionId);
return returnDataList(result); return returnDataList(result);

15
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java

@ -259,10 +259,7 @@ public class ExecutorService extends BaseService{
// checkTenantExists(); // checkTenantExists();
Tenant tenant = processService.getTenantForProcess(processDefinition.getTenantId(), Tenant tenant = processService.getTenantForProcess(processDefinition.getTenantId(),
processDefinition.getUserId()); processDefinition.getUserId());
if(tenant == null){ return tenant != null;
return false;
}
return true;
} }
/** /**
@ -298,6 +295,7 @@ public class ExecutorService extends BaseService{
if (executionStatus.typeIsPause()|| executionStatus.typeIsCancel()) { if (executionStatus.typeIsPause()|| executionStatus.typeIsCancel()) {
checkResult = true; checkResult = true;
} }
break;
default: default:
break; break;
} }
@ -369,7 +367,7 @@ public class ExecutorService extends BaseService{
* @return check result code * @return check result code
*/ */
public Map<String, Object> startCheckByProcessDefinedId(int processDefineId) { public Map<String, Object> startCheckByProcessDefinedId(int processDefineId) {
Map<String, Object> result = new HashMap<String, Object>(); Map<String, Object> result = new HashMap<>();
if (processDefineId == 0){ if (processDefineId == 0){
logger.error("process definition id is null"); logger.error("process definition id is null");
@ -378,10 +376,9 @@ public class ExecutorService extends BaseService{
List<Integer> ids = new ArrayList<>(); List<Integer> ids = new ArrayList<>();
processService.recurseFindSubProcessId(processDefineId, ids); processService.recurseFindSubProcessId(processDefineId, ids);
Integer[] idArray = ids.toArray(new Integer[ids.size()]); Integer[] idArray = ids.toArray(new Integer[ids.size()]);
if (ids.size() > 0){ if (!ids.isEmpty()){
List<ProcessDefinition> processDefinitionList; List<ProcessDefinition> processDefinitionList = processDefinitionMapper.queryDefinitionListByIdList(idArray);
processDefinitionList = processDefinitionMapper.queryDefinitionListByIdList(idArray); if (processDefinitionList != null){
if (processDefinitionList != null && processDefinitionList.size() > 0){
for (ProcessDefinition processDefinition : processDefinitionList){ for (ProcessDefinition processDefinition : processDefinitionList){
/** /**
* if there is no online process, exit directly * if there is no online process, exit directly

1
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java

@ -28,7 +28,6 @@ import org.apache.dolphinscheduler.dao.entity.ZookeeperRecord;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;

1
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/LoginControllerTest.java

@ -28,7 +28,6 @@ import org.springframework.test.web.servlet.MvcResult;
import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap; import org.springframework.util.MultiValueMap;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;

1
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/MonitorControllerTest.java

@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.controller;
import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import com.alibaba.fastjson.JSONObject;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.slf4j.Logger; import org.slf4j.Logger;

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java

@ -29,8 +29,6 @@ import org.springframework.test.web.servlet.MvcResult;
import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap; import org.springframework.util.MultiValueMap;
import javax.ws.rs.POST;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java

@ -18,13 +18,11 @@ package org.apache.dolphinscheduler.api.controller;
import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;

5
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskRecordControllerTest.java

@ -18,9 +18,6 @@ package org.apache.dolphinscheduler.api.controller;
import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.enums.FailureStrategy;
import org.apache.dolphinscheduler.common.enums.Priority;
import org.apache.dolphinscheduler.common.enums.WarningType;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
@ -31,9 +28,7 @@ import org.springframework.test.web.servlet.MvcResult;
import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap; import org.springframework.util.MultiValueMap;
import static org.junit.Assert.*;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;

1
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java

@ -29,7 +29,6 @@ import org.springframework.test.web.servlet.MvcResult;
import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap; import org.springframework.util.MultiValueMap;
import static org.junit.Assert.*;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;

8
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java

@ -175,6 +175,14 @@ public class TenantServiceTest {
logger.info(result.toString()); logger.info(result.toString());
List<Tenant> tenantList = (List<Tenant>) result.get(Constants.DATA_LIST); List<Tenant> tenantList = (List<Tenant>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(tenantList)); Assert.assertTrue(CollectionUtils.isNotEmpty(tenantList));
Mockito.when( tenantMapper.queryByTenantCode("1")).thenReturn(getList());
Map<String, Object> successRes = tenantService.queryTenantList("1");
Assert.assertEquals(Status.SUCCESS,successRes.get(Constants.STATUS));
Mockito.when( tenantMapper.queryByTenantCode("1")).thenReturn(null);
Map<String, Object> tenantNotExistRes = tenantService.queryTenantList("1");
Assert.assertEquals(Status.TENANT_NOT_EXIST,tenantNotExistRes.get(Constants.STATUS));
} }
@Test @Test

1
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMainTest.java

@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.utils;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.mockito.InjectMocks; import org.mockito.InjectMocks;

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitorUtilsTest.java

@ -28,7 +28,7 @@ public class ZookeeperMonitorUtilsTest {
@Test @Test
public void testGetMasterLsit(){ public void testGetMasterList(){
ZookeeperMonitor zookeeperMonitor = new ZookeeperMonitor(); ZookeeperMonitor zookeeperMonitor = new ZookeeperMonitor();

2
dolphinscheduler-common/pom.xml

@ -25,7 +25,7 @@
</parent> </parent>
<artifactId>dolphinscheduler-common</artifactId> <artifactId>dolphinscheduler-common</artifactId>
<name>dolphinscheduler-common</name> <name>dolphinscheduler-common</name>
<url>http://maven.apache.org</url>
<packaging>jar</packaging> <packaging>jar</packaging>
<properties> <properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@ -746,7 +746,7 @@ public final class Constants {
* application regex * application regex
*/ */
public static final String APPLICATION_REGEX = "application_\\d+_\\d+"; public static final String APPLICATION_REGEX = "application_\\d+_\\d+";
public static final String PID = "pid"; public static final String PID = OSUtils.isWindows() ? "handle" : "pid";
/** /**
* month_begin * month_begin
*/ */

3
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/WorkerLogFilter.java

@ -20,9 +20,6 @@ import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.filter.Filter; import ch.qos.logback.core.filter.Filter;
import ch.qos.logback.core.spi.FilterReply; import ch.qos.logback.core.spi.FilterReply;
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import java.util.Arrays;
/** /**
* worker log filter * worker log filter

16
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java

@ -400,8 +400,7 @@ public class OSUtils {
* @return true if mac * @return true if mac
*/ */
public static boolean isMacOS() { public static boolean isMacOS() {
String os = System.getProperty("os.name"); return getOSName().startsWith("Mac");
return os.startsWith("Mac");
} }
@ -409,9 +408,16 @@ public class OSUtils {
* whether is windows * whether is windows
* @return true if windows * @return true if windows
*/ */
public static boolean isWindows() { public static boolean isWindows() { ;
String os = System.getProperty("os.name"); return getOSName().startsWith("Windows");
return os.startsWith("Windows"); }
/**
* get current OS name
* @return current OS name
*/
public static String getOSName() {
return System.getProperty("os.name");
} }
/** /**

14
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java

@ -119,6 +119,11 @@ public class ParameterUtils {
*/ */
public static String curingGlobalParams(Map<String,String> globalParamMap, List<Property> globalParamList, public static String curingGlobalParams(Map<String,String> globalParamMap, List<Property> globalParamList,
CommandType commandType, Date scheduleTime){ CommandType commandType, Date scheduleTime){
if (globalParamList == null || globalParamList.isEmpty()) {
return null;
}
Map<String, String> globalMap = new HashMap<>(); Map<String, String> globalMap = new HashMap<>();
if (globalParamMap!= null){ if (globalParamMap!= null){
globalMap.putAll(globalParamMap); globalMap.putAll(globalParamMap);
@ -132,9 +137,7 @@ public class ParameterUtils {
allParamMap.putAll(timeParams); allParamMap.putAll(timeParams);
} }
if (globalMap != null) {
allParamMap.putAll(globalMap); allParamMap.putAll(globalMap);
}
Set<Map.Entry<String, String>> entries = allParamMap.entrySet(); Set<Map.Entry<String, String>> entries = allParamMap.entrySet();
@ -146,12 +149,7 @@ public class ParameterUtils {
resolveMap.put(entry.getKey(),str); resolveMap.put(entry.getKey(),str);
} }
} }
if (globalMap != null){
globalMap.putAll(resolveMap); globalMap.putAll(resolveMap);
}
if (globalParamList != null && globalParamList.size() > 0){
for (Property property : globalParamList){ for (Property property : globalParamList){
String val = globalMap.get(property.getProp()); String val = globalMap.get(property.getProp());
@ -161,8 +159,6 @@ public class ParameterUtils {
} }
return JSONObject.toJSONString(globalParamList); return JSONObject.toJSONString(globalParamList);
} }
return null;
}
/** /**

1065
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32.java

File diff suppressed because it is too large Load Diff

286
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32.java

@ -0,0 +1,286 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils.process;
import com.sun.jna.platform.win32.Kernel32Util;
import java.util.*;
final class ProcessEnvironmentForWin32 extends HashMap<String,String> {
private static final long serialVersionUID = -8017839552603542824L;
private static String validateName(String name) {
// An initial `=' indicates a magic Windows variable name -- OK
if (name.indexOf('=', 1) != -1 ||
name.indexOf('\u0000') != -1)
throw new IllegalArgumentException
("Invalid environment variable name: \"" + name + "\"");
return name;
}
private static String validateValue(String value) {
if (value.indexOf('\u0000') != -1)
throw new IllegalArgumentException
("Invalid environment variable value: \"" + value + "\"");
return value;
}
private static String nonNullString(Object o) {
if (o == null)
throw new NullPointerException();
return (String) o;
}
public String put(String key, String value) {
return super.put(validateName(key), validateValue(value));
}
public String get(Object key) {
return super.get(nonNullString(key));
}
public boolean containsKey(Object key) {
return super.containsKey(nonNullString(key));
}
public boolean containsValue(Object value) {
return super.containsValue(nonNullString(value));
}
public String remove(Object key) {
return super.remove(nonNullString(key));
}
private static class CheckedEntry implements Entry<String,String> {
private final Entry<String,String> e;
public CheckedEntry(Entry<String,String> e) {this.e = e;}
public String getKey() { return e.getKey();}
public String getValue() { return e.getValue();}
public String setValue(String value) {
return e.setValue(validateValue(value));
}
public String toString() { return getKey() + "=" + getValue();}
public boolean equals(Object o) {return e.equals(o);}
public int hashCode() {return e.hashCode();}
}
private static class CheckedEntrySet extends AbstractSet<Entry<String,String>> {
private final Set<Entry<String,String>> s;
public CheckedEntrySet(Set<Entry<String,String>> s) {this.s = s;}
public int size() {return s.size();}
public boolean isEmpty() {return s.isEmpty();}
public void clear() { s.clear();}
public Iterator<Entry<String,String>> iterator() {
return new Iterator<Entry<String,String>>() {
Iterator<Entry<String,String>> i = s.iterator();
public boolean hasNext() { return i.hasNext();}
public Entry<String,String> next() {
return new CheckedEntry(i.next());
}
public void remove() { i.remove();}
};
}
private static Entry<String,String> checkedEntry(Object o) {
@SuppressWarnings("unchecked")
Entry<String,String> e = (Entry<String,String>) o;
nonNullString(e.getKey());
nonNullString(e.getValue());
return e;
}
public boolean contains(Object o) {return s.contains(checkedEntry(o));}
public boolean remove(Object o) {return s.remove(checkedEntry(o));}
}
private static class CheckedValues extends AbstractCollection<String> {
private final Collection<String> c;
public CheckedValues(Collection<String> c) {this.c = c;}
public int size() {return c.size();}
public boolean isEmpty() {return c.isEmpty();}
public void clear() { c.clear();}
public Iterator<String> iterator() {return c.iterator();}
public boolean contains(Object o) {return c.contains(nonNullString(o));}
public boolean remove(Object o) {return c.remove(nonNullString(o));}
}
private static class CheckedKeySet extends AbstractSet<String> {
private final Set<String> s;
public CheckedKeySet(Set<String> s) {this.s = s;}
public int size() {return s.size();}
public boolean isEmpty() {return s.isEmpty();}
public void clear() { s.clear();}
public Iterator<String> iterator() {return s.iterator();}
public boolean contains(Object o) {return s.contains(nonNullString(o));}
public boolean remove(Object o) {return s.remove(nonNullString(o));}
}
public Set<String> keySet() {
return new CheckedKeySet(super.keySet());
}
public Collection<String> values() {
return new CheckedValues(super.values());
}
public Set<Entry<String,String>> entrySet() {
return new CheckedEntrySet(super.entrySet());
}
private static final class NameComparator implements Comparator<String> {
public int compare(String s1, String s2) {
// We can't use String.compareToIgnoreCase since it
// canonicalizes to lower case, while Windows
// canonicalizes to upper case! For example, "_" should
// sort *after* "Z", not before.
int n1 = s1.length();
int n2 = s2.length();
int min = Math.min(n1, n2);
for (int i = 0; i < min; i++) {
char c1 = s1.charAt(i);
char c2 = s2.charAt(i);
if (c1 != c2) {
c1 = Character.toUpperCase(c1);
c2 = Character.toUpperCase(c2);
if (c1 != c2)
// No overflow because of numeric promotion
return c1 - c2;
}
}
return n1 - n2;
}
}
private static final class EntryComparator implements Comparator<Entry<String,String>> {
public int compare(Entry<String,String> e1,
Entry<String,String> e2) {
return nameComparator.compare(e1.getKey(), e2.getKey());
}
}
// Allow `=' as first char in name, e.g. =C:=C:\DIR
static final int MIN_NAME_LENGTH = 1;
private static final NameComparator nameComparator;
private static final EntryComparator entryComparator;
private static final ProcessEnvironmentForWin32 theEnvironment;
private static final Map<String,String> theUnmodifiableEnvironment;
private static final Map<String,String> theCaseInsensitiveEnvironment;
static {
nameComparator = new NameComparator();
entryComparator = new EntryComparator();
theEnvironment = new ProcessEnvironmentForWin32();
theUnmodifiableEnvironment = Collections.unmodifiableMap(theEnvironment);
theEnvironment.putAll(environmentBlock());
theCaseInsensitiveEnvironment = new TreeMap<>(nameComparator);
theCaseInsensitiveEnvironment.putAll(theEnvironment);
}
private ProcessEnvironmentForWin32() {
super();
}
private ProcessEnvironmentForWin32(int capacity) {
super(capacity);
}
// Only for use by System.getenv(String)
static String getenv(String name) {
// The original implementation used a native call to _wgetenv,
// but it turns out that _wgetenv is only consistent with
// GetEnvironmentStringsW (for non-ASCII) if `wmain' is used
// instead of `main', even in a process created using
// CREATE_UNICODE_ENVIRONMENT. Instead we perform the
// case-insensitive comparison ourselves. At least this
// guarantees that System.getenv().get(String) will be
// consistent with System.getenv(String).
return theCaseInsensitiveEnvironment.get(name);
}
// Only for use by System.getenv()
static Map<String,String> getenv() {
return theUnmodifiableEnvironment;
}
// Only for use by ProcessBuilder.environment()
@SuppressWarnings("unchecked")
static Map<String,String> environment() {
return (Map<String,String>) theEnvironment.clone();
}
// Only for use by ProcessBuilder.environment(String[] envp)
static Map<String,String> emptyEnvironment(int capacity) {
return new ProcessEnvironmentForWin32(capacity);
}
private static Map<String, String> environmentBlock() {
return Kernel32Util.getEnvironmentVariables();
}
// Only for use by ProcessImpl.start()
String toEnvironmentBlock() {
// Sort Unicode-case-insensitively by name
List<Entry<String,String>> list = new ArrayList<>(entrySet());
Collections.sort(list, entryComparator);
StringBuilder sb = new StringBuilder(size()*30);
int cmp = -1;
// Some versions of MSVCRT.DLL require SystemRoot to be set.
// So, we make sure that it is always set, even if not provided
// by the caller.
final String SYSTEMROOT = "SystemRoot";
for (Entry<String,String> e : list) {
String key = e.getKey();
String value = e.getValue();
if (cmp < 0 && (cmp = nameComparator.compare(key, SYSTEMROOT)) > 0) {
// Not set, so add it here
addToEnvIfSet(sb, SYSTEMROOT);
}
addToEnv(sb, key, value);
}
if (cmp < 0) {
// Got to end of list and still not found
addToEnvIfSet(sb, SYSTEMROOT);
}
if (sb.length() == 0) {
// Environment was empty and SystemRoot not set in parent
sb.append('\u0000');
}
// Block is double NUL terminated
sb.append('\u0000');
return sb.toString();
}
// add the environment variable to the child, if it exists in parent
private static void addToEnvIfSet(StringBuilder sb, String name) {
String s = getenv(name);
if (s != null)
addToEnv(sb, name, s);
}
private static void addToEnv(StringBuilder sb, String name, String val) {
sb.append(name).append('=').append(val).append('\u0000');
}
static String toEnvironmentBlock(Map<String,String> map) {
return map == null ? null : ((ProcessEnvironmentForWin32)map).toEnvironmentBlock();
}
}

785
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32.java

@ -0,0 +1,785 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils.process;
import com.sun.jna.Pointer;
import com.sun.jna.platform.win32.*;
import com.sun.jna.ptr.IntByReference;
import java.lang.reflect.Field;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import sun.security.action.GetPropertyAction;
import java.io.*;
import java.security.AccessController;
import java.security.PrivilegedAction;
import java.util.ArrayList;
import java.util.Locale;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static com.sun.jna.platform.win32.WinBase.STILL_ACTIVE;
import static java.util.Objects.requireNonNull;
public class ProcessImplForWin32 extends Process {
private static final Field FD_HANDLE;
static {
if (!OSUtils.isWindows()) {
throw new RuntimeException("ProcessImplForWin32 can be only initialized in " +
"Windows environment, but current OS is " + OSUtils.getOSName());
}
try {
FD_HANDLE = requireNonNull(FileDescriptor.class.getDeclaredField("handle"));
FD_HANDLE.setAccessible(true);
} catch (NoSuchFieldException e) {
throw new RuntimeException(e);
}
}
private static final int PIPE_SIZE = 4096 + 24;
private static final int HANDLE_STORAGE_SIZE = 6;
private static final int OFFSET_READ = 0;
private static final int OFFSET_WRITE = 1;
private static final WinNT.HANDLE JAVA_INVALID_HANDLE_VALUE = new WinNT.HANDLE(Pointer.createConstant(-1));
private static void setHandle(FileDescriptor obj, long handle) {
try {
FD_HANDLE.set(obj, handle);
} catch (IllegalAccessException e) {
throw new RuntimeException(e);
}
}
private static long getHandle(FileDescriptor obj) {
try {
return (Long) FD_HANDLE.get(obj);
} catch (IllegalAccessException e) {
throw new RuntimeException(e);
}
}
/**
* Open a file for writing. If {@code append} is {@code true} then the file
* is opened for atomic append directly and a FileOutputStream constructed
* with the resulting handle. This is because a FileOutputStream created
* to append to a file does not open the file in a manner that guarantees
* that writes by the child process will be atomic.
*/
private static FileOutputStream newFileOutputStream(File f, boolean append)
throws IOException
{
if (append) {
String path = f.getPath();
SecurityManager sm = System.getSecurityManager();
if (sm != null)
sm.checkWrite(path);
long handle = openForAtomicAppend(path);
final FileDescriptor fd = new FileDescriptor();
setHandle(fd, handle);
return AccessController.doPrivileged(
new PrivilegedAction<FileOutputStream>() {
public FileOutputStream run() {
return new FileOutputStream(fd);
}
}
);
} else {
return new FileOutputStream(f);
}
}
// System-dependent portion of ProcessBuilderForWindows.start()
static Process start(String username,
String password,
String cmdarray[],
java.util.Map<String,String> environment,
String dir,
ProcessBuilderForWin32.Redirect[] redirects,
boolean redirectErrorStream)
throws IOException
{
String envblock = ProcessEnvironmentForWin32.toEnvironmentBlock(environment);
FileInputStream f0 = null;
FileOutputStream f1 = null;
FileOutputStream f2 = null;
try {
long[] stdHandles;
if (redirects == null) {
stdHandles = new long[] { -1L, -1L, -1L };
} else {
stdHandles = new long[3];
if (redirects[0] == ProcessBuilderForWin32.Redirect.PIPE)
stdHandles[0] = -1L;
else if (redirects[0] == ProcessBuilderForWin32.Redirect.INHERIT)
stdHandles[0] = getHandle(FileDescriptor.in);
else {
f0 = new FileInputStream(redirects[0].file());
stdHandles[0] = getHandle(f0.getFD());
}
if (redirects[1] == ProcessBuilderForWin32.Redirect.PIPE)
stdHandles[1] = -1L;
else if (redirects[1] == ProcessBuilderForWin32.Redirect.INHERIT)
stdHandles[1] = getHandle(FileDescriptor.out);
else {
f1 = newFileOutputStream(redirects[1].file(),
redirects[1].append());
stdHandles[1] = getHandle(f1.getFD());
}
if (redirects[2] == ProcessBuilderForWin32.Redirect.PIPE)
stdHandles[2] = -1L;
else if (redirects[2] == ProcessBuilderForWin32.Redirect.INHERIT)
stdHandles[2] = getHandle(FileDescriptor.err);
else {
f2 = newFileOutputStream(redirects[2].file(),
redirects[2].append());
stdHandles[2] = getHandle(f2.getFD());
}
}
return new ProcessImplForWin32(username, password, cmdarray, envblock, dir, stdHandles, redirectErrorStream);
} finally {
// In theory, close() can throw IOException
// (although it is rather unlikely to happen here)
try { if (f0 != null) f0.close(); }
finally {
try { if (f1 != null) f1.close(); }
finally { if (f2 != null) f2.close(); }
}
}
}
private static class LazyPattern {
// Escape-support version:
// "(\")((?:\\\\\\1|.)+?)\\1|([^\\s\"]+)";
private static final Pattern PATTERN =
Pattern.compile("[^\\s\"]+|\"[^\"]*\"");
};
/* Parses the command string parameter into the executable name and
* program arguments.
*
* The command string is broken into tokens. The token separator is a space
* or quota character. The space inside quotation is not a token separator.
* There are no escape sequences.
*/
private static String[] getTokensFromCommand(String command) {
ArrayList<String> matchList = new ArrayList<>(8);
Matcher regexMatcher = ProcessImplForWin32.LazyPattern.PATTERN.matcher(command);
while (regexMatcher.find())
matchList.add(regexMatcher.group());
return matchList.toArray(new String[matchList.size()]);
}
private static final int VERIFICATION_CMD_BAT = 0;
private static final int VERIFICATION_WIN32 = 1;
private static final int VERIFICATION_WIN32_SAFE = 2; // inside quotes not allowed
private static final int VERIFICATION_LEGACY = 3;
// See Command shell overview for documentation of special characters.
// https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-xp/bb490954(v=technet.10)
private static final char ESCAPE_VERIFICATION[][] = {
// We guarantee the only command file execution for implicit [cmd.exe] run.
// http://technet.microsoft.com/en-us/library/bb490954.aspx
{' ', '\t', '<', '>', '&', '|', '^'},
{' ', '\t', '<', '>'},
{' ', '\t', '<', '>'},
{' ', '\t'}
};
private static String createCommandLine(int verificationType,
final String executablePath,
final String cmd[])
{
StringBuilder cmdbuf = new StringBuilder(80);
cmdbuf.append(executablePath);
for (int i = 1; i < cmd.length; ++i) {
cmdbuf.append(' ');
String s = cmd[i];
if (needsEscaping(verificationType, s)) {
cmdbuf.append('"');
if (verificationType == VERIFICATION_WIN32_SAFE) {
// Insert the argument, adding '\' to quote any interior quotes
int length = s.length();
for (int j = 0; j < length; j++) {
char c = s.charAt(j);
if (c == DOUBLEQUOTE) {
int count = countLeadingBackslash(verificationType, s, j);
while (count-- > 0) {
cmdbuf.append(BACKSLASH); // double the number of backslashes
}
cmdbuf.append(BACKSLASH); // backslash to quote the quote
}
cmdbuf.append(c);
}
} else {
cmdbuf.append(s);
}
// The code protects the [java.exe] and console command line
// parser, that interprets the [\"] combination as an escape
// sequence for the ["] char.
// http://msdn.microsoft.com/en-us/library/17w5ykft.aspx
//
// If the argument is an FS path, doubling of the tail [\]
// char is not a problem for non-console applications.
//
// The [\"] sequence is not an escape sequence for the [cmd.exe]
// command line parser. The case of the [""] tail escape
// sequence could not be realized due to the argument validation
// procedure.
int count = countLeadingBackslash(verificationType, s, s.length());
while (count-- > 0) {
cmdbuf.append(BACKSLASH); // double the number of backslashes
}
cmdbuf.append('"');
} else {
cmdbuf.append(s);
}
}
return cmdbuf.toString();
}
/**
* Return the argument without quotes (1st and last) if present, else the arg.
* @param str a string
* @return the string without 1st and last quotes
*/
private static String unQuote(String str) {
int len = str.length();
return (len >= 2 && str.charAt(0) == DOUBLEQUOTE && str.charAt(len - 1) == DOUBLEQUOTE)
? str.substring(1, len - 1)
: str;
}
private static boolean needsEscaping(int verificationType, String arg) {
// Switch off MS heuristic for internal ["].
// Please, use the explicit [cmd.exe] call
// if you need the internal ["].
// Example: "cmd.exe", "/C", "Extended_MS_Syntax"
// For [.exe] or [.com] file the unpaired/internal ["]
// in the argument is not a problem.
String unquotedArg = unQuote(arg);
boolean argIsQuoted = !arg.equals(unquotedArg);
boolean embeddedQuote = unquotedArg.indexOf(DOUBLEQUOTE) >= 0;
switch (verificationType) {
case VERIFICATION_CMD_BAT:
if (embeddedQuote) {
throw new IllegalArgumentException("Argument has embedded quote, " +
"use the explicit CMD.EXE call.");
}
break; // break determine whether to quote
case VERIFICATION_WIN32_SAFE:
if (argIsQuoted && embeddedQuote) {
throw new IllegalArgumentException("Malformed argument has embedded quote: "
+ unquotedArg);
}
break;
default:
break;
}
if (!argIsQuoted) {
char testEscape[] = ESCAPE_VERIFICATION[verificationType];
for (int i = 0; i < testEscape.length; ++i) {
if (arg.indexOf(testEscape[i]) >= 0) {
return true;
}
}
}
return false;
}
private static String getExecutablePath(String path)
throws IOException
{
String name = unQuote(path);
if (name.indexOf(DOUBLEQUOTE) >= 0) {
throw new IllegalArgumentException("Executable name has embedded quote, " +
"split the arguments: " + name);
}
// Win32 CreateProcess requires path to be normalized
File fileToRun = new File(name);
// From the [CreateProcess] function documentation:
//
// "If the file name does not contain an extension, .exe is appended.
// Therefore, if the file name extension is .com, this parameter
// must include the .com extension. If the file name ends in
// a period (.) with no extension, or if the file name contains a path,
// .exe is not appended."
//
// "If the file name !does not contain a directory path!,
// the system searches for the executable file in the following
// sequence:..."
//
// In practice ANY non-existent path is extended by [.exe] extension
// in the [CreateProcess] function with the only exception:
// the path ends by (.)
return fileToRun.getPath();
}
/**
* An executable is any program that is an EXE or does not have an extension
* and the Windows createProcess will be looking for .exe.
* The comparison is case insensitive based on the name.
* @param executablePath the executable file
* @return true if the path ends in .exe or does not have an extension.
*/
private boolean isExe(String executablePath) {
File file = new File(executablePath);
String upName = file.getName().toUpperCase(Locale.ROOT);
return (upName.endsWith(".EXE") || upName.indexOf('.') < 0);
}
// Old version that can be bypassed
private boolean isShellFile(String executablePath) {
String upPath = executablePath.toUpperCase();
return (upPath.endsWith(".CMD") || upPath.endsWith(".BAT"));
}
private String quoteString(String arg) {
StringBuilder argbuf = new StringBuilder(arg.length() + 2);
return argbuf.append('"').append(arg).append('"').toString();
}
// Count backslashes before start index of string.
// .bat files don't include backslashes as part of the quote
private static int countLeadingBackslash(int verificationType,
CharSequence input, int start) {
if (verificationType == VERIFICATION_CMD_BAT)
return 0;
int j;
for (j = start - 1; j >= 0 && input.charAt(j) == BACKSLASH; j--) {
// just scanning backwards
}
return (start - 1) - j; // number of BACKSLASHES
}
private static final char DOUBLEQUOTE = '\"';
private static final char BACKSLASH = '\\';
private WinNT.HANDLE handle;
private OutputStream stdin_stream;
private InputStream stdout_stream;
private InputStream stderr_stream;
private ProcessImplForWin32(
String username,
String password,
String cmd[],
final String envblock,
final String path,
final long[] stdHandles,
final boolean redirectErrorStream)
throws IOException
{
String cmdstr;
final SecurityManager security = System.getSecurityManager();
GetPropertyAction action = new GetPropertyAction("jdk.lang.Process.allowAmbiguousCommands",
(security == null) ? "true" : "false");
final boolean allowAmbiguousCommands = !"false".equalsIgnoreCase(action.run());
if (allowAmbiguousCommands && security == null) {
// Legacy mode.
// Normalize path if possible.
String executablePath = new File(cmd[0]).getPath();
// No worry about internal, unpaired ["], and redirection/piping.
if (needsEscaping(VERIFICATION_LEGACY, executablePath) )
executablePath = quoteString(executablePath);
cmdstr = createCommandLine(
//legacy mode doesn't worry about extended verification
VERIFICATION_LEGACY,
executablePath,
cmd);
} else {
String executablePath;
try {
executablePath = getExecutablePath(cmd[0]);
} catch (IllegalArgumentException e) {
// Workaround for the calls like
// Runtime.getRuntime().exec("\"C:\\Program Files\\foo\" bar")
// No chance to avoid CMD/BAT injection, except to do the work
// right from the beginning. Otherwise we have too many corner
// cases from
// Runtime.getRuntime().exec(String[] cmd [, ...])
// calls with internal ["] and escape sequences.
// Restore original command line.
StringBuilder join = new StringBuilder();
// terminal space in command line is ok
for (String s : cmd)
join.append(s).append(' ');
// Parse the command line again.
cmd = getTokensFromCommand(join.toString());
executablePath = getExecutablePath(cmd[0]);
// Check new executable name once more
if (security != null)
security.checkExec(executablePath);
}
// Quotation protects from interpretation of the [path] argument as
// start of longer path with spaces. Quotation has no influence to
// [.exe] extension heuristic.
boolean isShell = allowAmbiguousCommands ? isShellFile(executablePath)
: !isExe(executablePath);
cmdstr = createCommandLine(
// We need the extended verification procedures
isShell ? VERIFICATION_CMD_BAT
: (allowAmbiguousCommands ? VERIFICATION_WIN32 : VERIFICATION_WIN32_SAFE),
quoteString(executablePath),
cmd);
}
handle = create(username, password, cmdstr, envblock, path, stdHandles, redirectErrorStream);
AccessController.doPrivileged(
new PrivilegedAction<Void>() {
public Void run() {
if (stdHandles[0] == -1L)
stdin_stream = ProcessBuilderForWin32.NullOutputStream.INSTANCE;
else {
FileDescriptor stdin_fd = new FileDescriptor();
setHandle(stdin_fd, stdHandles[0]);
stdin_stream = new BufferedOutputStream(
new FileOutputStream(stdin_fd));
}
if (stdHandles[1] == -1L)
stdout_stream = ProcessBuilderForWin32.NullInputStream.INSTANCE;
else {
FileDescriptor stdout_fd = new FileDescriptor();
setHandle(stdout_fd, stdHandles[1]);
stdout_stream = new BufferedInputStream(
new FileInputStream(stdout_fd));
}
if (stdHandles[2] == -1L)
stderr_stream = ProcessBuilderForWin32.NullInputStream.INSTANCE;
else {
FileDescriptor stderr_fd = new FileDescriptor();
setHandle(stderr_fd, stdHandles[2]);
stderr_stream = new FileInputStream(stderr_fd);
}
return null; }});
}
public OutputStream getOutputStream() {
return stdin_stream;
}
public InputStream getInputStream() {
return stdout_stream;
}
public InputStream getErrorStream() {
return stderr_stream;
}
protected void finalize() {
closeHandle(handle);
}
public int exitValue() {
int exitCode = getExitCodeProcess(handle);
if (exitCode == STILL_ACTIVE)
throw new IllegalThreadStateException("process has not exited");
return exitCode;
}
public int waitFor() throws InterruptedException {
waitForInterruptibly(handle);
if (Thread.interrupted())
throw new InterruptedException();
return exitValue();
}
@Override
public boolean waitFor(long timeout, TimeUnit unit)
throws InterruptedException
{
if (getExitCodeProcess(handle) != STILL_ACTIVE) return true;
if (timeout <= 0) return false;
long remainingNanos = unit.toNanos(timeout);
long deadline = System.nanoTime() + remainingNanos ;
do {
// Round up to next millisecond
long msTimeout = TimeUnit.NANOSECONDS.toMillis(remainingNanos + 999_999L);
waitForTimeoutInterruptibly(handle, msTimeout);
if (Thread.interrupted())
throw new InterruptedException();
if (getExitCodeProcess(handle) != STILL_ACTIVE) {
return true;
}
remainingNanos = deadline - System.nanoTime();
} while (remainingNanos > 0);
return (getExitCodeProcess(handle) != STILL_ACTIVE);
}
public void destroy() { terminateProcess(handle); }
public Process destroyForcibly() {
destroy();
return this;
}
public boolean isAlive() {
return isProcessAlive(handle);
}
private static boolean initHolder(WinNT.HANDLEByReference pjhandles,
WinNT.HANDLEByReference[] pipe,
int offset,
WinNT.HANDLEByReference phStd) {
if (!pjhandles.getValue().equals(JAVA_INVALID_HANDLE_VALUE)) {
phStd.setValue(pjhandles.getValue());
pjhandles.setValue(JAVA_INVALID_HANDLE_VALUE);
} else {
if (!Kernel32.INSTANCE.CreatePipe(pipe[0], pipe[1], null, PIPE_SIZE)) {
throw new Win32Exception(Kernel32.INSTANCE.GetLastError());
} else {
WinNT.HANDLE thisProcessEnd = offset == OFFSET_READ ? pipe[1].getValue() : pipe[0].getValue();
phStd.setValue(pipe[offset].getValue());
pjhandles.setValue(thisProcessEnd);
}
}
Kernel32.INSTANCE.SetHandleInformation(phStd.getValue(), Kernel32.HANDLE_FLAG_INHERIT, Kernel32.HANDLE_FLAG_INHERIT);
return true;
}
private static void releaseHolder(boolean complete, WinNT.HANDLEByReference[] pipe, int offset) {
closeHandle(pipe[offset].getValue());
if (complete) {
closeHandle(pipe[offset == OFFSET_READ ? OFFSET_WRITE : OFFSET_READ].getValue());
}
}
private static void prepareIOEHandleState(WinNT.HANDLE[] stdIOE, Boolean[] inherit) {
for(int i = 0; i < HANDLE_STORAGE_SIZE; ++i) {
WinNT.HANDLE hstd = stdIOE[i];
if (!Kernel32.INVALID_HANDLE_VALUE.equals(hstd)) {
inherit[i] = Boolean.TRUE;
Kernel32.INSTANCE.SetHandleInformation(hstd, Kernel32.HANDLE_FLAG_INHERIT, 0);
}
}
}
private static void restoreIOEHandleState(WinNT.HANDLE[] stdIOE, Boolean[] inherit) {
for (int i = HANDLE_STORAGE_SIZE - 1; i >= 0; --i) {
if (!Kernel32.INVALID_HANDLE_VALUE.equals(stdIOE[i])) {
Kernel32.INSTANCE.SetHandleInformation(stdIOE[i], Kernel32.HANDLE_FLAG_INHERIT, inherit[i] ? Kernel32.HANDLE_FLAG_INHERIT : 0);
}
}
}
private static WinNT.HANDLE processCreate(String username,
String password,
String cmd,
final String envblock,
final String path,
final WinNT.HANDLEByReference[] stdHandles,
final boolean redirectErrorStream) {
WinNT.HANDLE ret = new WinNT.HANDLE(Pointer.createConstant(0));
WinNT.HANDLE[] stdIOE = new WinNT.HANDLE[] {
Kernel32.INVALID_HANDLE_VALUE, Kernel32.INVALID_HANDLE_VALUE, Kernel32.INVALID_HANDLE_VALUE,
stdHandles[0].getValue(), stdHandles[1].getValue(), stdHandles[2].getValue()
};
stdIOE[0] = Kernel32.INSTANCE.GetStdHandle(Kernel32.STD_INPUT_HANDLE);
stdIOE[1] = Kernel32.INSTANCE.GetStdHandle(Kernel32.STD_OUTPUT_HANDLE);
stdIOE[2] = Kernel32.INSTANCE.GetStdHandle(Kernel32.STD_ERROR_HANDLE);
Boolean[] inherit = new Boolean[] {
Boolean.FALSE, Boolean.FALSE, Boolean.FALSE,
Boolean.FALSE, Boolean.FALSE, Boolean.FALSE
};
prepareIOEHandleState(stdIOE, inherit);
// input
WinNT.HANDLEByReference hStdInput = new WinNT.HANDLEByReference();
WinNT.HANDLEByReference[] pipeIn = new WinNT.HANDLEByReference[] {
new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE), new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE) };
// output
WinNT.HANDLEByReference hStdOutput = new WinNT.HANDLEByReference();
WinNT.HANDLEByReference[] pipeOut = new WinNT.HANDLEByReference[] {
new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE), new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE) };
// error
WinNT.HANDLEByReference hStdError = new WinNT.HANDLEByReference();
WinNT.HANDLEByReference[] pipeError = new WinNT.HANDLEByReference[] {
new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE), new WinNT.HANDLEByReference(Kernel32.INVALID_HANDLE_VALUE) };
boolean success;
if (initHolder(stdHandles[0], pipeIn, OFFSET_READ, hStdInput)) {
if (initHolder(stdHandles[1], pipeOut, OFFSET_WRITE, hStdOutput)) {
WinBase.STARTUPINFO si = new WinBase.STARTUPINFO();
si.hStdInput = hStdInput.getValue();
si.hStdOutput = hStdOutput.getValue();
if (redirectErrorStream) {
si.hStdError = si.hStdOutput;
stdHandles[2].setValue(JAVA_INVALID_HANDLE_VALUE);
success = true;
} else {
success = initHolder(stdHandles[2], pipeError, OFFSET_WRITE, hStdError);
si.hStdError = hStdError.getValue();
}
if (success) {
WTypes.LPSTR lpEnvironment = envblock == null ? new WTypes.LPSTR() : new WTypes.LPSTR(envblock);
Kernel32.PROCESS_INFORMATION pi = new WinBase.PROCESS_INFORMATION();
si.dwFlags = Kernel32.STARTF_USESTDHANDLES;
if (!Advapi32.INSTANCE.CreateProcessWithLogonW(
username
, null
, password
, Advapi32.LOGON_WITH_PROFILE
, null
, cmd
, Kernel32.CREATE_NO_WINDOW
, lpEnvironment.getPointer()
, path
, si
, pi)) {
throw new Win32Exception(Kernel32.INSTANCE.GetLastError());
} else {
closeHandle(pi.hThread);
ret = pi.hProcess;
}
}
releaseHolder(ret.getPointer().equals(Pointer.createConstant(0)), pipeError, OFFSET_WRITE);
releaseHolder(ret.getPointer().equals(Pointer.createConstant(0)), pipeOut, OFFSET_WRITE);
}
releaseHolder(ret.getPointer().equals(Pointer.createConstant(0)), pipeIn, OFFSET_READ);
}
restoreIOEHandleState(stdIOE, inherit);
return ret;
}
private static synchronized WinNT.HANDLE create(String username,
String password,
String cmd,
final String envblock,
final String path,
final long[] stdHandles,
final boolean redirectErrorStream) {
WinNT.HANDLE ret = new WinNT.HANDLE(Pointer.createConstant(0));
WinNT.HANDLEByReference[] handles = new WinNT.HANDLEByReference[stdHandles.length];
for (int i = 0; i < stdHandles.length; i++) {
handles[i] = new WinNT.HANDLEByReference(new WinNT.HANDLE(Pointer.createConstant(stdHandles[i])));
}
if (cmd != null) {
if (username != null && password != null) {
ret = processCreate(username, password, cmd, envblock, path, handles, redirectErrorStream);
}
}
for (int i = 0; i < stdHandles.length; i++) {
stdHandles[i] = handles[i].getPointer().getLong(0);
}
return ret;
}
private static int getExitCodeProcess(WinNT.HANDLE handle) {
IntByReference exitStatus = new IntByReference();
if (!Kernel32.INSTANCE.GetExitCodeProcess(handle, exitStatus)) {
throw new Win32Exception(Kernel32.INSTANCE.GetLastError());
}
return exitStatus.getValue();
}
private static void terminateProcess(WinNT.HANDLE handle) {
Kernel32.INSTANCE.TerminateProcess(handle, 1);
}
private static boolean isProcessAlive(WinNT.HANDLE handle) {
IntByReference exitStatus = new IntByReference();
Kernel32.INSTANCE.GetExitCodeProcess(handle, exitStatus);
return exitStatus.getValue() == STILL_ACTIVE;
}
private static void closeHandle(WinNT.HANDLE handle) {
Kernel32Util.closeHandle(handle);
}
/**
* Opens a file for atomic append. The file is created if it doesn't
* already exist.
*
* @param path the file to open or create
* @return the native HANDLE
*/
private static long openForAtomicAppend(String path) throws IOException {
int access = Kernel32.GENERIC_READ | Kernel32.GENERIC_WRITE;
int sharing = Kernel32.FILE_SHARE_READ | Kernel32.FILE_SHARE_WRITE;
int disposition = Kernel32.OPEN_ALWAYS;
int flagsAndAttributes = Kernel32.FILE_ATTRIBUTE_NORMAL;
if (path == null || path.isEmpty()) {
return -1;
} else {
WinNT.HANDLE handle = Kernel32.INSTANCE.CreateFile(path, access, sharing, null, disposition, flagsAndAttributes, null);
if (handle == Kernel32.INVALID_HANDLE_VALUE) {
throw new Win32Exception(Kernel32.INSTANCE.GetLastError());
}
return handle.getPointer().getLong(0);
}
}
private static void waitForInterruptibly(WinNT.HANDLE handle) {
int result = Kernel32.INSTANCE.WaitForMultipleObjects(1, new WinNT.HANDLE[]{handle}, false, Kernel32.INFINITE);
if (result == Kernel32.WAIT_FAILED) {
throw new Win32Exception(Kernel32.INSTANCE.GetLastError());
}
}
private static void waitForTimeoutInterruptibly(WinNT.HANDLE handle, long timeout) {
int result = Kernel32.INSTANCE.WaitForMultipleObjects(1, new WinNT.HANDLE[]{handle}, false, (int) timeout);
if (result == Kernel32.WAIT_FAILED) {
throw new Win32Exception(Kernel32.INSTANCE.GetLastError());
}
}
}

40
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java

@ -0,0 +1,40 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.junit.Assert;
import org.junit.Test;
/**
* Constants Test
*/
public class ConstantsTest {
/**
* Test PID via env
*/
@Test
public void testPID() {
if (OSUtils.isWindows()) {
Assert.assertEquals(Constants.PID, "handle");
} else {
Assert.assertEquals(Constants.PID, "pid");
}
}
}

2
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java

@ -27,8 +27,6 @@ import org.slf4j.Marker;
import java.util.Map; import java.util.Map;
import static org.junit.Assert.*;
public class TaskLogDiscriminatorTest { public class TaskLogDiscriminatorTest {
/** /**

1
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java

@ -21,7 +21,6 @@ import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.classic.spi.IThrowableProxy; import ch.qos.logback.classic.spi.IThrowableProxy;
import ch.qos.logback.classic.spi.LoggerContextVO; import ch.qos.logback.classic.spi.LoggerContextVO;
import ch.qos.logback.core.spi.FilterReply; import ch.qos.logback.core.spi.FilterReply;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;

1
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java

@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.common.utils;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;

4
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java

@ -39,6 +39,7 @@ public class OSUtilsTest {
@Test @Test
public void testOSMetric(){ public void testOSMetric(){
if (!OSUtils.isWindows()) {
double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize();
Assert.assertTrue(availablePhysicalMemorySize > 0.0f); Assert.assertTrue(availablePhysicalMemorySize > 0.0f);
double totalMemorySize = OSUtils.totalMemorySize(); double totalMemorySize = OSUtils.totalMemorySize();
@ -49,6 +50,9 @@ public class OSUtilsTest {
Assert.assertTrue(memoryUsage > 0.0f); Assert.assertTrue(memoryUsage > 0.0f);
double cpuUsage = OSUtils.cpuUsage(); double cpuUsage = OSUtils.cpuUsage();
Assert.assertTrue(cpuUsage > 0.0f); Assert.assertTrue(cpuUsage > 0.0f);
} else {
// TODO window ut
}
} }
@Test @Test

3
dolphinscheduler-service/src/test/java/utils/PreconditionsTest.java → dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java

@ -14,9 +14,8 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.utils.Preconditions;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.slf4j.Logger; import org.slf4j.Logger;

3
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java

@ -19,15 +19,12 @@ package org.apache.dolphinscheduler.common.utils;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito; import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner; import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.File; import java.io.File;
import java.io.FileNotFoundException;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;

2
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java

@ -19,8 +19,6 @@ package org.apache.dolphinscheduler.common.utils;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import java.util.ArrayList;
public class StringUtilsTest { public class StringUtilsTest {
@Test @Test
public void testIsNotEmpty() { public void testIsNotEmpty() {

3
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java

@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.common.utils;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner; import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;

210
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessBuilderForWin32Test.java

@ -0,0 +1,210 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils.process;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@RunWith(PowerMockRunner.class)
@PrepareForTest(OSUtils.class)
public class ProcessBuilderForWin32Test {
private static final Logger logger = LoggerFactory.getLogger(ProcessBuilderForWin32Test.class);
@Before
public void before() {
PowerMockito.mockStatic(OSUtils.class);
PowerMockito.when(OSUtils.isWindows()).thenReturn(true);
}
@Test
public void testCreateProcessBuilderForWin32() {
try {
ProcessBuilderForWin32 builder = new ProcessBuilderForWin32();
Assert.assertNotNull(builder);
builder = new ProcessBuilderForWin32("net");
Assert.assertNotNull(builder);
builder = new ProcessBuilderForWin32(Collections.singletonList("net"));
Assert.assertNotNull(builder);
builder = new ProcessBuilderForWin32((List<String>) null);
Assert.assertNotNull(builder);
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
@Test
public void testBuildUser() {
try {
ProcessBuilderForWin32 builder = new ProcessBuilderForWin32();
builder.user("test", StringUtils.EMPTY);
Assert.assertNotNull(builder);
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
@Test
public void testBuildCommand() {
try {
ProcessBuilderForWin32 builder = new ProcessBuilderForWin32();
builder.command(Collections.singletonList("net"));
Assert.assertNotEquals(0, builder.command().size());
builder = new ProcessBuilderForWin32();
builder.command("net");
Assert.assertNotEquals(0, builder.command().size());
builder = new ProcessBuilderForWin32();
builder.command((List<String>) null);
Assert.assertNotEquals(0, builder.command().size());
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
@Test
public void testEnvironment() {
try {
ProcessBuilderForWin32 builder = new ProcessBuilderForWin32();
Assert.assertNotNull(builder.environment());
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
try {
ProcessBuilderForWin32 builder = new ProcessBuilderForWin32();
builder.environment(new String[]{ "a=123" });
Assert.assertNotEquals(0, builder.environment().size());
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
@Test
public void testDirectory() {
try {
ProcessBuilderForWin32 builder = new ProcessBuilderForWin32();
builder.directory(new File("/tmp"));
Assert.assertNotNull(builder.directory());
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
@Test
public void testStream() {
try {
InputStream in = ProcessBuilderForWin32.NullInputStream.INSTANCE;
Assert.assertNotNull(in);
Assert.assertEquals(-1, in.read());
Assert.assertEquals(0, in.available());
OutputStream out = ProcessBuilderForWin32.NullOutputStream.INSTANCE;
Assert.assertNotNull(out);
out.write(new byte[] {1});
} catch (Exception e) {
logger.error(e.getMessage());
}
}
@Test
public void testRedirect() {
try {
ProcessBuilderForWin32 builder = new ProcessBuilderForWin32();
builder.redirectInput(new File("/tmp"));
Assert.assertNotNull(builder.redirectInput());
Assert.assertNotNull(builder.redirectInput().file());
builder.redirectOutput(new File("/tmp"));
Assert.assertNotNull(builder.redirectOutput());
Assert.assertNotNull(builder.redirectOutput().file());
builder.redirectError(new File("/tmp"));
Assert.assertNotNull(builder.redirectError());
Assert.assertNotNull(builder.redirectError().file());
builder.redirectInput(builder.redirectOutput());
builder.redirectOutput(builder.redirectInput());
builder.redirectError(builder.redirectInput());
Assert.assertNotNull(ProcessBuilderForWin32.Redirect.PIPE.type());
Assert.assertNotNull(ProcessBuilderForWin32.Redirect.PIPE.toString());
Assert.assertNotNull(ProcessBuilderForWin32.Redirect.INHERIT.type());
Assert.assertNotNull(ProcessBuilderForWin32.Redirect.INHERIT.toString());
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
@Test
public void testRedirectErrorStream() {
try {
ProcessBuilderForWin32 builder = new ProcessBuilderForWin32();
builder.redirectErrorStream(true);
Assert.assertTrue(builder.redirectErrorStream());
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
@Test
public void runCmdViaUser() {
try {
ProcessBuilderForWin32 builder = new ProcessBuilderForWin32();
builder.user("test123", StringUtils.EMPTY);
List<String> commands = new ArrayList<>();
commands.add("cmd.exe");
commands.add("/c");
commands.add("net user");
builder.command(commands);
Process process = builder.start();
BufferedReader inReader = new BufferedReader(new InputStreamReader(process.getInputStream(), Charset.forName("GBK")));
String line;
StringBuilder sb = new StringBuilder();
while ((line = inReader.readLine()) != null) {
sb.append(line);
}
logger.info("net user: {}", sb.toString());
Assert.assertNotEquals(StringUtils.EMPTY, sb.toString());
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
}

124
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessEnvironmentForWin32Test.java

@ -0,0 +1,124 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils.process;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collection;
import java.util.Map;
import java.util.Set;
@RunWith(PowerMockRunner.class)
@PrepareForTest({OSUtils.class, ProcessEnvironmentForWin32.class})
public class ProcessEnvironmentForWin32Test {
private static final Logger logger = LoggerFactory.getLogger(ProcessBuilderForWin32Test.class);
@Before
public void before() {
try {
PowerMockito.mockStatic(OSUtils.class);
PowerMockito.when(OSUtils.isWindows()).thenReturn(true);
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
@Test
public void testPutAndGet() {
try {
ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0);
processEnvironmentForWin32.put("a", "123");
Assert.assertEquals("123", processEnvironmentForWin32.get("a"));
Assert.assertTrue(processEnvironmentForWin32.containsKey("a"));
Assert.assertTrue(processEnvironmentForWin32.containsValue("123"));
Assert.assertEquals("123", processEnvironmentForWin32.remove("a"));
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
try {
ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0);
processEnvironmentForWin32.put("b=", "123");
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
try {
ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0);
processEnvironmentForWin32.put("b", "\u0000");
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
try {
ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0);
processEnvironmentForWin32.get(null);
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
@Test
public void testEntrySet() {
try {
ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0);
processEnvironmentForWin32.clear();
processEnvironmentForWin32.put("a", "123");
Assert.assertEquals(0, processEnvironmentForWin32.entrySet().size());
Assert.assertTrue(processEnvironmentForWin32.entrySet().isEmpty());
for (Map.Entry<String, String> entry : processEnvironmentForWin32.entrySet()) {
Assert.assertNotNull(entry);
Assert.assertNotNull(entry.getKey());
Assert.assertNotNull(entry.getValue());
Assert.assertNotNull(entry.setValue("123"));
}
processEnvironmentForWin32.clear();
Set<String> keys = processEnvironmentForWin32.keySet();
Assert.assertEquals(0, keys.size());
Assert.assertTrue(keys.isEmpty());
processEnvironmentForWin32.clear();
Collection<String> values = processEnvironmentForWin32.values();
Assert.assertEquals(0, keys.size());
Assert.assertTrue(keys.isEmpty());
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
@Test
public void testToEnvironmentBlock() {
try {
ProcessEnvironmentForWin32 processEnvironmentForWin32 = (ProcessEnvironmentForWin32) ProcessEnvironmentForWin32.emptyEnvironment(0);
Assert.assertNotNull(processEnvironmentForWin32.toEnvironmentBlock());
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
}

70
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/process/ProcessImplForWin32Test.java

@ -0,0 +1,70 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils.process;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sun.security.action.GetPropertyAction;
@RunWith(PowerMockRunner.class)
@PrepareForTest({OSUtils.class, GetPropertyAction.class})
public class ProcessImplForWin32Test {
private static final Logger logger = LoggerFactory.getLogger(ProcessBuilderForWin32Test.class);
@Before
public void before() {
PowerMockito.mockStatic(OSUtils.class);
PowerMockito.mockStatic(GetPropertyAction.class);
PowerMockito.when(OSUtils.isWindows()).thenReturn(true);
}
@Test
public void testStart() {
try {
Process process = ProcessImplForWin32.start(
"test123", StringUtils.EMPTY, new String[]{"net"},
null, null, null, false);
Assert.assertNotNull(process);
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
try {
Process process = ProcessImplForWin32.start(
"test123", StringUtils.EMPTY, new String[]{"net"},
null, null, new ProcessBuilderForWin32.Redirect[]{
ProcessBuilderForWin32.Redirect.PIPE,
ProcessBuilderForWin32.Redirect.PIPE,
ProcessBuilderForWin32.Redirect.PIPE
}, false);
Assert.assertNotNull(process);
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
}

2
dolphinscheduler-dao/pom.xml

@ -25,7 +25,7 @@
</parent> </parent>
<artifactId>dolphinscheduler-dao</artifactId> <artifactId>dolphinscheduler-dao</artifactId>
<name>${project.artifactId}</name> <name>${project.artifactId}</name>
<url>http://maven.apache.org</url>
<properties> <properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties> </properties>

6
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java

@ -144,9 +144,11 @@ public class AlertDao extends AbstractBaseDao {
* @param taskId taskId * @param taskId taskId
* @param taskName taskName * @param taskName taskName
*/ */
public void sendTaskTimeoutAlert(int alertgroupId,String receivers,String receiversCc,int taskId,String taskName){ public void sendTaskTimeoutAlert(int alertgroupId,String receivers,String receiversCc, int processInstanceId,
String processInstanceName, int taskId,String taskName){
Alert alert = new Alert(); Alert alert = new Alert();
String content = String.format("[{'id':'%d','name':'%s','event':'timeout','warnLevel':'middle'}]",taskId,taskName); String content = String.format("[{'process instance id':'%d','task name':'%s','task id':'%d','task name':'%s'," +
"'event':'timeout','warnLevel':'middle'}]", processInstanceId, processInstanceName, taskId, taskName);
alert.setTitle("Task Timeout Warn"); alert.setTitle("Task Timeout Warn");
alert.setShowType(ShowType.TABLE); alert.setShowType(ShowType.TABLE);
alert.setContent(content); alert.setContent(content);

1
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java

@ -29,7 +29,6 @@ import com.baomidou.mybatisplus.core.toolkit.StringUtils;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors; import java.util.stream.Collectors;

1
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java

@ -16,7 +16,6 @@
*/ */
package org.apache.dolphinscheduler.dao.entity; package org.apache.dolphinscheduler.dao.entity;
import com.baomidou.mybatisplus.annotation.TableField;
import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.enums.UdfType;
import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableId;

1
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/MysqlUpgradeDao.java

@ -17,7 +17,6 @@
package org.apache.dolphinscheduler.dao.upgrade; package org.apache.dolphinscheduler.dao.upgrade;
import org.apache.dolphinscheduler.common.utils.ConnectionUtils; import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;

1
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/PostgresqlUpgradeDao.java

@ -17,7 +17,6 @@
package org.apache.dolphinscheduler.dao.upgrade; package org.apache.dolphinscheduler.dao.upgrade;
import org.apache.dolphinscheduler.common.utils.ConnectionUtils; import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;

4
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java

@ -67,7 +67,7 @@ public class ErrorCommandMapperTest {
//update //update
errorCommand.setUpdateTime(new Date()); errorCommand.setUpdateTime(new Date());
int update = errorCommandMapper.updateById(errorCommand); int update = errorCommandMapper.updateById(errorCommand);
Assert.assertEquals(update, 1); Assert.assertEquals(1,update);
errorCommandMapper.deleteById(errorCommand.getId()); errorCommandMapper.deleteById(errorCommand.getId());
} }
@ -79,7 +79,7 @@ public class ErrorCommandMapperTest {
ErrorCommand errorCommand = insertOne(); ErrorCommand errorCommand = insertOne();
int delete = errorCommandMapper.deleteById(errorCommand.getId()); int delete = errorCommandMapper.deleteById(errorCommand.getId());
Assert.assertEquals(delete, 1); Assert.assertEquals(1,delete);
} }
/** /**

2
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java

@ -17,8 +17,6 @@
package org.apache.dolphinscheduler.dao.mapper; package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.dao.entity.ProjectUser;
import org.apache.dolphinscheduler.dao.entity.ProjectUser; import org.apache.dolphinscheduler.dao.entity.ProjectUser;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;

1
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java

@ -17,7 +17,6 @@
package org.apache.dolphinscheduler.dao.mapper; package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.dao.entity.Queue;
import org.apache.dolphinscheduler.dao.entity.Queue; import org.apache.dolphinscheduler.dao.entity.Queue;
import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.baomidou.mybatisplus.extension.plugins.pagination.Page;

2
dolphinscheduler-remote/pom.xml

@ -12,8 +12,6 @@
<artifactId>dolphinscheduler-remote</artifactId> <artifactId>dolphinscheduler-remote</artifactId>
<name>dolphinscheduler-remote</name> <name>dolphinscheduler-remote</name>
<!-- FIXME change it to the project's website -->
<url>http://www.example.com</url>
<properties> <properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

2
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java

@ -145,7 +145,7 @@ public class NettyRemotingServer {
try { try {
future = serverBootstrap.bind(serverConfig.getListenPort()).sync(); future = serverBootstrap.bind(serverConfig.getListenPort()).sync();
} catch (Exception e) { } catch (Exception e) {
logger.error("NettyRemotingServer bind fail {}, exit", e); logger.error("NettyRemotingServer bind fail {}, exit",e.getMessage(), e);
throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort())); throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort()));
} }
if (future.isSuccess()) { if (future.isSuccess()) {

2
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java

@ -16,8 +16,6 @@
*/ */
package org.apache.dolphinscheduler.remote.command; package org.apache.dolphinscheduler.remote.command;
import com.sun.org.apache.regexp.internal.RE;
import java.io.Serializable; import java.io.Serializable;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;

2
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskRequestCommand.java

@ -1 +1 @@
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; import java.util.List; import java.util.concurrent.atomic.AtomicLong; /** * execute task request command */ public class ExecuteTaskRequestCommand implements Serializable { /** * task id */ private String taskId; /** * attempt id */ private String attemptId; /** * application name */ private String applicationName; /** * group name */ private String groupName; /** * task name */ private String taskName; /** * connector port */ private int connectorPort; /** * description info */ private String description; /** * class name */ private String className; /** * method name */ private String methodName; /** * parameters */ private String params; /** * shard itemds */ private List<Integer> shardItems; public List<Integer> getShardItems() { return shardItems; } public void setShardItems(List<Integer> shardItems) { this.shardItems = shardItems; } public String getParams() { return params; } public void setParams(String params) { this.params = params; } public String getTaskId() { return taskId; } public void setTaskId(String taskId) { this.taskId = taskId; } public String getApplicationName() { return applicationName; } public void setApplicationName(String applicationName) { this.applicationName = applicationName; } public String getGroupName() { return groupName; } public void setGroupName(String groupName) { this.groupName = groupName; } public String getTaskName() { return taskName; } public void setTaskName(String taskName) { this.taskName = taskName; } public int getConnectorPort() { return connectorPort; } public void setConnectorPort(int connectorPort) { this.connectorPort = connectorPort; } public String getDescription() { return description; } public void setDescription(String description) { this.description = description; } public String getClassName() { return className; } public void setClassName(String className) { this.className = className; } public String getMethodName() { return methodName; } public void setMethodName(String methodName) { this.methodName = methodName; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.EXECUTE_TASK_REQUEST); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } } /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; import java.util.List; /** * execute task request command */ public class ExecuteTaskRequestCommand implements Serializable { /** * task id */ private String taskId; /** * attempt id */ private String attemptId; /** * application name */ private String applicationName; /** * group name */ private String groupName; /** * task name */ private String taskName; /** * connector port */ private int connectorPort; /** * description info */ private String description; /** * class name */ private String className; /** * method name */ private String methodName; /** * parameters */ private String params; /** * shard itemds */ private List<Integer> shardItems; public List<Integer> getShardItems() { return shardItems; } public void setShardItems(List<Integer> shardItems) { this.shardItems = shardItems; } public String getParams() { return params; } public void setParams(String params) { this.params = params; } public String getTaskId() { return taskId; } public void setTaskId(String taskId) { this.taskId = taskId; } public String getApplicationName() { return applicationName; } public void setApplicationName(String applicationName) { this.applicationName = applicationName; } public String getGroupName() { return groupName; } public void setGroupName(String groupName) { this.groupName = groupName; } public String getTaskName() { return taskName; } public void setTaskName(String taskName) { this.taskName = taskName; } public int getConnectorPort() { return connectorPort; } public void setConnectorPort(int connectorPort) { this.connectorPort = connectorPort; } public String getDescription() { return description; } public void setDescription(String description) { this.description = description; } public String getClassName() { return className; } public void setClassName(String className) { this.className = className; } public String getMethodName() { return methodName; } public void setMethodName(String methodName) { this.methodName = methodName; } /** * package request command * * @return command */ public Command convert2Command(){ Command command = new Command(); command.setType(CommandType.EXECUTE_TASK_REQUEST); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } }

2
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ExecuteTaskResponseCommand.java

@ -1 +1 @@
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; import java.util.concurrent.atomic.AtomicLong; /** * execute task response command */ public class ExecuteTaskResponseCommand implements Serializable { /** * task id */ private String taskId; /** * attempt id */ private String attemptId; /** * return result */ private Object result; /** * received time */ private long receivedTime; /** * execute count */ private int executeCount; /** * execute time */ private long executeTime; public String getAttemptId() { return attemptId; } public void setAttemptId(String attemptId) { this.attemptId = attemptId; } public String getTaskId() { return taskId; } public void setTaskId(String taskId) { this.taskId = taskId; } public Object getResult() { return result; } public void setResult(Object result) { this.result = result; } public long getReceivedTime() { return receivedTime; } public void setReceivedTime(long receivedTime) { this.receivedTime = receivedTime; } public int getExecuteCount() { return executeCount; } public void setExecuteCount(int executeCount) { this.executeCount = executeCount; } public long getExecuteTime() { return executeTime; } public void setExecuteTime(long executeTime) { this.executeTime = executeTime; } public Command convert2Command(long opaque){ Command command = new Command(); command.setType(CommandType.EXECUTE_TASK_RESPONSE); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } } /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import java.io.Serializable; /** * execute task response command */ public class ExecuteTaskResponseCommand implements Serializable { /** * task id */ private String taskId; /** * attempt id */ private String attemptId; /** * return result */ private Object result; /** * received time */ private long receivedTime; /** * execute count */ private int executeCount; /** * execute time */ private long executeTime; public String getAttemptId() { return attemptId; } public void setAttemptId(String attemptId) { this.attemptId = attemptId; } public String getTaskId() { return taskId; } public void setTaskId(String taskId) { this.taskId = taskId; } public Object getResult() { return result; } public void setResult(Object result) { this.result = result; } public long getReceivedTime() { return receivedTime; } public void setReceivedTime(long receivedTime) { this.receivedTime = receivedTime; } public int getExecuteCount() { return executeCount; } public void setExecuteCount(int executeCount) { this.executeCount = executeCount; } public long getExecuteTime() { return executeTime; } public void setExecuteTime(long executeTime) { this.executeTime = executeTime; } public Command convert2Command(long opaque){ Command command = new Command(); command.setType(CommandType.EXECUTE_TASK_RESPONSE); byte[] body = FastJsonSerializer.serialize(this); command.setBody(body); return command; } }

1
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Ping.java

@ -21,7 +21,6 @@ import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled; import io.netty.buffer.Unpooled;
import java.io.Serializable; import java.io.Serializable;
import java.util.concurrent.atomic.AtomicLong;
/** /**
* ping machine * ping machine

1
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java

@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.remote.command.CommandType;
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import java.io.Serializable; import java.io.Serializable;
import java.util.concurrent.atomic.AtomicLong;
/** /**
* get log bytes request command * get log bytes request command

1
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java

@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.remote.command.CommandType;
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import java.io.Serializable; import java.io.Serializable;
import java.util.concurrent.atomic.AtomicLong;
/** /**
* roll view log request command * roll view log request command

1
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java

@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.remote.command.CommandType;
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import java.io.Serializable; import java.io.Serializable;
import java.util.concurrent.atomic.AtomicLong;
/** /**
* view log request command * view log request command

2
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java

@ -107,7 +107,7 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter {
*/ */
@Override @Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
logger.error("exceptionCaught : {}", cause); logger.error("exceptionCaught : {}",cause.getMessage(), cause);
nettyRemotingClient.closeChannel(ChannelUtils.toAddress(ctx.channel())); nettyRemotingClient.closeChannel(ChannelUtils.toAddress(ctx.channel()));
ctx.channel().close(); ctx.channel().close();
} }

2
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java

@ -140,7 +140,7 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter {
*/ */
@Override @Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
logger.error("exceptionCaught : {}", cause); logger.error("exceptionCaught : {}",cause.getMessage(), cause);
ctx.channel().close(); ctx.channel().close();
} }

8
dolphinscheduler-server/pom.xml

@ -25,7 +25,7 @@
</parent> </parent>
<artifactId>dolphinscheduler-server</artifactId> <artifactId>dolphinscheduler-server</artifactId>
<name>dolphinscheduler-server</name> <name>dolphinscheduler-server</name>
<url>http://maven.apache.org</url>
<packaging>jar</packaging> <packaging>jar</packaging>
<properties> <properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@ -86,6 +86,12 @@
<dependency> <dependency>
<groupId>org.apache.curator</groupId> <groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId> <artifactId>curator-recipes</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.zookeeper</groupId> <groupId>org.apache.zookeeper</groupId>

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java

@ -565,7 +565,7 @@ public class MasterExecThread implements Runnable {
TaskInstance taskInstance = completeTaskList.get(nodeName); TaskInstance taskInstance = completeTaskList.get(nodeName);
if(taskInstance == null){ if(taskInstance == null){
logger.error("task instance cannot find, please check it!", nodeName); logger.error("task instance {} cannot find, please check it!", nodeName);
return conditionTaskList; return conditionTaskList;
} }

4
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java

@ -128,7 +128,9 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread {
// process define // process define
ProcessDefinition processDefine = processService.findProcessDefineById(processInstance.getProcessDefinitionId()); ProcessDefinition processDefine = processService.findProcessDefineById(processInstance.getProcessDefinitionId());
// send warn mail // send warn mail
alertDao.sendTaskTimeoutAlert(processInstance.getWarningGroupId(),processDefine.getReceivers(),processDefine.getReceiversCc(),taskInstance.getId(),taskInstance.getName()); alertDao.sendTaskTimeoutAlert(processInstance.getWarningGroupId(),processDefine.getReceivers(),
processDefine.getReceiversCc(), processInstance.getId(), processInstance.getName(),
taskInstance.getId(),taskInstance.getName());
checkTimeout = false; checkTimeout = false;
} }
} }

4
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java

@ -133,6 +133,8 @@ public class AlertManager {
continue; continue;
} }
LinkedHashMap<String, String> failedTaskMap = new LinkedHashMap(); LinkedHashMap<String, String> failedTaskMap = new LinkedHashMap();
failedTaskMap.put("process instance id", String.valueOf(processInstance.getId()));
failedTaskMap.put("process instance name", processInstance.getName());
failedTaskMap.put("task id", String.valueOf(task.getId())); failedTaskMap.put("task id", String.valueOf(task.getId()));
failedTaskMap.put("task name", task.getName()); failedTaskMap.put("task name", task.getName());
failedTaskMap.put("task type", task.getTaskType()); failedTaskMap.put("task type", task.getTaskType());
@ -193,7 +195,7 @@ public class AlertManager {
logger.info("add alert to db , alert : {}", alert.toString()); logger.info("add alert to db , alert : {}", alert.toString());
}catch (Exception e){ }catch (Exception e){
logger.error("send alert failed! " + e); logger.error("send alert failed:{} ", e.getMessage());
} }
} }

58
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java

@ -16,21 +16,29 @@
*/ */
package org.apache.dolphinscheduler.server.worker.task; package org.apache.dolphinscheduler.server.worker.task;
import com.sun.jna.platform.win32.Kernel32;
import com.sun.jna.platform.win32.WinNT;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.common.utils.process.ProcessBuilderForWin32;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.server.utils.ProcessUtils; import org.apache.dolphinscheduler.server.utils.ProcessUtils;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
import org.slf4j.Logger; import org.slf4j.Logger;
import java.io.*; import java.io.*;
import java.lang.reflect.Field; import java.lang.reflect.Field;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.*; import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.function.Consumer; import java.util.function.Consumer;
@ -194,26 +202,49 @@ public abstract class AbstractCommandExecutor {
* @throws IOException IO Exception * @throws IOException IO Exception
*/ */
private void buildProcess(String commandFile) throws IOException { private void buildProcess(String commandFile) throws IOException {
// command list
List<String> command = new ArrayList<>();
//init process builder //init process builder
if (OSUtils.isWindows()) {
ProcessBuilderForWin32 processBuilder = new ProcessBuilderForWin32();
// setting up a working directory
processBuilder.directory(new File(taskDir));
processBuilder.user(tenantCode, StringUtils.EMPTY);
// merge error information to standard output stream
processBuilder.redirectErrorStream(true);
// setting up user to run commands
command.add(commandInterpreter());
command.add("/c");
command.addAll(commandOptions());
command.add(commandFile);
// setting commands
processBuilder.command(command);
process = processBuilder.start();
} else {
ProcessBuilder processBuilder = new ProcessBuilder(); ProcessBuilder processBuilder = new ProcessBuilder();
// setting up a working directory // setting up a working directory
processBuilder.directory(new File(taskDir)); processBuilder.directory(new File(taskDir));
// merge error information to standard output stream // merge error information to standard output stream
processBuilder.redirectErrorStream(true); processBuilder.redirectErrorStream(true);
// setting up user to run commands // setting up user to run commands
List<String> command = new LinkedList<>();
command.add("sudo"); command.add("sudo");
command.add("-u"); command.add("-u");
command.add(tenantCode); command.add(tenantCode);
command.add(commandInterpreter()); command.add(commandInterpreter());
command.addAll(commandOptions()); command.addAll(commandOptions());
command.add(commandFile); command.add(commandFile);
processBuilder.command(command);
// setting commands
processBuilder.command(command);
process = processBuilder.start(); process = processBuilder.start();
}
// print command // print command
printCommand(processBuilder); printCommand(command);
} }
/** /**
@ -320,13 +351,13 @@ public abstract class AbstractCommandExecutor {
/** /**
* print command * print command
* @param processBuilder process builder * @param command command
*/ */
private void printCommand(ProcessBuilder processBuilder) { private void printCommand(List<String> command) {
String cmdStr; String cmdStr;
try { try {
cmdStr = ProcessUtils.buildCommandStr(processBuilder.command()); cmdStr = ProcessUtils.buildCommandStr(command);
logger.info("task run command:\n{}", cmdStr); logger.info("task run command:\n{}", cmdStr);
} catch (IOException e) { } catch (IOException e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
@ -358,7 +389,11 @@ public abstract class AbstractCommandExecutor {
BufferedReader inReader = null; BufferedReader inReader = null;
try { try {
if (OSUtils.isWindows()) {
inReader = new BufferedReader(new InputStreamReader(process.getInputStream(), Charset.forName("GBK")));
} else {
inReader = new BufferedReader(new InputStreamReader(process.getInputStream())); inReader = new BufferedReader(new InputStreamReader(process.getInputStream()));
}
String line; String line;
long lastFlushTime = System.currentTimeMillis(); long lastFlushTime = System.currentTimeMillis();
@ -510,12 +545,15 @@ public abstract class AbstractCommandExecutor {
*/ */
private int getProcessId(Process process) { private int getProcessId(Process process) {
int processId = 0; int processId = 0;
try { try {
Field f = process.getClass().getDeclaredField(Constants.PID); Field f = process.getClass().getDeclaredField(Constants.PID);
f.setAccessible(true); f.setAccessible(true);
if (OSUtils.isWindows()) {
WinNT.HANDLE handle = (WinNT.HANDLE) f.get(process);
processId = Kernel32.INSTANCE.GetProcessId(handle);
} else {
processId = f.getInt(process); processId = f.getInt(process);
}
} catch (Throwable e) { } catch (Throwable e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
} }

29
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java

@ -17,11 +17,12 @@
package org.apache.dolphinscheduler.server.worker.task; package org.apache.dolphinscheduler.server.worker.task;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.Charset; import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.Date; import java.util.Date;
@ -34,10 +35,15 @@ import java.util.function.Consumer;
public class ShellCommandExecutor extends AbstractCommandExecutor { public class ShellCommandExecutor extends AbstractCommandExecutor {
/** /**
* sh * For Unix-like, using sh
*/ */
public static final String SH = "sh"; public static final String SH = "sh";
/**
* For Windows, using cmd.exe
*/
public static final String CMD = "cmd.exe";
/** /**
* constructor * constructor
* @param logHandler log handler * @param logHandler log handler
@ -66,7 +72,7 @@ public class ShellCommandExecutor extends AbstractCommandExecutor {
@Override @Override
protected String buildCommandFilePath() { protected String buildCommandFilePath() {
// command file // command file
return String.format("%s/%s.command", taskDir, taskAppId); return String.format("%s/%s.%s", taskDir, taskAppId, OSUtils.isWindows() ? "bat" : "command");
} }
/** /**
@ -75,7 +81,7 @@ public class ShellCommandExecutor extends AbstractCommandExecutor {
*/ */
@Override @Override
protected String commandInterpreter() { protected String commandInterpreter() {
return SH; return OSUtils.isWindows() ? CMD : SH;
} }
/** /**
@ -103,21 +109,26 @@ public class ShellCommandExecutor extends AbstractCommandExecutor {
logger.info("create command file:{}", commandFile); logger.info("create command file:{}", commandFile);
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
if (OSUtils.isWindows()) {
sb.append("@echo off\n");
sb.append("cd /d %~dp0\n");
if (envFile != null) {
sb.append("call ").append(envFile).append("\n");
}
} else {
sb.append("#!/bin/sh\n"); sb.append("#!/bin/sh\n");
sb.append("BASEDIR=$(cd `dirname $0`; pwd)\n"); sb.append("BASEDIR=$(cd `dirname $0`; pwd)\n");
sb.append("cd $BASEDIR\n"); sb.append("cd $BASEDIR\n");
if (envFile != null) { if (envFile != null) {
sb.append("source " + envFile + "\n"); sb.append("source ").append(envFile).append("\n");
}
} }
sb.append("\n\n");
sb.append(execCommand); sb.append(execCommand);
logger.info("command : {}", sb.toString()); logger.info("command : {}", sb.toString());
// write data to file // write data to file
FileUtils.writeStringToFile(new File(commandFile), sb.toString(), FileUtils.writeStringToFile(new File(commandFile), sb.toString(), StandardCharsets.UTF_8);
Charset.forName("UTF-8"));
} }
} }

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskProps.java

@ -78,7 +78,7 @@ public class TaskProps {
private Map<String, String> definedParams; private Map<String, String> definedParams;
/** /**
* task path * task app id
*/ */
private String taskAppId; private String taskAppId;

9
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java

@ -44,6 +44,7 @@ import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.task.datax.DataxParameters; import org.apache.dolphinscheduler.common.task.datax.DataxParameters;
import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; import org.apache.dolphinscheduler.dao.datasource.BaseDataSource;
import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory;
@ -338,7 +339,7 @@ public class DataxTask extends AbstractTask {
private String buildShellCommandFile(String jobConfigFilePath) private String buildShellCommandFile(String jobConfigFilePath)
throws Exception { throws Exception {
// generate scripts // generate scripts
String fileName = String.format("%s/%s_node.sh", taskDir, taskProps.getTaskAppId()); String fileName = String.format("%s/%s_node.%s", taskDir, taskProps.getTaskAppId(), OSUtils.isWindows() ? "bat" : "sh");
Path path = new File(fileName).toPath(); Path path = new File(fileName).toPath();
if (Files.exists(path)) { if (Files.exists(path)) {
@ -370,7 +371,13 @@ public class DataxTask extends AbstractTask {
// create shell command file // create shell command file
Set<PosixFilePermission> perms = PosixFilePermissions.fromString(Constants.RWXR_XR_X); Set<PosixFilePermission> perms = PosixFilePermissions.fromString(Constants.RWXR_XR_X);
FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(perms); FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(perms);
if (OSUtils.isWindows()) {
Files.createFile(path);
} else {
Files.createFile(path, attr); Files.createFile(path, attr);
}
Files.write(path, dataxCommand.getBytes(), StandardOpenOption.APPEND); Files.write(path, dataxCommand.getBytes(), StandardOpenOption.APPEND);
return fileName; return fileName;

7
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java

@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.task.shell.ShellParameters; import org.apache.dolphinscheduler.common.task.shell.ShellParameters;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.dolphinscheduler.server.utils.ParamUtils;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
@ -123,7 +124,7 @@ public class ShellTask extends AbstractTask {
*/ */
private String buildCommand() throws Exception { private String buildCommand() throws Exception {
// generate scripts // generate scripts
String fileName = String.format("%s/%s_node.sh", taskDir, taskProps.getTaskAppId()); String fileName = String.format("%s/%s_node.%s", taskDir, taskProps.getTaskAppId(), OSUtils.isWindows() ? "bat" : "sh");
Path path = new File(fileName).toPath(); Path path = new File(fileName).toPath();
if (Files.exists(path)) { if (Files.exists(path)) {
@ -154,7 +155,11 @@ public class ShellTask extends AbstractTask {
Set<PosixFilePermission> perms = PosixFilePermissions.fromString(Constants.RWXR_XR_X); Set<PosixFilePermission> perms = PosixFilePermissions.fromString(Constants.RWXR_XR_X);
FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(perms); FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(perms);
if (OSUtils.isWindows()) {
Files.createFile(path);
} else {
Files.createFile(path, attr); Files.createFile(path, attr);
}
Files.write(path, shellParameters.getRawScript().getBytes(), StandardOpenOption.APPEND); Files.write(path, shellParameters.getRawScript().getBytes(), StandardOpenOption.APPEND);

17
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java

@ -21,7 +21,12 @@ import org.junit.Test;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class ProcessUtilsTest { public class ProcessUtilsTest {
private static final Logger logger = LoggerFactory.getLogger(ProcessUtilsTest.class); private static final Logger logger = LoggerFactory.getLogger(ProcessUtilsTest.class);
@Test @Test
@ -30,4 +35,16 @@ public class ProcessUtilsTest {
Assert.assertNotEquals("The child process of process 1 should not be empty", pidList, ""); Assert.assertNotEquals("The child process of process 1 should not be empty", pidList, "");
logger.info("Sub process list : {}", pidList); logger.info("Sub process list : {}", pidList);
} }
@Test
public void testBuildCommandStr() {
List<String> commands = new ArrayList<>();
commands.add("sudo");
try {
Assert.assertEquals(ProcessUtils.buildCommandStr(commands), "sudo");
} catch (IOException e) {
Assert.fail(e.getMessage());
}
}
} }

199
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java

@ -0,0 +1,199 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.worker.task.shell;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.junit.After;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationContext;
import java.util.Date;
/**
* shell task test
*/
@RunWith(PowerMockRunner.class)
@PrepareForTest(OSUtils.class)
public class ShellTaskTest {
private static final Logger logger = LoggerFactory.getLogger(ShellTaskTest.class);
private ShellTask shellTask;
private ProcessService processService;
private ShellCommandExecutor shellCommandExecutor;
private ApplicationContext applicationContext;
@Before
public void before() throws Exception {
PowerMockito.mockStatic(OSUtils.class);
processService = PowerMockito.mock(ProcessService.class);
shellCommandExecutor = PowerMockito.mock(ShellCommandExecutor.class);
applicationContext = PowerMockito.mock(ApplicationContext.class);
SpringApplicationContext springApplicationContext = new SpringApplicationContext();
springApplicationContext.setApplicationContext(applicationContext);
PowerMockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService);
TaskProps props = new TaskProps();
props.setTaskDir("/tmp");
props.setTaskAppId(String.valueOf(System.currentTimeMillis()));
props.setTaskInstId(1);
props.setTenantCode("1");
props.setEnvFile(".dolphinscheduler_env.sh");
props.setTaskStartTime(new Date());
props.setTaskTimeout(0);
props.setTaskParams("{\"rawScript\": \" echo 'hello world!'\"}");
shellTask = new ShellTask(props, logger);
shellTask.init();
PowerMockito.when(processService.findDataSourceById(1)).thenReturn(getDataSource());
PowerMockito.when(processService.findDataSourceById(2)).thenReturn(getDataSource());
PowerMockito.when(processService.findProcessInstanceByTaskId(1)).thenReturn(getProcessInstance());
String fileName = String.format("%s/%s_node.%s", props.getTaskDir(), props.getTaskAppId(), OSUtils.isWindows() ? "bat" : "sh");
PowerMockito.when(shellCommandExecutor.run(fileName, processService)).thenReturn(0);
}
private DataSource getDataSource() {
DataSource dataSource = new DataSource();
dataSource.setType(DbType.MYSQL);
dataSource.setConnectionParams(
"{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:mysql://127.0.0.1:3306\",\"database\":\"test\",\"jdbcUrl\":\"jdbc:mysql://127.0.0.1:3306/test\"}");
dataSource.setUserId(1);
return dataSource;
}
private ProcessInstance getProcessInstance() {
ProcessInstance processInstance = new ProcessInstance();
processInstance.setCommandType(CommandType.START_PROCESS);
processInstance.setScheduleTime(new Date());
return processInstance;
}
@After
public void after() {}
/**
* Method: ShellTask()
*/
@Test
public void testShellTask()
throws Exception {
TaskProps props = new TaskProps();
props.setTaskDir("/tmp");
props.setTaskAppId(String.valueOf(System.currentTimeMillis()));
props.setTaskInstId(1);
props.setTenantCode("1");
ShellTask shellTaskTest = new ShellTask(props, logger);
Assert.assertNotNull(shellTaskTest);
}
/**
* Method: init for Unix-like
*/
@Test
public void testInitForUnix() {
try {
PowerMockito.when(OSUtils.isWindows()).thenReturn(false);
shellTask.init();
Assert.assertTrue(true);
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
/**
* Method: init for Windows
*/
@Test
public void testInitForWindows() {
try {
PowerMockito.when(OSUtils.isWindows()).thenReturn(true);
shellTask.init();
Assert.assertTrue(true);
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
/**
* Method: handle() for Unix-like
*/
@Test
public void testHandleForUnix() throws Exception {
try {
PowerMockito.when(OSUtils.isWindows()).thenReturn(false);
shellTask.handle();
Assert.assertTrue(true);
} catch (Error | Exception e) {
if (!e.getMessage().contains("process error . exitCode is : -1")
&& !System.getProperty("os.name").startsWith("Windows")) {
logger.error(e.getMessage());
}
}
}
/**
* Method: handle() for Windows
*/
@Test
public void testHandleForWindows() throws Exception {
try {
Assume.assumeTrue(OSUtils.isWindows());
shellTask.handle();
Assert.assertTrue(true);
} catch (Error | Exception e) {
if (!e.getMessage().contains("process error . exitCode is : -1")) {
logger.error(e.getMessage());
}
}
}
/**
* Method: cancelApplication()
*/
@Test
public void testCancelApplication() throws Exception {
try {
shellTask.cancelApplication(true);
Assert.assertTrue(true);
} catch (Error | Exception e) {
logger.error(e.getMessage());
}
}
}

6
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java

@ -19,12 +19,6 @@ package org.apache.dolphinscheduler.server.worker.task.sqoop;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceHdfsParameter;
import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceHiveParameter;
import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceMysqlParameter;
import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetHdfsParameter;
import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetHiveParameter;
import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetMysqlParameter;
import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.server.worker.task.TaskProps; import org.apache.dolphinscheduler.server.worker.task.TaskProps;
import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.SqoopJobGenerator; import org.apache.dolphinscheduler.server.worker.task.sqoop.generator.SqoopJobGenerator;

1
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java

@ -16,7 +16,6 @@
*/ */
package org.apache.dolphinscheduler.service.log; package org.apache.dolphinscheduler.service.log;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.remote.NettyRemotingClient; import org.apache.dolphinscheduler.remote.NettyRemotingClient;
import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.log.*; import org.apache.dolphinscheduler.remote.command.log.*;

6
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java

@ -116,12 +116,12 @@ public class ProcessService {
ProcessInstance processInstance = constructProcessInstance(command, host); ProcessInstance processInstance = constructProcessInstance(command, host);
//cannot construct process instance, return null; //cannot construct process instance, return null;
if(processInstance == null){ if(processInstance == null){
logger.error("scan command, command parameter is error: %s", command.toString()); logger.error("scan command, command parameter is error: {}", command);
moveToErrorCommand(command, "process instance is null"); moveToErrorCommand(command, "process instance is null");
return null; return null;
} }
if(!checkThreadNum(command, validThreadNum)){ if(!checkThreadNum(command, validThreadNum)){
logger.info("there is not enough thread for this command: {}",command.toString() ); logger.info("there is not enough thread for this command: {}", command);
return setWaitingThreadProcess(command, processInstance); return setWaitingThreadProcess(command, processInstance);
} }
processInstance.setCommandType(command.getCommandType()); processInstance.setCommandType(command.getCommandType());
@ -991,7 +991,7 @@ public class ProcessService {
return insertQueueResult; return insertQueueResult;
}catch (Exception e){ }catch (Exception e){
logger.error("submit task to queue Exception: ", e); logger.error("submit task to queue Exception: ", e);
logger.error("task queue error : %s", JSONUtils.toJson(taskInstance)); logger.error("task queue error : {}", JSONUtils.toJson(taskInstance));
return false; return false;
} }
} }

18
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/AbstractZKClient.java

@ -65,14 +65,16 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator {
if (splits.length != Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH){ if (splits.length != Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH){
return; return;
} }
String str = splits[0] + Constants.COMMA StringBuilder sb = new StringBuilder();
+ splits[1] + Constants.COMMA sb.append(splits[0]).append(Constants.COMMA)
+ OSUtils.cpuUsage() + Constants.COMMA .append(splits[1]).append(Constants.COMMA)
+ OSUtils.memoryUsage() + Constants.COMMA .append(OSUtils.cpuUsage()).append(Constants.COMMA)
+ OSUtils.loadAverage() + Constants.COMMA .append(OSUtils.memoryUsage()).append(Constants.COMMA)
+ splits[5] + Constants.COMMA .append(OSUtils.loadAverage()).append(Constants.COMMA)
+ DateUtils.dateToString(new Date()); .append(splits[5]).append(Constants.COMMA)
zkClient.setData().forPath(znode,str.getBytes()); .append(DateUtils.dateToString(new Date()));
zkClient.setData().forPath(znode, sb.toString().getBytes());
} catch (Exception e) { } catch (Exception e) {
logger.error("heartbeat for zk failed", e); logger.error("heartbeat for zk failed", e);

10
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProvider.java

@ -45,4 +45,14 @@ public class DefaultEnsembleProvider implements EnsembleProvider {
public void close() throws IOException { public void close() throws IOException {
//NOP //NOP
} }
@Override
public void setConnectionString(String connectionString) {
//NOP
}
@Override
public boolean updateServerListEnabled() {
return false;
}
} }

3
dolphinscheduler-service/src/test/java/cron/CronUtilsTest.java → dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtilsTest.java

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package cron; package org.apache.dolphinscheduler.service.quartz.cron;
import com.cronutils.builder.CronBuilder; import com.cronutils.builder.CronBuilder;
import com.cronutils.model.Cron; import com.cronutils.model.Cron;
@ -25,7 +25,6 @@ import com.cronutils.model.field.CronFieldName;
import com.cronutils.model.field.expression.*; import com.cronutils.model.field.expression.*;
import org.apache.dolphinscheduler.common.enums.CycleEnum; import org.apache.dolphinscheduler.common.enums.CycleEnum;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.slf4j.Logger; import org.slf4j.Logger;

2
dolphinscheduler-service/src/test/java/queue/BaseTaskQueueTest.java → dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/BaseTaskQueueTest.java

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package queue; package org.apache.dolphinscheduler.service.queue;
import org.apache.dolphinscheduler.service.queue.ITaskQueue; import org.apache.dolphinscheduler.service.queue.ITaskQueue;
import org.apache.dolphinscheduler.service.queue.TaskQueueFactory; import org.apache.dolphinscheduler.service.queue.TaskQueueFactory;

2
dolphinscheduler-service/src/test/java/queue/TaskQueueZKImplTest.java → dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskQueueZKImplTest.java

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package queue; package org.apache.dolphinscheduler.service.queue;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.IpUtils; import org.apache.dolphinscheduler.common.utils.IpUtils;

65
dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/DefaultEnsembleProviderTest.java

@ -0,0 +1,65 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.service.zk;
import org.apache.curator.ensemble.EnsembleProvider;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.IOException;
import static org.junit.Assert.*;
public class DefaultEnsembleProviderTest {
private static final String DEFAULT_SERVER_LIST = "localhost:2181";
@Test
public void startAndClose() {
EnsembleProvider ensembleProvider = new DefaultEnsembleProvider(DEFAULT_SERVER_LIST);
try {
ensembleProvider.start();
} catch (Exception e) {
Assert.fail("EnsembleProvider start error: " + e.getMessage());
}
try {
ensembleProvider.close();
} catch (IOException e) {
Assert.fail("EnsembleProvider close error: " + e.getMessage());
}
}
@Test
public void getConnectionString() {
EnsembleProvider ensembleProvider = new DefaultEnsembleProvider(DEFAULT_SERVER_LIST);
Assert.assertEquals(DEFAULT_SERVER_LIST, ensembleProvider.getConnectionString());
}
@Test
public void setConnectionString() {
EnsembleProvider ensembleProvider = new DefaultEnsembleProvider(DEFAULT_SERVER_LIST);
ensembleProvider.setConnectionString("otherHost:2181");
Assert.assertEquals(DEFAULT_SERVER_LIST, ensembleProvider.getConnectionString());
}
@Test
public void updateServerListEnabled() {
EnsembleProvider ensembleProvider = new DefaultEnsembleProvider(DEFAULT_SERVER_LIST);
Assert.assertFalse(ensembleProvider.updateServerListEnabled());
}
}

2
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue

@ -280,7 +280,7 @@
import mDependent from './tasks/dependent' import mDependent from './tasks/dependent'
import mHttp from './tasks/http' import mHttp from './tasks/http'
import mDatax from './tasks/datax' import mDatax from './tasks/datax'
import mConditions from './tasks/CONDITIONS' import mConditions from './tasks/conditions'
import mSqoop from './tasks/sqoop' import mSqoop from './tasks/sqoop'
import mSubProcess from './tasks/sub_process' import mSubProcess from './tasks/sub_process'
import mSelectInput from './_source/selectInput' import mSelectInput from './_source/selectInput'

10
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue

@ -27,17 +27,17 @@
</x-select> </x-select>
<template v-if="isInstance"> <template v-if="isInstance">
<span class="instance-state"> <span class="instance-state">
<i class="iconfont" :class="'icon-' + el.state" v-if="el.state === 'SUCCESS'" data-toggle="tooltip" data-container="body" :title="$t('success')">&#xe607;</i> <em class="iconfont ans-icon-success-solid" :class="'icon-' + el.state" v-if="el.state === 'SUCCESS'" data-toggle="tooltip" data-container="body" :title="$t('success')"></em>
<i class="iconfont" :class="'icon-' + el.state" v-if="el.state === 'WAITING'" data-toggle="tooltip" data-container="body" :title="$t('waiting')">&#xe62a;</i> <em class="iconfont ans-icon-clock" :class="'icon-' + el.state" v-if="el.state === 'WAITING'" data-toggle="tooltip" data-container="body" :title="$t('waiting')"></em>
<i class="iconfont" :class="'icon-' + el.state" v-if="el.state === 'FAILED'" data-toggle="tooltip" data-container="body" :title="$t('failed')">&#xe626;</i> <em class="iconfont ans-icon-fail-solid" :class="'icon-' + el.state" v-if="el.state === 'FAILED'" data-toggle="tooltip" data-container="body" :title="$t('failed')"></em>
</span> </span>
</template> </template>
<span class="operation"> <span class="operation">
<a href="javascript:" class="delete" @click="!isDetails && _remove($index)"> <a href="javascript:" class="delete" @click="!isDetails && _remove($index)">
<i class="iconfont" :class="_isDetails" data-toggle="tooltip" data-container="body" :title="$t('delete')" >&#xe611;</i> <em class="iconfont ans-icon-trash" :class="_isDetails" data-toggle="tooltip" data-container="body" :title="$t('delete')" ></em>
</a> </a>
<a href="javascript:" class="add" @click="!isDetails && _add()" v-if="$index === (dependItemList.length - 1)"> <a href="javascript:" class="add" @click="!isDetails && _add()" v-if="$index === (dependItemList.length - 1)">
<i class="iconfont" :class="_isDetails" data-toggle="tooltip" data-container="body" :title="$t('Add')">&#xe636;</i> <em class="iconfont ans-icon-increase" :class="_isDetails" data-toggle="tooltip" data-container="body" :title="$t('Add')"></em>
</a> </a>
</span> </span>
</div> </div>

13
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/conditions.vue

@ -23,12 +23,8 @@
<a href="javascript:" <a href="javascript:"
@click="!isDetails && _addDep()" @click="!isDetails && _addDep()"
class="add-dep"> class="add-dep">
<i v-if="!isLoading" class="iconfont" :class="_isDetails" data-toggle="tooltip" :title="$t('Add')"> <em v-if="!isLoading" class="ans-icon-increase" :class="_isDetails" data-toggle="tooltip" :title="$t('Add')"></em>
&#xe636; <em v-if="isLoading" class="ans-icon-spinner2 as as-spin" data-toggle="tooltip" :title="$t('Add')"></em>
</i>
<i v-if="isLoading" class="iconfont fa fa-spin" data-toggle="tooltip" :title="$t('Add')">
&#xe6af;
</i>
</a> </a>
</div> </div>
<div class="dep-box"> <div class="dep-box">
@ -44,14 +40,13 @@
@click="!isDetails && _setRelation($index)"> @click="!isDetails && _setRelation($index)">
{{el.relation === 'AND' ? $t('and') : $t('or')}} {{el.relation === 'AND' ? $t('and') : $t('or')}}
</span> </span>
<i class="iconfont dep-delete" <em class="ans-icon-trash dep-delete"
data-toggle="tooltip" data-toggle="tooltip"
data-container="body" data-container="body"
:class="_isDetails" :class="_isDetails"
@click="!isDetails && _deleteDep($index)" @click="!isDetails && _deleteDep($index)"
:title="$t('delete')" > :title="$t('delete')" >
&#xe611; </em>
</i>
<m-node-status <m-node-status
:dependTaskList='dependTaskList' :dependTaskList='dependTaskList'
v-model="el.dependItemList" v-model="el.dependItemList"

49
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/datax.vue

@ -232,6 +232,8 @@
* Processing code highlighting * Processing code highlighting
*/ */
_handlerEditor () { _handlerEditor () {
this._destroyEditor()
// editor // editor
editor = codemirror('code-sql-mirror', { editor = codemirror('code-sql-mirror', {
mode: 'sql', mode: 'sql',
@ -249,9 +251,34 @@
// Monitor keyboard // Monitor keyboard
editor.on('keypress', this.keypress) editor.on('keypress', this.keypress)
editor.on('changes', () => {
this._cacheParams()
})
editor.setValue(this.sql) editor.setValue(this.sql)
return editor return editor
},
_cacheParams () {
this.$emit('on-cache-params', {
dsType: this.dsType,
dataSource: this.rtDatasource,
dtType: this.dtType,
dataTarget: this.rtDatatarget,
sql: editor?editor.getValue():'',
targetTable: this.targetTable,
jobSpeedByte: this.jobSpeedByte * 1024,
jobSpeedRecord: this.jobSpeedRecord,
preStatements: this.preStatements,
postStatements: this.postStatements
});
},
_destroyEditor () {
if (editor) {
editor.toTextArea() // Uninstall
editor.off($('.code-sql-mirror'), 'keypress', this.keypress)
editor.off($('.code-sql-mirror'), 'changes', this.changes)
}
} }
}, },
created () { created () {
@ -286,7 +313,27 @@
editor.off($('.code-sql-mirror'), 'keypress', this.keypress) editor.off($('.code-sql-mirror'), 'keypress', this.keypress)
} }
}, },
computed: {}, watch: {
//Watch the cacheParams
cacheParams (val) {
this._cacheParams();
}
},
computed: {
cacheParams () {
return {
dsType: this.dsType,
dataSource: this.rtDatasource,
dtType: this.dtType,
dataTarget: this.rtDatatarget,
targetTable: this.targetTable,
jobSpeedByte: this.jobSpeedByte * 1024,
jobSpeedRecord: this.jobSpeedRecord,
preStatements: this.preStatements,
postStatements: this.postStatements
}
}
},
components: { mListBox, mDatasource, mLocalParams, mStatementList, mSelectInput } components: { mListBox, mDatasource, mLocalParams, mStatementList, mSelectInput }
} }
</script> </script>

30
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue

@ -129,6 +129,8 @@
* Processing code highlighting * Processing code highlighting
*/ */
_handlerEditor () { _handlerEditor () {
this._destroyEditor()
// editor // editor
editor = codemirror('code-python-mirror', { editor = codemirror('code-python-mirror', {
mode: 'python', mode: 'python',
@ -143,26 +145,45 @@
} }
} }
this.changes = () => {
this._cacheParams()
}
// Monitor keyboard // Monitor keyboard
editor.on('keypress', this.keypress) editor.on('keypress', this.keypress)
editor.on('changes', this.changes)
editor.setValue(this.rawScript) editor.setValue(this.rawScript)
return editor return editor
},
_cacheParams () {
this.$emit('on-cache-params', {
resourceList: this.cacheResourceList,
localParams: this.localParams,
rawScript: editor ? editor.getValue() : ''
});
},
_destroyEditor () {
if (editor) {
editor.toTextArea() // Uninstall
editor.off($('.code-python-mirror'), 'keypress', this.keypress)
editor.off($('.code-python-mirror'), 'changes', this.changes)
}
} }
}, },
watch: { watch: {
//Watch the cacheParams //Watch the cacheParams
cacheParams (val) { cacheParams (val) {
this.$emit('on-cache-params', val); this._cacheParams()
} }
}, },
computed: { computed: {
cacheParams () { cacheParams () {
return { return {
resourceList: this.cacheResourceList, resourceList: this.cacheResourceList,
localParams: this.localParams, localParams: this.localParams
rawScript: editor ? editor.getValue() : ''
} }
} }
}, },
@ -193,8 +214,11 @@
}, 200) }, 200)
}, },
destroyed () { destroyed () {
if (editor) {
editor.toTextArea() // Uninstall editor.toTextArea() // Uninstall
editor.off($('.code-python-mirror'), 'keypress', this.keypress) editor.off($('.code-python-mirror'), 'keypress', this.keypress)
editor.off($('.code-python-mirror'), 'changes', this.changes)
}
}, },
components: { mLocalParams, mListBox, mResources } components: { mLocalParams, mListBox, mResources }
} }

29
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue

@ -163,6 +163,8 @@
* Processing code highlighting * Processing code highlighting
*/ */
_handlerEditor () { _handlerEditor () {
this._destroyEditor()
// editor // editor
editor = codemirror('code-shell-mirror', { editor = codemirror('code-shell-mirror', {
mode: 'shell', mode: 'shell',
@ -177,25 +179,45 @@
} }
} }
this.changes = () => {
this._cacheParams()
}
// Monitor keyboard // Monitor keyboard
editor.on('keypress', this.keypress) editor.on('keypress', this.keypress)
editor.on('changes', this.changes)
editor.setValue(this.rawScript) editor.setValue(this.rawScript)
return editor return editor
},
_cacheParams () {
this.$emit('on-cache-params', {
resourceList: this.cacheResourceList,
localParams: this.localParams,
rawScript: editor ? editor.getValue() : ''
});
},
_destroyEditor () {
if (editor) {
editor.toTextArea() // Uninstall
editor.off($('.code-sql-mirror'), 'keypress', this.keypress)
editor.off($('.code-sql-mirror'), 'changes', this.changes)
}
} }
}, },
watch: { watch: {
//Watch the cacheParams //Watch the cacheParams
cacheParams (val) { cacheParams (val) {
this.$emit('on-cache-params', val); this._cacheParams()
} }
}, },
computed: { computed: {
cacheParams () { cacheParams () {
return { return {
resourceList: this.cacheResourceList, resourceList: this.cacheResourceList,
localParams: this.localParams, localParams: this.localParams
rawScript: editor ? editor.getValue() : ''
} }
} }
}, },
@ -229,6 +251,7 @@
if (editor) { if (editor) {
editor.toTextArea() // Uninstall editor.toTextArea() // Uninstall
editor.off($('.code-shell-mirror'), 'keypress', this.keypress) editor.off($('.code-shell-mirror'), 'keypress', this.keypress)
editor.off($('.code-shell-mirror'), 'changes', this.changes)
} }
}, },
components: { mLocalParams, mListBox, mResources, mScriptBox } components: { mLocalParams, mListBox, mResources, mScriptBox }

44
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/sql.vue

@ -318,6 +318,8 @@
* Processing code highlighting * Processing code highlighting
*/ */
_handlerEditor () { _handlerEditor () {
this._destroyEditor()
// editor // editor
editor = codemirror('code-sql-mirror', { editor = codemirror('code-sql-mirror', {
mode: 'sql', mode: 'sql',
@ -332,9 +334,15 @@
} }
} }
this.changes = () => {
this._cacheParams()
}
// Monitor keyboard // Monitor keyboard
editor.on('keypress', this.keypress) editor.on('keypress', this.keypress)
editor.on('changes', this.changes)
editor.setValue(this.sql) editor.setValue(this.sql)
return editor return editor
@ -351,6 +359,38 @@
this.receivers = res.receivers && res.receivers.split(',') || [] this.receivers = res.receivers && res.receivers.split(',') || []
this.receiversCc = res.receiversCc && res.receiversCc.split(',') || [] this.receiversCc = res.receiversCc && res.receiversCc.split(',') || []
}) })
},
_cacheParams () {
this.$emit('on-cache-params', {
type: this.type,
datasource: this.rtDatasource,
sql: editor ? editor.getValue() : '',
udfs: this.udfs,
sqlType: this.sqlType,
title: this.title,
receivers: this.receivers.join(','),
receiversCc: this.receiversCc.join(','),
showType: (() => {
let showType = this.showType
if (showType.length === 2 && showType[0] === 'ATTACHMENT') {
return [showType[1], showType[0]].join(',')
} else {
return showType.join(',')
}
})(),
localParams: this.localParams,
connParams: this.connParams,
preStatements: this.preStatements,
postStatements: this.postStatements
});
},
_destroyEditor () {
if (editor) {
editor.toTextArea() // Uninstall
editor.off($('.code-sql-mirror'), 'keypress', this.keypress)
editor.off($('.code-sql-mirror'), 'changes', this.changes)
}
} }
}, },
watch: { watch: {
@ -373,7 +413,7 @@
}, },
//Watch the cacheParams //Watch the cacheParams
cacheParams (val) { cacheParams (val) {
this.$emit('on-cache-params', val); this._cacheParams()
} }
}, },
created () { created () {
@ -418,6 +458,7 @@
if (editor) { if (editor) {
editor.toTextArea() // Uninstall editor.toTextArea() // Uninstall
editor.off($('.code-sql-mirror'), 'keypress', this.keypress) editor.off($('.code-sql-mirror'), 'keypress', this.keypress)
editor.off($('.code-sql-mirror'), 'changes', this.changes)
} }
}, },
computed: { computed: {
@ -425,7 +466,6 @@
return { return {
type: this.type, type: this.type,
datasource: this.rtDatasource, datasource: this.rtDatasource,
sql: editor ? editor.getValue() : '',
udfs: this.udfs, udfs: this.udfs,
sqlType: this.sqlType, sqlType: this.sqlType,
title: this.title, title: this.title,

13
pom.xml

@ -59,7 +59,7 @@
<properties> <properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<curator.version>2.12.0</curator.version> <curator.version>4.3.0</curator.version>
<spring.version>5.1.5.RELEASE</spring.version> <spring.version>5.1.5.RELEASE</spring.version>
<spring.boot.version>2.1.3.RELEASE</spring.boot.version> <spring.boot.version>2.1.3.RELEASE</spring.boot.version>
<java.version>1.8</java.version> <java.version>1.8</java.version>
@ -244,6 +244,12 @@
<groupId>org.apache.curator</groupId> <groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId> <artifactId>curator-recipes</artifactId>
<version>${curator.version}</version> <version>${curator.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.zookeeper</groupId> <groupId>org.apache.zookeeper</groupId>
@ -678,6 +684,9 @@
<configuration> <configuration>
<includes> <includes>
<include>**/common/utils/*.java</include> <include>**/common/utils/*.java</include>
<include>**/common/utils/process/ProcessBuilderForWin32Test.java</include>
<include>**/common/utils/process/ProcessEnvironmentForWin32Test.java</include>
<include>**/common/utils/process/ProcessImplForWin32Test.java</include>
<include>**/common/log/*.java</include> <include>**/common/log/*.java</include>
<include>**/common/threadutils/*.java</include> <include>**/common/threadutils/*.java</include>
<include>**/common/graph/*.java</include> <include>**/common/graph/*.java</include>
@ -726,8 +735,10 @@
<include>**/alert/template/AlertTemplateFactoryTest.java</include> <include>**/alert/template/AlertTemplateFactoryTest.java</include>
<include>**/alert/template/impl/DefaultHTMLTemplateTest.java</include> <include>**/alert/template/impl/DefaultHTMLTemplateTest.java</include>
<include>**/server/worker/task/datax/DataxTaskTest.java</include> <include>**/server/worker/task/datax/DataxTaskTest.java</include>
<include>**/server/worker/task/shell/ShellTaskTest.java</include>
<include>**/server/worker/task/sqoop/SqoopTaskTest.java</include> <include>**/server/worker/task/sqoop/SqoopTaskTest.java</include>
<include>**/server/utils/DataxUtilsTest.java</include> <include>**/server/utils/DataxUtilsTest.java</include>
<include>**/service/zk/DefaultEnsembleProviderTest.java</include>
</includes> </includes>
<!-- <skip>true</skip> --> <!-- <skip>true</skip> -->
</configuration> </configuration>

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save