Browse Source

Merge pull request #2 from apache/dev

update
pull/3/MERGE
yinancx 4 years ago committed by GitHub
parent
commit
fcc66d8252
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 128
      .gitignore
  2. BIN
      .mvn/wrapper/maven-wrapper.jar
  3. 467
      ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-application.xml
  4. 6
      ambari_plugin/common-services/DOLPHIN/1.3.3/alerts.json
  5. 7
      ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-alert.xml
  6. 16
      ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-application-api.xml
  7. 0
      ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-common.xml
  8. 206
      ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-datasource.xml
  9. 0
      ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-env.xml
  10. 88
      ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-master.xml
  11. 23
      ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-quartz.xml
  12. 67
      ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-worker.xml
  13. 76
      ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-zookeeper.xml
  14. 6
      ambari_plugin/common-services/DOLPHIN/1.3.3/metainfo.xml
  15. 0
      ambari_plugin/common-services/DOLPHIN/1.3.3/package/alerts/alert_dolphin_scheduler_status.py
  16. 3
      ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_alert_service.py
  17. 3
      ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_api_service.py
  18. 46
      ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_env.py
  19. 4
      ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_logger_service.py
  20. 3
      ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_master_service.py
  21. 3
      ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_worker_service.py
  22. 85
      ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/params.py
  23. 0
      ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/service_check.py
  24. 0
      ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/status_params.py
  25. 0
      ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/alert.properties.j2
  26. 0
      ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/application-api.properties.j2
  27. 0
      ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/common.properties.j2
  28. 20
      ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/datasource.properties.j2
  29. 13
      ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/dolphin-daemon.sh.j2
  30. 20
      ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/master.properties.j2
  31. 0
      ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/quartz.properties.j2
  32. 20
      ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/worker.properties.j2
  33. 20
      ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/zookeeper.properties.j2
  34. 0
      ambari_plugin/common-services/DOLPHIN/1.3.3/quicklinks/quicklinks.json
  35. 84
      ambari_plugin/common-services/DOLPHIN/1.3.3/themes/theme.json
  36. 2
      docker/build/Dockerfile
  37. 4
      docker/build/startup.sh
  38. 13
      docker/kubernetes/dolphinscheduler/requirements.yaml
  39. 2
      dolphinscheduler-alert/pom.xml
  40. 6
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java
  41. 4
      dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java
  42. 8
      dolphinscheduler-api/pom.xml
  43. 6
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java
  44. 17
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java
  45. 242
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
  46. 42
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java
  47. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskRecordController.java
  48. 23
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java
  49. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java
  50. 28
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CommandStateCount.java
  51. 81
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java
  52. 24
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskStateCount.java
  53. 8
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
  54. 7
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java
  55. 19
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java
  56. 30
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java
  57. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java
  58. 7
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectService.java
  59. 181
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
  60. 7
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java
  61. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java
  62. 20
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java
  63. 51
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java
  64. 24
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java
  65. 169
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java
  66. 1
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java
  67. 117
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
  68. 22
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java
  69. 58
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java
  70. 47
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegexUtils.java
  71. 2
      dolphinscheduler-api/src/main/resources/i18n/messages.properties
  72. 2
      dolphinscheduler-api/src/main/resources/i18n/messages_en_US.properties
  73. 2
      dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties
  74. 1
      dolphinscheduler-api/src/main/resources/logback-api.xml
  75. 4
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java
  76. 72
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java
  77. 8
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java
  78. 16
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java
  79. 137
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java
  80. 90
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java
  81. 24
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java
  82. 312
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java
  83. 23
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskInstanceServiceTest.java
  84. 13
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java
  85. 8
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java
  86. 10
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java
  87. 47
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/RegexUtilsTest.java
  88. 2
      dolphinscheduler-common/pom.xml
  89. 47
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
  90. 18
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java
  91. 41
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SqoopQueryType.java
  92. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ResourceInfo.java
  93. 3
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/conditions/ConditionsParameters.java
  94. 65
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java
  95. 1
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java
  96. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/spark/SparkParameters.java
  97. 5
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
  98. 3
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
  99. 39
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java
  100. 6
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java
  101. Some files were not shown because too many files have changed in this diff Show More

128
.gitignore vendored

@ -4,25 +4,27 @@
.zip
.gz
.DS_Store
.idea
.idea/
dist/
all-dependencies.txt
self-modules.txt
third-party-dependencies.txt
**/target/
.idea/*
.target
.target/
**/**/target/**
target/*
*/target
*/target/*
.settings
.nbproject
.classpath
.project
**/*.iml
*.iml
*.ipr
*.iws
*.tgz
.*.swp
.factorypath
.vim
.tmp
**/node_modules
node_modules
npm-debug.log
.vscode
logs/*
@ -39,10 +41,110 @@ dolphinscheduler-alert/logs/
dolphinscheduler-alert/src/main/resources/alert.properties_bak
dolphinscheduler-alert/src/main/resources/logback.xml
dolphinscheduler-server/src/main/resources/logback.xml
dolphinscheduler-ui/dist/
dolphinscheduler-ui/dist
dolphinscheduler-ui/node
dolphinscheduler-dao/src/main/resources/dao/data_source.properties
dolphinscheduler-ui/dist/css/common.16ac5d9.css
dolphinscheduler-ui/dist/css/home/index.b444b91.css
dolphinscheduler-ui/dist/css/login/index.5866c64.css
dolphinscheduler-ui/dist/js/0.ac94e5d.js
dolphinscheduler-ui/dist/js/0.ac94e5d.js.map
dolphinscheduler-ui/dist/js/1.0b043a3.js
dolphinscheduler-ui/dist/js/1.0b043a3.js.map
dolphinscheduler-ui/dist/js/10.1bce3dc.js
dolphinscheduler-ui/dist/js/10.1bce3dc.js.map
dolphinscheduler-ui/dist/js/11.79f04d8.js
dolphinscheduler-ui/dist/js/11.79f04d8.js.map
dolphinscheduler-ui/dist/js/12.420daa5.js
dolphinscheduler-ui/dist/js/12.420daa5.js.map
dolphinscheduler-ui/dist/js/13.e5bae1c.js
dolphinscheduler-ui/dist/js/13.e5bae1c.js.map
dolphinscheduler-ui/dist/js/14.f2a0dca.js
dolphinscheduler-ui/dist/js/14.f2a0dca.js.map
dolphinscheduler-ui/dist/js/15.45373e8.js
dolphinscheduler-ui/dist/js/15.45373e8.js.map
dolphinscheduler-ui/dist/js/16.fecb0fc.js
dolphinscheduler-ui/dist/js/16.fecb0fc.js.map
dolphinscheduler-ui/dist/js/17.84be279.js
dolphinscheduler-ui/dist/js/17.84be279.js.map
dolphinscheduler-ui/dist/js/18.307ea70.js
dolphinscheduler-ui/dist/js/18.307ea70.js.map
dolphinscheduler-ui/dist/js/19.144db9c.js
dolphinscheduler-ui/dist/js/19.144db9c.js.map
dolphinscheduler-ui/dist/js/2.8b4ef29.js
dolphinscheduler-ui/dist/js/2.8b4ef29.js.map
dolphinscheduler-ui/dist/js/20.4c527e9.js
dolphinscheduler-ui/dist/js/20.4c527e9.js.map
dolphinscheduler-ui/dist/js/21.831b2a2.js
dolphinscheduler-ui/dist/js/21.831b2a2.js.map
dolphinscheduler-ui/dist/js/22.2b4bb2a.js
dolphinscheduler-ui/dist/js/22.2b4bb2a.js.map
dolphinscheduler-ui/dist/js/23.81467ef.js
dolphinscheduler-ui/dist/js/23.81467ef.js.map
dolphinscheduler-ui/dist/js/24.54a00e4.js
dolphinscheduler-ui/dist/js/24.54a00e4.js.map
dolphinscheduler-ui/dist/js/25.8d7bd36.js
dolphinscheduler-ui/dist/js/25.8d7bd36.js.map
dolphinscheduler-ui/dist/js/26.2ec5e78.js
dolphinscheduler-ui/dist/js/26.2ec5e78.js.map
dolphinscheduler-ui/dist/js/27.3ab48c2.js
dolphinscheduler-ui/dist/js/27.3ab48c2.js.map
dolphinscheduler-ui/dist/js/28.363088a.js
dolphinscheduler-ui/dist/js/28.363088a.js.map
dolphinscheduler-ui/dist/js/29.6c5853a.js
dolphinscheduler-ui/dist/js/29.6c5853a.js.map
dolphinscheduler-ui/dist/js/3.a0edb5b.js
dolphinscheduler-ui/dist/js/3.a0edb5b.js.map
dolphinscheduler-ui/dist/js/30.940fdd3.js
dolphinscheduler-ui/dist/js/30.940fdd3.js.map
dolphinscheduler-ui/dist/js/31.168a460.js
dolphinscheduler-ui/dist/js/31.168a460.js.map
dolphinscheduler-ui/dist/js/32.8df6594.js
dolphinscheduler-ui/dist/js/32.8df6594.js.map
dolphinscheduler-ui/dist/js/33.4480bbe.js
dolphinscheduler-ui/dist/js/33.4480bbe.js.map
dolphinscheduler-ui/dist/js/34.b407fe1.js
dolphinscheduler-ui/dist/js/34.b407fe1.js.map
dolphinscheduler-ui/dist/js/35.f340b0a.js
dolphinscheduler-ui/dist/js/35.f340b0a.js.map
dolphinscheduler-ui/dist/js/36.8880c2d.js
dolphinscheduler-ui/dist/js/36.8880c2d.js.map
dolphinscheduler-ui/dist/js/37.ea2a25d.js
dolphinscheduler-ui/dist/js/37.ea2a25d.js.map
dolphinscheduler-ui/dist/js/38.98a59ee.js
dolphinscheduler-ui/dist/js/38.98a59ee.js.map
dolphinscheduler-ui/dist/js/39.a5e958a.js
dolphinscheduler-ui/dist/js/39.a5e958a.js.map
dolphinscheduler-ui/dist/js/4.4ca44db.js
dolphinscheduler-ui/dist/js/4.4ca44db.js.map
dolphinscheduler-ui/dist/js/40.e187b1e.js
dolphinscheduler-ui/dist/js/40.e187b1e.js.map
dolphinscheduler-ui/dist/js/41.0e89182.js
dolphinscheduler-ui/dist/js/41.0e89182.js.map
dolphinscheduler-ui/dist/js/42.341047c.js
dolphinscheduler-ui/dist/js/42.341047c.js.map
dolphinscheduler-ui/dist/js/43.27b8228.js
dolphinscheduler-ui/dist/js/43.27b8228.js.map
dolphinscheduler-ui/dist/js/44.e8869bc.js
dolphinscheduler-ui/dist/js/44.e8869bc.js.map
dolphinscheduler-ui/dist/js/45.8d54901.js
dolphinscheduler-ui/dist/js/45.8d54901.js.map
dolphinscheduler-ui/dist/js/5.e1ed7f3.js
dolphinscheduler-ui/dist/js/5.e1ed7f3.js.map
dolphinscheduler-ui/dist/js/6.241ba07.js
dolphinscheduler-ui/dist/js/6.241ba07.js.map
dolphinscheduler-ui/dist/js/7.ab2e297.js
dolphinscheduler-ui/dist/js/7.ab2e297.js.map
dolphinscheduler-ui/dist/js/8.83ff814.js
dolphinscheduler-ui/dist/js/8.83ff814.js.map
dolphinscheduler-ui/dist/js/9.39cb29f.js
dolphinscheduler-ui/dist/js/9.39cb29f.js.map
dolphinscheduler-ui/dist/js/common.733e342.js
dolphinscheduler-ui/dist/js/common.733e342.js.map
dolphinscheduler-ui/dist/js/home/index.78a5d12.js
dolphinscheduler-ui/dist/js/home/index.78a5d12.js.map
dolphinscheduler-ui/dist/js/login/index.291b8e3.js
dolphinscheduler-ui/dist/js/login/index.291b8e3.js.map
dolphinscheduler-ui/dist/lib/external/
dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/taskInstance/index.vue
/dolphinscheduler-dao/src/main/resources/dao/data_source.properties
.mvn/wrapper/*.jar
!/zookeeper_data/

BIN
.mvn/wrapper/maven-wrapper.jar vendored

Binary file not shown.

467
ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-application.xml

@ -1,467 +0,0 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration>
<property>
<name>spring.datasource.initialSize</name>
<value>5</value>
<description>
Init connection number
</description>
<value-attributes>
<type>int</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.minIdle</name>
<value>5</value>
<description>
Min connection number
</description>
<value-attributes>
<type>int</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.maxActive</name>
<value>50</value>
<description>
Max connection number
</description>
<value-attributes>
<type>int</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.maxWait</name>
<value>60000</value>
<description>
Max wait time for get a connection in milliseconds.
If configuring maxWait, fair locks are enabled by default and concurrency efficiency decreases.
If necessary, unfair locks can be used by configuring the useUnfairLock attribute to true.
</description>
<value-attributes>
<type>int</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.timeBetweenEvictionRunsMillis</name>
<value>60000</value>
<description>
Milliseconds for check to close free connections
</description>
<value-attributes>
<type>int</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.timeBetweenConnectErrorMillis</name>
<value>60000</value>
<description>
The Destroy thread detects the connection interval and closes the physical connection in milliseconds
if the connection idle time is greater than or equal to minEvictableIdleTimeMillis.
</description>
<value-attributes>
<type>int</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.minEvictableIdleTimeMillis</name>
<value>300000</value>
<description>
The longest time a connection remains idle without being evicted, in milliseconds
</description>
<value-attributes>
<type>int</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.validationQuery</name>
<value>SELECT 1</value>
<description>
The SQL used to check whether the connection is valid requires a query statement.
If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work.
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.validationQueryTimeout</name>
<value>3</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>
Check whether the connection is valid for timeout, in seconds
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.testWhileIdle</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description>
When applying for a connection,
if it is detected that the connection is idle longer than time Between Eviction Runs Millis,
validation Query is performed to check whether the connection is valid
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.testOnBorrow</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description>
Execute validation to check if the connection is valid when applying for a connection
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.testOnReturn</name>
<value>false</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description>
Execute validation to check if the connection is valid when the connection is returned
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.defaultAutoCommit</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description>
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.keepAlive</name>
<value>false</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description>
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.poolPreparedStatements</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description>
Open PSCache, specify count PSCache for every connection
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.maxPoolPreparedStatementPerConnectionSize</name>
<value>20</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.spring.datasource.filters</name>
<value>stat,wall,log4j</value>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.connectionProperties</name>
<value>druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000</value>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.mapper-locations</name>
<value>classpath*:/org.apache.dolphinscheduler.dao.mapper/*.xml</value>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.typeEnumsPackage</name>
<value>org.apache.dolphinscheduler.*.enums</value>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.typeAliasesPackage</name>
<value>org.apache.dolphinscheduler.dao.entity</value>
<description>
Entity scan, where multiple packages are separated by a comma or semicolon
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.global-config.db-config.id-type</name>
<value>AUTO</value>
<value-attributes>
<type>value-list</type>
<entries>
<entry>
<value>AUTO</value>
<label>AUTO</label>
</entry>
<entry>
<value>INPUT</value>
<label>INPUT</label>
</entry>
<entry>
<value>ID_WORKER</value>
<label>ID_WORKER</label>
</entry>
<entry>
<value>UUID</value>
<label>UUID</label>
</entry>
</entries>
<selection-cardinality>1</selection-cardinality>
</value-attributes>
<description>
Primary key type AUTO:" database ID AUTO ",
INPUT:" user INPUT ID",
ID_WORKER:" global unique ID (numeric type unique ID)",
UUID:" global unique ID UUID";
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.global-config.db-config.field-strategy</name>
<value>NOT_NULL</value>
<value-attributes>
<type>value-list</type>
<entries>
<entry>
<value>IGNORED</value>
<label>IGNORED</label>
</entry>
<entry>
<value>NOT_NULL</value>
<label>NOT_NULL</label>
</entry>
<entry>
<value>NOT_EMPTY</value>
<label>NOT_EMPTY</label>
</entry>
</entries>
<selection-cardinality>1</selection-cardinality>
</value-attributes>
<description>
Field policy IGNORED:" ignore judgment ",
NOT_NULL:" not NULL judgment "),
NOT_EMPTY:" not NULL judgment"
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.global-config.db-config.column-underline</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.global-config.db-config.logic-delete-value</name>
<value>1</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.global-config.db-config.logic-not-delete-value</name>
<value>0</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.global-config.db-config.banner</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.configuration.map-underscore-to-camel-case</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.configuration.cache-enabled</name>
<value>false</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.configuration.call-setters-on-nulls</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>mybatis-plus.configuration.jdbc-type-for-null</name>
<value>null</value>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.exec.threads</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.exec.task.num</name>
<value>20</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.heartbeat.interval</name>
<value>10</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.task.commit.retryTimes</name>
<value>5</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.task.commit.interval</name>
<value>1000</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.max.cpuload.avg</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.reserved.memory</name>
<value>0.1</value>
<value-attributes>
<type>float</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.exec.threads</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.heartbeat.interval</name>
<value>10</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.fetch.task.num</name>
<value>3</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.max.cpuload.avg</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.reserved.memory</name>
<value>0.1</value>
<value-attributes>
<type>float</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
</configuration>

6
ambari_plugin/common-services/DOLPHIN/1.2.1/alerts.json → ambari_plugin/common-services/DOLPHIN/1.3.3/alerts.json

@ -65,7 +65,7 @@
"enabled": true,
"source": {
"type": "SCRIPT",
"path": "DOLPHIN/1.2.1/package/alerts/alert_dolphin_scheduler_status.py",
"path": "DOLPHIN/1.3.3/package/alerts/alert_dolphin_scheduler_status.py",
"parameters": [
{
@ -98,7 +98,7 @@
"enabled": true,
"source": {
"type": "SCRIPT",
"path": "DOLPHIN/1.2.1/package/alerts/alert_dolphin_scheduler_status.py",
"path": "DOLPHIN/1.3.3/package/alerts/alert_dolphin_scheduler_status.py",
"parameters": [
{
@ -131,7 +131,7 @@
"enabled": true,
"source": {
"type": "SCRIPT",
"path": "DOLPHIN/1.2.1/package/alerts/alert_dolphin_scheduler_status.py",
"path": "DOLPHIN/1.3.3/package/alerts/alert_dolphin_scheduler_status.py",
"parameters": [
{

7
ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-alert.xml → ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-alert.xml

@ -90,13 +90,6 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>xls.file.path</name>
<value>/tmp/xls</value>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>enterprise.wechat.enable</name>
<value>false</value>

16
ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-application-api.xml → ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-application-api.xml

@ -34,6 +34,12 @@
<description>
</description>
</property>
<property>
<name>server.servlet.context-path</name>
<value>/dolphinscheduler/</value>
<description>
</description>
</property>
<property>
<name>spring.servlet.multipart.max-file-size</name>
<value>1024</value>
@ -68,4 +74,14 @@
<value>UTF-8</value>
<description></description>
</property>
<property>
<name>spring.messages.basename</name>
<value>i18n/messages</value>
<description></description>
</property>
<property>
<name>security.authentication.type</name>
<value>PASSWORD</value>
<description></description>
</property>
</configuration>

0
ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-common.xml → ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-common.xml

206
ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-datasource.xml

@ -0,0 +1,206 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration>
<property>
<name>spring.datasource.initialSize</name>
<value>5</value>
<description>
Init connection number
</description>
<value-attributes>
<type>int</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.minIdle</name>
<value>5</value>
<description>
Min connection number
</description>
<value-attributes>
<type>int</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.maxActive</name>
<value>50</value>
<description>
Max connection number
</description>
<value-attributes>
<type>int</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.maxWait</name>
<value>60000</value>
<description>
Max wait time for get a connection in milliseconds.
If configuring maxWait, fair locks are enabled by default and concurrency efficiency decreases.
If necessary, unfair locks can be used by configuring the useUnfairLock attribute to true.
</description>
<value-attributes>
<type>int</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.timeBetweenEvictionRunsMillis</name>
<value>60000</value>
<description>
Milliseconds for check to close free connections
</description>
<value-attributes>
<type>int</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.timeBetweenConnectErrorMillis</name>
<value>60000</value>
<description>
The Destroy thread detects the connection interval and closes the physical connection in milliseconds
if the connection idle time is greater than or equal to minEvictableIdleTimeMillis.
</description>
<value-attributes>
<type>int</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.minEvictableIdleTimeMillis</name>
<value>300000</value>
<description>
The longest time a connection remains idle without being evicted, in milliseconds
</description>
<value-attributes>
<type>int</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.validationQuery</name>
<value>SELECT 1</value>
<description>
The SQL used to check whether the connection is valid requires a query statement.
If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work.
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.validationQueryTimeout</name>
<value>3</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>
Check whether the connection is valid for timeout, in seconds
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.testWhileIdle</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description>
When applying for a connection,
if it is detected that the connection is idle longer than time Between Eviction Runs Millis,
validation Query is performed to check whether the connection is valid
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.testOnBorrow</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description>
Execute validation to check if the connection is valid when applying for a connection
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.testOnReturn</name>
<value>false</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description>
Execute validation to check if the connection is valid when the connection is returned
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.defaultAutoCommit</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description>
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.keepAlive</name>
<value>false</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description>
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.poolPreparedStatements</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description>
Open PSCache, specify count PSCache for every connection
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.maxPoolPreparedStatementPerConnectionSize</name>
<value>20</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.spring.datasource.filters</name>
<value>stat,wall,log4j</value>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>spring.datasource.connectionProperties</name>
<value>druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000</value>
<description></description>
<on-ambari-upgrade add="true"/>
</property>
</configuration>

0
ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-env.xml → ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-env.xml

88
ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-master.xml

@ -0,0 +1,88 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration>
<property>
<name>master.exec.threads</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>master execute thread num</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.exec.task.num</name>
<value>20</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>master execute task number in parallel</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.heartbeat.interval</name>
<value>10</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>master heartbeat interval</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.task.commit.retryTimes</name>
<value>5</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>master commit task retry times</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.task.commit.interval</name>
<value>1000</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>master commit task interval</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.max.cpuload.avg</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>only less than cpu avg load, master server can work. default value : the number of cpu cores * 2</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.reserved.memory</name>
<value>0.3</value>
<description>only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G.</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>master.listen.port</name>
<value>5678</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>master listen port</description>
<on-ambari-upgrade add="true"/>
</property>
</configuration>

23
ambari_plugin/common-services/DOLPHIN/1.2.1/configuration/dolphin-quartz.xml → ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-quartz.xml

@ -106,26 +106,21 @@
<description></description>
</property>
<property>
<name>org.quartz.jobStore.dataSource</name>
<value>myDs</value>
<description></description>
</property>
<property>
<name>org.quartz.dataSource.myDs.connectionProvider.class</name>
<value>org.apache.dolphinscheduler.server.quartz.DruidConnectionProvider</value>
<name>org.quartz.jobStore.acquireTriggersWithinLock</name>
<value>true</value>
<value-attributes>
<type>boolean</type>
</value-attributes>
<description></description>
</property>
<property>
<name>org.quartz.dataSource.myDs.maxConnections</name>
<value>10</value>
<value-attributes>
<type>int</type>
</value-attributes>
<name>org.quartz.jobStore.dataSource</name>
<value>myDs</value>
<description></description>
</property>
<property>
<name>org.quartz.dataSource.myDs.validationQuery</name>
<value>select 1</value>
<name>org.quartz.dataSource.myDs.connectionProvider.class</name>
<value>org.apache.dolphinscheduler.service.quartz.DruidConnectionProvider</value>
<description></description>
</property>
</configuration>

67
ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-worker.xml

@ -0,0 +1,67 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration>
<property>
<name>worker.exec.threads</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>worker execute thread num</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.heartbeat.interval</name>
<value>10</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>worker heartbeat interval</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.max.cpuload.avg</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.reserved.memory</name>
<value>0.3</value>
<description>only larger than reserved memory, worker server can work. default value : physical memory * 1/10, unit is G.</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.listen.port</name>
<value>1234</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>worker listen port</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>worker.groups</name>
<value>default</value>
<description>default worker group</description>
<on-ambari-upgrade add="true"/>
</property>
</configuration>

76
ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-zookeeper.xml

@ -0,0 +1,76 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration>
<property>
<name>zookeeper.dolphinscheduler.root</name>
<value>/dolphinscheduler</value>
<description>
dolphinscheduler root directory
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>zookeeper.session.timeout</name>
<value>300</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>zookeeper.connection.timeout</name>
<value>300</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>zookeeper.retry.base.sleep</name>
<value>100</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>zookeeper.retry.max.sleep</name>
<value>30000</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>
</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
<name>zookeeper.retry.maxtime</name>
<value>5</value>
<value-attributes>
<type>int</type>
</value-attributes>
<description>
</description>
<on-ambari-upgrade add="true"/>
</property>
</configuration>

6
ambari_plugin/common-services/DOLPHIN/1.2.1/metainfo.xml → ambari_plugin/common-services/DOLPHIN/1.3.3/metainfo.xml

@ -22,7 +22,7 @@
<name>DOLPHIN</name>
<displayName>Dolphin Scheduler</displayName>
<comment>分布式易扩展的可视化DAG工作流任务调度系统</comment>
<version>1.2.1</version>
<version>1.3.3</version>
<components>
<component>
<name>DOLPHIN_MASTER</name>
@ -103,7 +103,7 @@
<osFamily>any</osFamily>
<packages>
<package>
<name>apache-dolphinscheduler-incubating-1.2.1*</name>
<name>apache-dolphinscheduler-incubating*</name>
</package>
</packages>
</osSpecific>
@ -134,4 +134,4 @@
</quickLinksConfigurations>
</service>
</services>
</metainfo>
</metainfo>

0
ambari_plugin/common-services/DOLPHIN/1.2.1/package/alerts/alert_dolphin_scheduler_status.py → ambari_plugin/common-services/DOLPHIN/1.3.3/package/alerts/alert_dolphin_scheduler_status.py

3
ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_alert_service.py → ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_alert_service.py

@ -26,7 +26,8 @@ class DolphinAlertService(Script):
import params
env.set_params(params)
self.install_packages(env)
Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True)
Execute(('chmod', '-R', '777', params.dolphin_home))
Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home))
def configure(self, env):
import params

3
ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_api_service.py → ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_api_service.py

@ -26,7 +26,8 @@ class DolphinApiService(Script):
import params
env.set_params(params)
self.install_packages(env)
Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True)
Execute(('chmod', '-R', '777', params.dolphin_home))
Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home))
def configure(self, env):
import params

46
ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_env.py → ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_env.py

@ -42,31 +42,12 @@ def dolphin_env():
create_parents=True
)
Directory(params.dolphin_alert_map['xls.file.path'],
mode=0777,
owner=params.dolphin_user,
group=params.dolphin_group,
create_parents=True
)
Directory(params.dolphin_common_map['data.basedir.path'],
mode=0777,
owner=params.dolphin_user,
group=params.dolphin_group,
create_parents=True
)
Directory(params.dolphin_common_map['data.download.basedir.path'],
mode=0777,
owner=params.dolphin_user,
group=params.dolphin_group,
create_parents=True
)
Directory(params.dolphin_common_map['process.exec.basepath'],
mode=0777,
owner=params.dolphin_user,
group=params.dolphin_group,
create_parents=True
)
File(format(params.dolphin_env_path),
@ -79,11 +60,25 @@ def dolphin_env():
File(format(params.dolphin_bin_dir + "/dolphinscheduler-daemon.sh"),
mode=0755,
content=Template("dolphin-daemon.j2"),
content=Template("dolphin-daemon.sh.j2"),
owner=params.dolphin_user,
group=params.dolphin_group
)
File(format(params.dolphin_conf_dir + "/master.properties"),
mode=0755,
content=Template("master.properties.j2"),
owner=params.dolphin_user,
group=params.dolphin_group
)
File(format(params.dolphin_conf_dir + "/worker.properties"),
mode=0755,
content=Template("worker.properties.j2"),
owner=params.dolphin_user,
group=params.dolphin_group
)
File(format(params.dolphin_conf_dir + "/alert.properties"),
mode=0755,
@ -92,9 +87,9 @@ def dolphin_env():
group=params.dolphin_group
)
File(format(params.dolphin_conf_dir + "/application.properties"),
File(format(params.dolphin_conf_dir + "/datasource.properties"),
mode=0755,
content=Template("application.properties.j2"),
content=Template("datasource.properties.j2"),
owner=params.dolphin_user,
group=params.dolphin_group
)
@ -119,3 +114,10 @@ def dolphin_env():
owner=params.dolphin_user,
group=params.dolphin_group
)
File(format(params.dolphin_conf_dir + "/zookeeper.properties"),
mode=0755,
content=Template("zookeeper.properties.j2"),
owner=params.dolphin_user,
group=params.dolphin_group
)

4
ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_logger_service.py → ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_logger_service.py

@ -26,8 +26,8 @@ class DolphinLoggerService(Script):
import params
env.set_params(params)
self.install_packages(env)
Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True)
Execute(('chmod', '-R', '777', params.dolphin_home))
Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home))
def configure(self, env):
import params
params.pika_slave = True

3
ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_master_service.py → ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_master_service.py

@ -27,7 +27,8 @@ class DolphinMasterService(Script):
import params
env.set_params(params)
self.install_packages(env)
Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True)
Execute(('chmod', '-R', '777', params.dolphin_home))
Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home))
def configure(self, env):
import params

3
ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/dolphin_worker_service.py → ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/dolphin_worker_service.py

@ -26,7 +26,8 @@ class DolphinWorkerService(Script):
import params
env.set_params(params)
self.install_packages(env)
Execute(('chmod', '-R', '777', params.dolphin_home), user=params.dolphin_user, sudo=True)
Execute(('chmod', '-R', '777', params.dolphin_home))
Execute(('chown', '-R', params.dolphin_user + ":" + params.dolphin_group, params.dolphin_home))
def configure(self, env):
import params

85
ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/params.py → ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/params.py

@ -54,11 +54,8 @@ dolphin_env_content = dolphin_env_map['dolphinscheduler-env-content']
# database config
dolphin_database_config = {}
dolphin_database_config['dolphin_database_type'] = dolphin_env_map['dolphin.database.type']
dolphin_database_config['dolphin_database_host'] = dolphin_env_map['dolphin.database.host']
dolphin_database_config['dolphin_database_port'] = dolphin_env_map['dolphin.database.port']
dolphin_database_config['dolphin_database_username'] = dolphin_env_map['dolphin.database.username']
dolphin_database_config['dolphin_database_password'] = dolphin_env_map['dolphin.database.password']
if 'mysql' == dolphin_database_config['dolphin_database_type']:
dolphin_database_config['dolphin_database_driver'] = 'com.mysql.jdbc.Driver'
dolphin_database_config['driverDelegateClass'] = 'org.quartz.impl.jdbcjobstore.StdJDBCDelegate'
@ -72,6 +69,10 @@ else:
+ ':' + dolphin_env_map['dolphin.database.port'] \
+ '/dolphinscheduler'
# application-alert.properties
dolphin_alert_map = {}
wechat_push_url = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=$token'
@ -79,27 +80,22 @@ wechat_token_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=$corpId&
wechat_team_send_msg = '{\"toparty\":\"{toParty}\",\"agentid\":\"{agentId}\",\"msgtype\":\"text\",\"text\":{\"content\":\"{msg}\"},\"safe\":\"0\"}'
wechat_user_send_msg = '{\"touser\":\"{toUser}\",\"agentid\":\"{agentId}\",\"msgtype\":\"markdown\",\"markdown\":{\"content\":\"{msg}\"}}'
dolphin_alert_map['enterprise.wechat.push.ur'] = wechat_push_url
dolphin_alert_map['enterprise.wechat.token.url'] = wechat_token_url
dolphin_alert_map['enterprise.wechat.team.send.msg'] = wechat_team_send_msg
dolphin_alert_map['enterprise.wechat.user.send.msg'] = wechat_user_send_msg
dolphin_alert_map.update(config['configurations']['dolphin-alert'])
dolphin_alert_config_map = config['configurations']['dolphin-alert']
if dolphin_alert_config_map['enterprise.wechat.enable']:
dolphin_alert_map['enterprise.wechat.push.ur'] = wechat_push_url
dolphin_alert_map['enterprise.wechat.token.url'] = wechat_token_url
dolphin_alert_map['enterprise.wechat.team.send.msg'] = wechat_team_send_msg
dolphin_alert_map['enterprise.wechat.user.send.msg'] = wechat_user_send_msg
dolphin_alert_map.update(dolphin_alert_config_map)
# application-api.properties
dolphin_app_api_map = {}
dolphin_app_api_map['logging.config'] = 'classpath:apiserver_logback.xml'
dolphin_app_api_map['spring.messages.basename'] = 'i18n/messages'
dolphin_app_api_map['server.servlet.context-path'] = '/dolphinscheduler/'
dolphin_app_api_map.update(config['configurations']['dolphin-application-api'])
# application-dao.properties
dolphin_application_map = {}
dolphin_application_map['spring.datasource.type'] = 'com.alibaba.druid.pool.DruidDataSource'
dolphin_application_map['spring.datasource.driver-class-name'] = dolphin_database_config['dolphin_database_driver']
dolphin_application_map['spring.datasource.url'] = dolphin_database_config['dolphin_database_url']
dolphin_application_map['spring.datasource.username'] = dolphin_database_config['dolphin_database_username']
dolphin_application_map['spring.datasource.password'] = dolphin_database_config['dolphin_database_password']
dolphin_application_map.update(config['configurations']['dolphin-application'])
# common.properties
dolphin_common_map = {}
@ -118,33 +114,42 @@ else:
dolphin_common_map_tmp = config['configurations']['dolphin-common']
data_basedir_path = dolphin_common_map_tmp['data.basedir.path']
process_exec_basepath = data_basedir_path + '/exec'
data_download_basedir_path = data_basedir_path + '/download'
dolphin_common_map['process.exec.basepath'] = process_exec_basepath
dolphin_common_map['data.download.basedir.path'] = data_download_basedir_path
dolphin_common_map['dolphinscheduler.env.path'] = dolphin_env_path
dolphin_common_map.update(config['configurations']['dolphin-common'])
zookeeperHosts = default("/clusterHostInfo/zookeeper_hosts", [])
if len(zookeeperHosts) > 0 and "clientPort" in config['configurations']['zoo.cfg']:
clientPort = config['configurations']['zoo.cfg']['clientPort']
zookeeperPort = ":" + clientPort + ","
dolphin_common_map['zookeeper.quorum'] = zookeeperPort.join(zookeeperHosts) + ":" + clientPort
# datasource.properties
dolphin_datasource_map = {}
dolphin_datasource_map['spring.datasource.type'] = 'com.alibaba.druid.pool.DruidDataSource'
dolphin_datasource_map['spring.datasource.driver-class-name'] = dolphin_database_config['dolphin_database_driver']
dolphin_datasource_map['spring.datasource.url'] = dolphin_database_config['dolphin_database_url']
dolphin_datasource_map['spring.datasource.username'] = dolphin_database_config['dolphin_database_username']
dolphin_datasource_map['spring.datasource.password'] = dolphin_database_config['dolphin_database_password']
dolphin_datasource_map.update(config['configurations']['dolphin-datasource'])
dolphin_common_map.update(config['configurations']['dolphin-common'])
# master.properties
dolphin_master_map = config['configurations']['dolphin-master']
# quartz.properties
dolphin_quartz_map = {}
dolphin_quartz_map['org.quartz.jobStore.driverDelegateClass'] = dolphin_database_config['driverDelegateClass']
dolphin_quartz_map['org.quartz.dataSource.myDs.driver'] = dolphin_database_config['dolphin_database_driver']
dolphin_quartz_map['org.quartz.dataSource.myDs.URL'] = dolphin_database_config['dolphin_database_url']
dolphin_quartz_map['org.quartz.dataSource.myDs.user'] = dolphin_database_config['dolphin_database_username']
dolphin_quartz_map['org.quartz.dataSource.myDs.password'] = dolphin_database_config['dolphin_database_password']
dolphin_quartz_map.update(config['configurations']['dolphin-quartz'])
# if 'ganglia_server_host' in config['clusterHostInfo'] and \
# len(config['clusterHostInfo']['ganglia_server_host'])>0:
# ganglia_installed = True
# ganglia_server = config['clusterHostInfo']['ganglia_server_host'][0]
# ganglia_report_interval = 60
# else:
# ganglia_installed = False
# worker.properties
dolphin_worker_map = config['configurations']['dolphin-worker']
# zookeeper.properties
dolphin_zookeeper_map={}
zookeeperHosts = default("/clusterHostInfo/zookeeper_hosts", [])
if len(zookeeperHosts) > 0 and "clientPort" in config['configurations']['zoo.cfg']:
clientPort = config['configurations']['zoo.cfg']['clientPort']
zookeeperPort = ":" + clientPort + ","
dolphin_zookeeper_map['zookeeper.quorum'] = zookeeperPort.join(zookeeperHosts) + ":" + clientPort
dolphin_zookeeper_map.update(config['configurations']['dolphin-zookeeper'])
if 'spring.servlet.multipart.max-file-size' in dolphin_app_api_map:
file_size = dolphin_app_api_map['spring.servlet.multipart.max-file-size']
dolphin_app_api_map['spring.servlet.multipart.max-file-size'] = file_size + "MB"
if 'spring.servlet.multipart.max-request-size' in dolphin_app_api_map:
request_size = dolphin_app_api_map['spring.servlet.multipart.max-request-size']
dolphin_app_api_map['spring.servlet.multipart.max-request-size'] = request_size + "MB"

0
ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/service_check.py → ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/service_check.py

0
ambari_plugin/common-services/DOLPHIN/1.2.1/package/scripts/status_params.py → ambari_plugin/common-services/DOLPHIN/1.3.3/package/scripts/status_params.py

0
ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/alert.properties.j2 → ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/alert.properties.j2

0
ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/application-api.properties.j2 → ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/application-api.properties.j2

0
ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/common.properties.j2 → ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/common.properties.j2

20
ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/datasource.properties.j2

@ -0,0 +1,20 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{% for key, value in dolphin_datasource_map.iteritems() -%}
{{key}}={{value}}
{% endfor %}

13
ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/dolphin-daemon.j2 → ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/dolphin-daemon.sh.j2

@ -48,22 +48,19 @@ pid={{dolphin_pidfile_dir}}/$command.pid
cd $DOLPHINSCHEDULER_HOME
if [ "$command" = "api-server" ]; then
LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/apiserver_logback.xml -Dspring.profiles.active=api"
LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-api.xml -Dspring.profiles.active=api"
CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer
elif [ "$command" = "master-server" ]; then
LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/master_logback.xml -Ddruid.mysql.usePingMethod=false"
LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-master.xml -Ddruid.mysql.usePingMethod=false"
CLASS=org.apache.dolphinscheduler.server.master.MasterServer
elif [ "$command" = "worker-server" ]; then
LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/worker_logback.xml -Ddruid.mysql.usePingMethod=false"
LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-worker.xml -Ddruid.mysql.usePingMethod=false"
CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer
elif [ "$command" = "alert-server" ]; then
LOG_FILE="-Dlogback.configurationFile={{dolphin_conf_dir}}/alert_logback.xml"
LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/logback-alert.xml"
CLASS=org.apache.dolphinscheduler.alert.AlertServer
elif [ "$command" = "logger-server" ]; then
CLASS=org.apache.dolphinscheduler.server.rpc.LoggerServer
elif [ "$command" = "combined-server" ]; then
LOG_FILE="-Dlogging.config={{dolphin_conf_dir}}/combined_logback.xml -Dspring.profiles.active=api -Dserver.is-combined-server=true"
CLASS=org.apache.dolphinscheduler.api.CombinedApplicationServer
CLASS=org.apache.dolphinscheduler.server.log.LoggerServer
else
echo "Error: No command named \`$command' was found."
exit 1

20
ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/master.properties.j2

@ -0,0 +1,20 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{% for key, value in dolphin_master_map.iteritems() -%}
{{key}}={{value}}
{% endfor %}

0
ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/quartz.properties.j2 → ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/quartz.properties.j2

20
ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/worker.properties.j2

@ -0,0 +1,20 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{% for key, value in dolphin_worker_map.iteritems() -%}
{{key}}={{value}}
{% endfor %}

20
ambari_plugin/common-services/DOLPHIN/1.3.3/package/templates/zookeeper.properties.j2

@ -0,0 +1,20 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{% for key, value in dolphin_zookeeper_map.iteritems() -%}
{{key}}={{value}}
{% endfor %}

0
ambari_plugin/common-services/DOLPHIN/1.2.1/quicklinks/quicklinks.json → ambari_plugin/common-services/DOLPHIN/1.3.3/quicklinks/quicklinks.json

84
ambari_plugin/common-services/DOLPHIN/1.2.1/themes/theme.json → ambari_plugin/common-services/DOLPHIN/1.3.3/themes/theme.json

@ -151,18 +151,40 @@
"subsection-name": "env-row1-col2"
},
{
"config": "dolphin-common/res.upload.startup.type",
"config": "dolphin-common/resource.storage.type",
"subsection-name": "dynamic-row1-col1"
},
{
"config": "dolphin-common/resource.upload.path",
"subsection-name": "dynamic-row1-col1",
"depends-on": [
{
"configs":[
"dolphin-common/resource.storage.type"
],
"if": "${dolphin-common/resource.storage.type} === HDFS || ${dolphin-common/resource.storage.type} === S3",
"then": {
"property_value_attributes": {
"visible": true
}
},
"else": {
"property_value_attributes": {
"visible": false
}
}
}
]
},
{
"config": "dolphin-common/hdfs.root.user",
"subsection-name": "dynamic-row1-col1",
"depends-on": [
{
"configs":[
"dolphin-common/res.upload.startup.type"
"dolphin-common/resource.storage.type"
],
"if": "${dolphin-common/res.upload.startup.type} === HDFS",
"if": "${dolphin-common/resource.storage.type} === HDFS",
"then": {
"property_value_attributes": {
"visible": true
@ -182,9 +204,9 @@
"depends-on": [
{
"configs":[
"dolphin-common/res.upload.startup.type"
"dolphin-common/resource.storage.type"
],
"if": "${dolphin-common/res.upload.startup.type} === HDFS",
"if": "${dolphin-common/resource.storage.type} === HDFS",
"then": {
"property_value_attributes": {
"visible": true
@ -204,9 +226,9 @@
"depends-on": [
{
"configs":[
"dolphin-common/res.upload.startup.type"
"dolphin-common/resource.storage.type"
],
"if": "${dolphin-common/res.upload.startup.type} === HDFS",
"if": "${dolphin-common/resource.storage.type} === HDFS",
"then": {
"property_value_attributes": {
"visible": true
@ -226,9 +248,9 @@
"depends-on": [
{
"configs":[
"dolphin-common/res.upload.startup.type"
"dolphin-common/resource.storage.type"
],
"if": "${dolphin-common/res.upload.startup.type} === S3",
"if": "${dolphin-common/resource.storage.type} === S3",
"then": {
"property_value_attributes": {
"visible": true
@ -248,9 +270,9 @@
"depends-on": [
{
"configs":[
"dolphin-common/res.upload.startup.type"
"dolphin-common/resource.storage.type"
],
"if": "${dolphin-common/res.upload.startup.type} === S3",
"if": "${dolphin-common/resource.storage.type} === S3",
"then": {
"property_value_attributes": {
"visible": true
@ -270,9 +292,9 @@
"depends-on": [
{
"configs":[
"dolphin-common/res.upload.startup.type"
"dolphin-common/resource.storage.type"
],
"if": "${dolphin-common/res.upload.startup.type} === S3",
"if": "${dolphin-common/resource.storage.type} === S3",
"then": {
"property_value_attributes": {
"visible": true
@ -356,6 +378,28 @@
}
]
},
{
"config": "dolphin-common/kerberos.expire.time",
"subsection-name": "dynamic-row1-col2",
"depends-on": [
{
"configs":[
"dolphin-common/hadoop.security.authentication.startup.state"
],
"if": "${dolphin-common/hadoop.security.authentication.startup.state}",
"then": {
"property_value_attributes": {
"visible": true
}
},
"else": {
"property_value_attributes": {
"visible": false
}
}
}
]
},
{
"config": "dolphin-alert/enterprise.wechat.enable",
"subsection-name": "dynamic-row1-col3"
@ -505,11 +549,17 @@
}
},
{
"config": "dolphin-common/res.upload.startup.type",
"config": "dolphin-common/resource.storage.type",
"widget": {
"type": "combo"
}
},
{
"config": "dolphin-common/resource.upload.path",
"widget": {
"type": "text-field"
}
},
{
"config": "dolphin-common/hdfs.root.user",
"widget": {
@ -570,6 +620,12 @@
"type": "text-field"
}
},
{
"config": "dolphin-common/kerberos.expire.time",
"widget": {
"type": "text-field"
}
},
{
"config": "dolphin-alert/enterprise.wechat.enable",
"widget": {

2
docker/build/Dockerfile

@ -42,7 +42,7 @@ ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz
RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ /opt/dolphinscheduler/
ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
#4. install database, if use mysql as your backend database, the `mysql-client` package should be installed
#4. install database, if use mysql as your backend database, you should append `mysql-client` at the end of the sentence
RUN apk add --update --no-cache postgresql postgresql-contrib
#5. modify nginx

4
docker/build/startup.sh

@ -38,13 +38,13 @@ initDatabase() {
echo "connect ${DATABASE_TYPE} service"
if [ ${DATABASE_TYPE} = "mysql" ]; then
v=$(mysql -h${DATABASE_HOST} -P${DATABASE_PORT} -u${DATABASE_USERNAME} --password=${DATABASE_PASSWORD} -D ${DATABASE_DATABASE} -e "select 1" 2>&1)
if [ "$(echo '${v}' | grep 'ERROR' | wc -l)" -eq 1 ]; then
if [ "$(echo ${v} | grep 'ERROR' | wc -l)" -eq 1 ]; then
echo "Error: Can't connect to database...${v}"
exit 1
fi
else
v=$(sudo -u postgres PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "select 1")
if [ "$(echo '${v}' | grep 'FATAL' | wc -l)" -eq 1 ]; then
if [ "$(echo ${v} | grep 'FATAL' | wc -l)" -eq 1 ]; then
echo "Error: Can't connect to database...${v}"
exit 1
fi

13
ambari_plugin/common-services/DOLPHIN/1.2.1/package/templates/application.properties.j2 → docker/kubernetes/dolphinscheduler/requirements.yaml

@ -14,7 +14,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
{% for key, value in dolphin_application_map.iteritems() -%}
{{key}}={{value}}
{% endfor %}
dependencies:
- name: postgresql
version: 8.x.x
repository: https://charts.bitnami.com/bitnami
condition: postgresql.enabled
- name: zookeeper
version: 5.x.x
repository: https://charts.bitnami.com/bitnami
condition: redis.enabled

2
dolphinscheduler-alert/pom.xml

@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
<version>1.3.2-SNAPSHOT</version>
<version>1.3.4-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-alert</artifactId>
<name>${project.artifactId}</name>

6
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java

@ -104,11 +104,7 @@ public class EnterpriseWeChatUtils {
}
Map<String, String> map = JSONUtils.toMap(resp);
if (map != null) {
return map.get("access_token");
} else {
return null;
}
return map == null ? null : map.get("access_token");
} finally {
httpClient.close();
}

4
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java

@ -74,7 +74,7 @@ public class EmailAlertPluginTest {
alertInfo.setAlertData(alertData);
List<String> list = new ArrayList<String>(){{ add("xx@xx.com"); }};
alertInfo.addProp("receivers", list);
Map<String, Object> ret = plugin.process(alertInfo);
assertFalse(Boolean.parseBoolean(String.valueOf(ret.get(Constants.STATUS))));
// Map<String, Object> ret = plugin.process(alertInfo);
// assertFalse(Boolean.parseBoolean(String.valueOf(ret.get(Constants.STATUS))));
}
}

8
dolphinscheduler-api/pom.xml

@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
<version>1.3.2-SNAPSHOT</version>
<version>1.3.4-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-api</artifactId>
<name>${project.artifactId}</name>
@ -162,6 +162,12 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>

6
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java

@ -86,7 +86,7 @@ public class AccessTokenController extends BaseController {
logger.info("login user {}, create token , userId : {} , token expire time : {} , token : {}", loginUser.getUserName(),
userId, expireTime, token);
Map<String, Object> result = accessTokenService.createToken(userId, expireTime, token);
Map<String, Object> result = accessTokenService.createToken(loginUser, userId, expireTime, token);
return returnDataList(result);
}
@ -106,7 +106,7 @@ public class AccessTokenController extends BaseController {
@RequestParam(value = "userId") int userId,
@RequestParam(value = "expireTime") String expireTime) {
logger.info("login user {}, generate token , userId : {} , token expire time : {}", loginUser, userId, expireTime);
Map<String, Object> result = accessTokenService.generateToken(userId, expireTime);
Map<String, Object> result = accessTokenService.generateToken(loginUser, userId, expireTime);
return returnDataList(result);
}
@ -185,7 +185,7 @@ public class AccessTokenController extends BaseController {
logger.info("login user {}, update token , userId : {} , token expire time : {} , token : {}", loginUser.getUserName(),
userId, expireTime, token);
Map<String, Object> result = accessTokenService.updateToken(id, userId, expireTime, token);
Map<String, Object> result = accessTokenService.updateToken(loginUser, id, userId, expireTime, token);
return returnDataList(result);
}

17
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java

@ -228,21 +228,20 @@ public class ProjectController extends BaseController {
}
/**
* query user created project
* query authorized and user created project
*
* @param loginUser login user
* @return projects which the user create
* @return projects which the user create and authorized
*/
@ApiOperation(value = "queryProjectCreatedByUser", notes = "QUERY_USER_CREATED_PROJECT_NOTES")
@GetMapping(value = "/login-user-created-project")
@ApiOperation(value = "queryProjectCreatedAndAuthorizedByUser", notes = "QUERY_AUTHORIZED_AND_USER_CREATED_PROJECT_NOTES")
@GetMapping(value = "/created-and-authorized-project")
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_USER_CREATED_PROJECT_ERROR)
public Result queryProjectCreatedByUser(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser) {
logger.info("login user {}, query authorized project by user id: {}.",
@ApiException(QUERY_AUTHORIZED_AND_USER_CREATED_PROJECT_ERROR)
public Result queryProjectCreatedAndAuthorizedByUser(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser) {
logger.info("login user {}, query authorized and user created project by user id: {}.",
StringUtils.replaceNRTtoUnderline(loginUser.getUserName()),
StringUtils.replaceNRTtoUnderline(String.valueOf(loginUser.getId())));
Map<String, Object> result = projectService.queryProjectCreatedByUser(loginUser);
Map<String, Object> result = projectService.queryProjectCreatedAndAuthorizedByUser(loginUser);
return returnDataList(result);
}

242
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java

@ -14,23 +14,48 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.controller;
import static org.apache.dolphinscheduler.api.enums.Status.AUTHORIZED_FILE_RESOURCE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.AUTHORIZED_UDF_FUNCTION_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.AUTHORIZE_RESOURCE_TREE;
import static org.apache.dolphinscheduler.api.enums.Status.CREATE_RESOURCE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.CREATE_RESOURCE_FILE_ON_LINE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.CREATE_UDF_FUNCTION_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.DELETE_RESOURCE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.DELETE_UDF_FUNCTION_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.DOWNLOAD_RESOURCE_FILE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.EDIT_RESOURCE_FILE_ON_LINE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_DATASOURCE_BY_TYPE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_RESOURCES_LIST_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_RESOURCES_LIST_PAGING;
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_UDF_FUNCTION_LIST_PAGING_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.RESOURCE_FILE_IS_EMPTY;
import static org.apache.dolphinscheduler.api.enums.Status.RESOURCE_NOT_EXIST;
import static org.apache.dolphinscheduler.api.enums.Status.UNAUTHORIZED_UDF_FUNCTION_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_RESOURCE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_UDF_FUNCTION_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_UDF_FUNCTION_NAME_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.VIEW_RESOURCE_FILE_ON_LINE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.VIEW_UDF_FUNCTION_ERROR;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ApiException;
import org.apache.dolphinscheduler.api.service.ResourcesService;
import org.apache.dolphinscheduler.api.service.UdfFuncService;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.User;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import org.apache.commons.lang.StringUtils;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@ -38,13 +63,21 @@ import org.springframework.core.io.Resource;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestAttribute;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import springfox.documentation.annotations.ApiIgnore;
import java.util.Map;
import static org.apache.dolphinscheduler.api.enums.Status.*;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import springfox.documentation.annotations.ApiIgnore;
/**
* resources controller
@ -56,37 +89,28 @@ public class ResourcesController extends BaseController {
private static final Logger logger = LoggerFactory.getLogger(ResourcesController.class);
@Autowired
private ResourcesService resourceService;
@Autowired
private UdfFuncService udfFuncService;
/**
* create directory
*
* @param loginUser login user
* @param alias alias
* @param description description
* @param type type
* @return create result code
*/
/**
* @param loginUser login user
* @param type type
* @param alias alias
* @param description description
* @param pid parent id
* @param currentDir current directory
* @return
* @return create result code
*/
@ApiOperation(value = "createDirctory", notes = "CREATE_RESOURCE_NOTES")
@ApiImplicitParams({
@ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType = "ResourceType"),
@ApiImplicitParam(name = "name", value = "RESOURCE_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "description", value = "RESOURCE_DESC", dataType = "String"),
@ApiImplicitParam(name = "file", value = "RESOURCE_FILE", required = true, dataType = "MultipartFile")
@ApiImplicitParam(name = "pid", value = "RESOURCE_PID", required = true, dataType = "Int"),
@ApiImplicitParam(name = "currentDir", value = "RESOURCE_CURRENTDIR", required = true, dataType = "String")
})
@PostMapping(value = "/directory/create")
@ApiException(CREATE_RESOURCE_ERROR)
@ -103,12 +127,13 @@ public class ResourcesController extends BaseController {
/**
* create resource
*
* @param loginUser login user
* @param alias alias
* @param description description
* @param type type
* @param file file
* @param loginUser
* @param type
* @param alias
* @param description
* @param file
* @param pid
* @param currentDir
* @return create result code
*/
@ApiOperation(value = "createResource", notes = "CREATE_RESOURCE_NOTES")
@ -116,7 +141,9 @@ public class ResourcesController extends BaseController {
@ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType = "ResourceType"),
@ApiImplicitParam(name = "name", value = "RESOURCE_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "description", value = "RESOURCE_DESC", dataType = "String"),
@ApiImplicitParam(name = "file", value = "RESOURCE_FILE", required = true, dataType = "MultipartFile")
@ApiImplicitParam(name = "file", value = "RESOURCE_FILE", required = true, dataType = "MultipartFile"),
@ApiImplicitParam(name = "pid", value = "RESOURCE_PID", required = true, dataType = "Int"),
@ApiImplicitParam(name = "currentDir", value = "RESOURCE_CURRENTDIR", required = true, dataType = "String")
})
@PostMapping(value = "/create")
@ApiException(CREATE_RESOURCE_ERROR)
@ -135,11 +162,12 @@ public class ResourcesController extends BaseController {
/**
* update resource
*
* @param loginUser login user
* @param alias alias
* @param resourceId resource id
* @param type resource type
* @param loginUser login user
* @param alias alias
* @param resourceId resource id
* @param type resource type
* @param description description
* @param file resource file
* @return update result code
*/
@ApiOperation(value = "updateResource", notes = "UPDATE_RESOURCE_NOTES")
@ -147,7 +175,8 @@ public class ResourcesController extends BaseController {
@ApiImplicitParam(name = "id", value = "RESOURCE_ID", required = true, dataType = "Int", example = "100"),
@ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType = "ResourceType"),
@ApiImplicitParam(name = "name", value = "RESOURCE_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "description", value = "RESOURCE_DESC", dataType = "String")
@ApiImplicitParam(name = "description", value = "RESOURCE_DESC", dataType = "String"),
@ApiImplicitParam(name = "file", value = "RESOURCE_FILE", required = true, dataType = "MultipartFile")
})
@PostMapping(value = "/update")
@ApiException(UPDATE_RESOURCE_ERROR)
@ -155,17 +184,18 @@ public class ResourcesController extends BaseController {
@RequestParam(value = "id") int resourceId,
@RequestParam(value = "type") ResourceType type,
@RequestParam(value = "name") String alias,
@RequestParam(value = "description", required = false) String description) {
logger.info("login user {}, update resource, type: {}, resource alias: {}, desc: {}",
loginUser.getUserName(), type, alias, description);
return resourceService.updateResource(loginUser, resourceId, alias, description, type);
@RequestParam(value = "description", required = false) String description,
@RequestParam(value = "file" ,required = false) MultipartFile file) {
logger.info("login user {}, update resource, type: {}, resource alias: {}, desc: {}, file: {}",
loginUser.getUserName(), type, alias, description, file);
return resourceService.updateResource(loginUser, resourceId, alias, description, type, file);
}
/**
* query resources list
*
* @param loginUser login user
* @param type resource type
* @param type resource type
* @return resource list
*/
@ApiOperation(value = "queryResourceList", notes = "QUERY_RESOURCE_LIST_NOTES")
@ -187,10 +217,10 @@ public class ResourcesController extends BaseController {
* query resources list paging
*
* @param loginUser login user
* @param type resource type
* @param type resource type
* @param searchVal search value
* @param pageNo page number
* @param pageSize page size
* @param pageNo page number
* @param pageSize page size
* @return resource list page
*/
@ApiOperation(value = "queryResourceListPaging", notes = "QUERY_RESOURCE_LIST_PAGING_NOTES")
@ -227,7 +257,7 @@ public class ResourcesController extends BaseController {
/**
* delete resource
*
* @param loginUser login user
* @param loginUser login user
* @param resourceId resource id
* @return delete result code
*/
@ -251,8 +281,8 @@ public class ResourcesController extends BaseController {
* verify resource by alias and type
*
* @param loginUser login user
* @param fullName resource full name
* @param type resource type
* @param fullName resource full name
* @param type resource type
* @return true if the resource name not exists, otherwise return false
*/
@ApiOperation(value = "verifyResourceName", notes = "VERIFY_RESOURCE_NAME_NOTES")
@ -277,10 +307,10 @@ public class ResourcesController extends BaseController {
* query resources jar list
*
* @param loginUser login user
* @param type resource type
* @param type resource type
* @return resource list
*/
@ApiOperation(value = "queryResourceJarList", notes = "QUERY_RESOURCE_LIST_NOTES")
@ApiOperation(value = "queryResourceByProgramType", notes = "QUERY_RESOURCE_LIST_NOTES")
@ApiImplicitParams({
@ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType = "ResourceType")
})
@ -288,10 +318,14 @@ public class ResourcesController extends BaseController {
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_RESOURCES_LIST_ERROR)
public Result queryResourceJarList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "type") ResourceType type
@RequestParam(value = "type") ResourceType type,
@RequestParam(value = "programType",required = false) ProgramType programType
) {
logger.info("query resource list, login user:{}, resource type:{}", loginUser.getUserName(), type.toString());
Map<String, Object> result = resourceService.queryResourceJarList(loginUser, type);
String programTypeName = programType == null ? "" : programType.name();
String userName = loginUser.getUserName();
userName = userName.replaceAll("[\n|\r|\t]", "_");
logger.info("query resource list, login user:{}, resource type:{}, program type:{}", userName,programTypeName);
Map<String, Object> result = resourceService.queryResourceByProgramType(loginUser, type,programType);
return returnDataList(result);
}
@ -299,14 +333,16 @@ public class ResourcesController extends BaseController {
* query resource by full name and type
*
* @param loginUser login user
* @param fullName resource full name
* @param type resource type
* @param fullName resource full name
* @param type resource type
* @param id resource id
* @return true if the resource name not exists, otherwise return false
*/
@ApiOperation(value = "queryResource", notes = "QUERY_BY_RESOURCE_NAME")
@ApiImplicitParams({
@ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType = "ResourceType"),
@ApiImplicitParam(name = "fullName", value = "RESOURCE_FULL_NAME", required = true, dataType = "String")
@ApiImplicitParam(name = "fullName", value = "RESOURCE_FULL_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "id", value = "RESOURCE_ID", required = false, dataType = "Int")
})
@GetMapping(value = "/queryResource")
@ResponseStatus(HttpStatus.OK)
@ -325,10 +361,10 @@ public class ResourcesController extends BaseController {
/**
* view resource file online
*
* @param loginUser login user
* @param resourceId resource id
* @param loginUser login user
* @param resourceId resource id
* @param skipLineNum skip line number
* @param limit limit
* @param limit limit
* @return resource content
*/
@ApiOperation(value = "viewResource", notes = "VIEW_RESOURCE_BY_ID_NOTES")
@ -352,13 +388,14 @@ public class ResourcesController extends BaseController {
/**
* create resource file online
*
* @param loginUser login user
* @param type resource type
* @param fileName file name
* @param fileSuffix file suffix
* @param description description
* @param content content
* @param loginUser
* @param type
* @param fileName
* @param fileSuffix
* @param description
* @param content
* @param pid
* @param currentDir
* @return create result code
*/
@ApiOperation(value = "onlineCreateResource", notes = "ONLINE_CREATE_RESOURCE_NOTES")
@ -367,7 +404,9 @@ public class ResourcesController extends BaseController {
@ApiImplicitParam(name = "fileName", value = "RESOURCE_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "suffix", value = "SUFFIX", required = true, dataType = "String"),
@ApiImplicitParam(name = "description", value = "RESOURCE_DESC", dataType = "String"),
@ApiImplicitParam(name = "content", value = "CONTENT", required = true, dataType = "String")
@ApiImplicitParam(name = "content", value = "CONTENT", required = true, dataType = "String"),
@ApiImplicitParam(name = "pid", value = "RESOURCE_PID", required = true, dataType = "Int"),
@ApiImplicitParam(name = "currentDir", value = "RESOURCE_CURRENTDIR", required = true, dataType = "String")
})
@PostMapping(value = "/online-create")
@ApiException(CREATE_RESOURCE_FILE_ON_LINE_ERROR)
@ -392,9 +431,9 @@ public class ResourcesController extends BaseController {
/**
* edit resource file online
*
* @param loginUser login user
* @param loginUser login user
* @param resourceId resource id
* @param content content
* @param content content
* @return update result code
*/
@ApiOperation(value = "updateResourceContent", notes = "UPDATE_RESOURCE_NOTES")
@ -420,7 +459,7 @@ public class ResourcesController extends BaseController {
/**
* download resource file
*
* @param loginUser login user
* @param loginUser login user
* @param resourceId resource id
* @return resource content
*/
@ -449,14 +488,14 @@ public class ResourcesController extends BaseController {
/**
* create udf function
*
* @param loginUser login user
* @param type udf type
* @param funcName function name
* @param argTypes argument types
* @param database database
* @param loginUser login user
* @param type udf type
* @param funcName function name
* @param argTypes argument types
* @param database database
* @param description description
* @param className class name
* @param resourceId resource id
* @param className class name
* @param resourceId resource id
* @return create result code
*/
@ApiOperation(value = "createUdfFunc", notes = "CREATE_UDF_FUNCTION_NOTES")
@ -490,12 +529,12 @@ public class ResourcesController extends BaseController {
* view udf function
*
* @param loginUser login user
* @param id resource id
* @param id resource id
* @return udf function detail
*/
@ApiOperation(value = "viewUIUdfFunction", notes = "VIEW_UDF_FUNCTION_NOTES")
@ApiImplicitParams({
@ApiImplicitParam(name = "resourceId", value = "RESOURCE_ID", required = true, dataType = "Int", example = "100")
@ApiImplicitParam(name = "id", value = "RESOURCE_ID", required = true, dataType = "Int", example = "100")
})
@GetMapping(value = "/udf-func/update-ui")
@ -512,26 +551,27 @@ public class ResourcesController extends BaseController {
/**
* update udf function
*
* @param loginUser login user
* @param type resource type
* @param funcName function name
* @param argTypes argument types
* @param database data base
* @param loginUser login user
* @param type resource type
* @param funcName function name
* @param argTypes argument types
* @param database data base
* @param description description
* @param resourceId resource id
* @param className class name
* @param udfFuncId udf function id
* @param resourceId resource id
* @param className class name
* @param udfFuncId udf function id
* @return update result code
*/
@ApiOperation(value = "updateUdfFunc", notes = "UPDATE_UDF_FUNCTION_NOTES")
@ApiImplicitParams({
@ApiImplicitParam(name = "id", value = "UDF_ID", required = true, dataType = "Int"),
@ApiImplicitParam(name = "type", value = "UDF_TYPE", required = true, dataType = "UdfType"),
@ApiImplicitParam(name = "funcName", value = "FUNC_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "suffix", value = "CLASS_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "className", value = "CLASS_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "argTypes", value = "ARG_TYPES", dataType = "String"),
@ApiImplicitParam(name = "database", value = "DATABASE_NAME", dataType = "String"),
@ApiImplicitParam(name = "description", value = "UDF_DESC", dataType = "String"),
@ApiImplicitParam(name = "id", value = "RESOURCE_ID", required = true, dataType = "Int", example = "100")
@ApiImplicitParam(name = "resourceId", value = "RESOURCE_ID", required = true, dataType = "Int", example = "100")
})
@PostMapping(value = "/udf-func/update")
@ -556,8 +596,8 @@ public class ResourcesController extends BaseController {
*
* @param loginUser login user
* @param searchVal search value
* @param pageNo page number
* @param pageSize page size
* @param pageNo page number
* @param pageSize page size
* @return udf function list page
*/
@ApiOperation(value = "queryUdfFuncListPaging", notes = "QUERY_UDF_FUNCTION_LIST_PAGING_NOTES")
@ -569,7 +609,7 @@ public class ResourcesController extends BaseController {
@GetMapping(value = "/udf-func/list-paging")
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_UDF_FUNCTION_LIST_PAGING_ERROR)
public Result queryUdfFuncList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
public Result<Object> queryUdfFuncListPaging(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam("pageNo") Integer pageNo,
@RequestParam(value = "searchVal", required = false) String searchVal,
@RequestParam("pageSize") Integer pageSize
@ -586,23 +626,25 @@ public class ResourcesController extends BaseController {
}
/**
* query resource list by type
* query udf func list by type
*
* @param loginUser login user
* @param type resource type
* @param type resource type
* @return resource list
*/
@ApiOperation(value = "queryResourceList", notes = "QUERY_RESOURCE_LIST_NOTES")
@ApiOperation(value = "queryUdfFuncList", notes = "QUERY_UDF_FUNC_LIST_NOTES")
@ApiImplicitParams({
@ApiImplicitParam(name = "type", value = "UDF_TYPE", required = true, dataType = "UdfType")
})
@GetMapping(value = "/udf-func/list")
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_DATASOURCE_BY_TYPE_ERROR)
public Result queryResourceList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
public Result<Object> queryUdfFuncList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam("type") UdfType type) {
logger.info("query datasource list, user:{}, type:{}", loginUser.getUserName(), type);
Map<String, Object> result = udfFuncService.queryResourceList(loginUser, type.ordinal());
String userName = loginUser.getUserName();
userName = userName.replaceAll("[\n|\r|\t]", "_");
logger.info("query udf func list, user:{}, type:{}", userName, type);
Map<String, Object> result = udfFuncService.queryUdfFuncList(loginUser, type.ordinal());
return returnDataList(result);
}
@ -610,7 +652,7 @@ public class ResourcesController extends BaseController {
* verify udf function name can use or not
*
* @param loginUser login user
* @param name name
* @param name name
* @return true if the name can user, otherwise return false
*/
@ApiOperation(value = "verifyUdfFuncName", notes = "VERIFY_UDF_FUNCTION_NAME_NOTES")
@ -655,7 +697,7 @@ public class ResourcesController extends BaseController {
* authorized file resource list
*
* @param loginUser login user
* @param userId user id
* @param userId user id
* @return authorized result
*/
@ApiOperation(value = "authorizedFile", notes = "AUTHORIZED_FILE_NOTES")
@ -677,7 +719,7 @@ public class ResourcesController extends BaseController {
* unauthorized file resource list
*
* @param loginUser login user
* @param userId user id
* @param userId user id
* @return unauthorized result code
*/
@ApiOperation(value = "authorizeResourceTree", notes = "AUTHORIZE_RESOURCE_TREE_NOTES")
@ -699,7 +741,7 @@ public class ResourcesController extends BaseController {
* unauthorized udf function
*
* @param loginUser login user
* @param userId user id
* @param userId user id
* @return unauthorized result code
*/
@ApiOperation(value = "unauthUDFFunc", notes = "UNAUTHORIZED_UDF_FUNC_NOTES")
@ -722,7 +764,7 @@ public class ResourcesController extends BaseController {
* authorized udf function
*
* @param loginUser login user
* @param userId user id
* @param userId user id
* @return authorized result code
*/
@ApiOperation(value = "authUDFFunc", notes = "AUTHORIZED_UDF_FUNC_NOTES")

42
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java

@ -14,8 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.controller;
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_TASK_LIST_PAGING_ERROR;
import org.apache.dolphinscheduler.api.exceptions.ApiException;
import org.apache.dolphinscheduler.api.service.TaskInstanceService;
@ -23,18 +25,29 @@ import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.User;
import io.swagger.annotations.*;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.*;
import springfox.documentation.annotations.ApiIgnore;
import java.util.Map;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestAttribute;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestController;
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_TASK_LIST_PAGING_ERROR;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import springfox.documentation.annotations.ApiIgnore;
/**
* task instance controller
@ -69,6 +82,7 @@ public class TaskInstanceController extends BaseController {
@ApiOperation(value = "queryTaskListPaging", notes = "QUERY_TASK_INSTANCE_LIST_PAGING_NOTES")
@ApiImplicitParams({
@ApiImplicitParam(name = "processInstanceId", value = "PROCESS_INSTANCE_ID", required = false, dataType = "Int", example = "100"),
@ApiImplicitParam(name = "processInstanceName", value = "PROCESS_INSTANCE_NAME", required = false, type = "String"),
@ApiImplicitParam(name = "searchVal", value = "SEARCH_VAL", type = "String"),
@ApiImplicitParam(name = "taskName", value = "TASK_NAME", type = "String"),
@ApiImplicitParam(name = "executorName", value = "EXECUTOR_NAME", type = "String"),
@ -85,6 +99,7 @@ public class TaskInstanceController extends BaseController {
public Result queryTaskListPaging(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName,
@RequestParam(value = "processInstanceId", required = false, defaultValue = "0") Integer processInstanceId,
@RequestParam(value = "processInstanceName", required = false) String processInstanceName,
@RequestParam(value = "searchVal", required = false) String searchVal,
@RequestParam(value = "taskName", required = false) String taskName,
@RequestParam(value = "executorName", required = false) String executorName,
@ -95,11 +110,20 @@ public class TaskInstanceController extends BaseController {
@RequestParam("pageNo") Integer pageNo,
@RequestParam("pageSize") Integer pageSize) {
logger.info("query task instance list, project name:{},process instance:{}, search value:{},task name:{}, executor name: {},state type:{}, host:{}, start:{}, end:{}",
projectName, processInstanceId, searchVal, taskName, executorName, stateType, host, startTime, endTime);
logger.info("query task instance list, projectName:{}, processInstanceId:{}, processInstanceName:{}, search value:{}, taskName:{}, executorName: {}, stateType:{}, host:{}, start:{}, end:{}",
StringUtils.replaceNRTtoUnderline(projectName),
processInstanceId,
StringUtils.replaceNRTtoUnderline(processInstanceName),
StringUtils.replaceNRTtoUnderline(searchVal),
StringUtils.replaceNRTtoUnderline(taskName),
StringUtils.replaceNRTtoUnderline(executorName),
stateType,
StringUtils.replaceNRTtoUnderline(host),
StringUtils.replaceNRTtoUnderline(startTime),
StringUtils.replaceNRTtoUnderline(endTime));
searchVal = ParameterUtils.handleEscapes(searchVal);
Map<String, Object> result = taskInstanceService.queryTaskListPaging(
loginUser, projectName, processInstanceId, taskName, executorName, startTime, endTime, searchVal, stateType, host, pageNo, pageSize);
loginUser, projectName, processInstanceId, processInstanceName, taskName, executorName, startTime, endTime, searchVal, stateType, host, pageNo, pageSize);
return returnDataListPaging(result);
}

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskRecordController.java

@ -22,11 +22,13 @@ import org.apache.dolphinscheduler.api.service.TaskRecordService;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.dao.entity.User;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.*;
import springfox.documentation.annotations.ApiIgnore;
import java.util.Map;
@ -59,7 +61,7 @@ public class TaskRecordController extends BaseController {
* @param taskDate task date
* @param startTime start time
* @param endTime end time
* @param pageNo page numbere
* @param pageNo page number
* @param pageSize page size
* @return task record list
*/

23
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java

@ -29,6 +29,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException;
import org.apache.dolphinscheduler.api.service.TenantService;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.dao.entity.User;
@ -72,7 +73,6 @@ public class TenantController extends BaseController {
*
* @param loginUser login user
* @param tenantCode tenant code
* @param tenantName tenant name
* @param queueId queue id
* @param description description
* @return create result code
@ -80,7 +80,6 @@ public class TenantController extends BaseController {
@ApiOperation(value = "createTenant", notes = "CREATE_TENANT_NOTES")
@ApiImplicitParams({
@ApiImplicitParam(name = "tenantCode", value = "TENANT_CODE", required = true, dataType = "String"),
@ApiImplicitParam(name = "tenantName", value = "TENANT_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "queueId", value = "QUEUE_ID", required = true, dataType = "Int", example = "100"),
@ApiImplicitParam(name = "description", value = "TENANT_DESC", dataType = "String")
@ -90,12 +89,13 @@ public class TenantController extends BaseController {
@ApiException(CREATE_TENANT_ERROR)
public Result createTenant(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "tenantCode") String tenantCode,
@RequestParam(value = "tenantName") String tenantName,
@RequestParam(value = "queueId") int queueId,
@RequestParam(value = "description", required = false) String description) throws Exception {
logger.info("login user {}, create tenant, tenantCode: {}, tenantName: {}, queueId: {}, desc: {}",
loginUser.getUserName(), tenantCode, tenantName, queueId, description);
Map<String, Object> result = tenantService.createTenant(loginUser, tenantCode, tenantName, queueId, description);
String userReplace = StringUtils.replaceNRTtoUnderline(loginUser.getUserName());
String tenantCodeReplace = StringUtils.replaceNRTtoUnderline(tenantCode);
String descReplace = StringUtils.replaceNRTtoUnderline(description);
logger.info("login user {}, create tenant, tenantCode: {}, queueId: {}, desc: {}", userReplace, tenantCodeReplace, queueId, descReplace);
Map<String, Object> result = tenantService.createTenant(loginUser, tenantCode, queueId, description);
return returnDataList(result);
}
@ -157,7 +157,6 @@ public class TenantController extends BaseController {
* @param loginUser login user
* @param id tennat id
* @param tenantCode tennat code
* @param tenantName tennat name
* @param queueId queue id
* @param description description
* @return update result code
@ -166,7 +165,6 @@ public class TenantController extends BaseController {
@ApiImplicitParams({
@ApiImplicitParam(name = "ID", value = "TENANT_ID", required = true, dataType = "Int", example = "100"),
@ApiImplicitParam(name = "tenantCode", value = "TENANT_CODE", required = true, dataType = "String"),
@ApiImplicitParam(name = "tenantName", value = "TENANT_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "queueId", value = "QUEUE_ID", required = true, dataType = "Int", example = "100"),
@ApiImplicitParam(name = "description", value = "TENANT_DESC", type = "String")
@ -177,12 +175,13 @@ public class TenantController extends BaseController {
public Result updateTenant(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "id") int id,
@RequestParam(value = "tenantCode") String tenantCode,
@RequestParam(value = "tenantName") String tenantName,
@RequestParam(value = "queueId") int queueId,
@RequestParam(value = "description", required = false) String description) throws Exception {
logger.info("login user {}, updateProcessInstance tenant, tenantCode: {}, tenantName: {}, queueId: {}, description: {}",
loginUser.getUserName(), tenantCode, tenantName, queueId, description);
Map<String, Object> result = tenantService.updateTenant(loginUser, id, tenantCode, tenantName, queueId, description);
String userReplace = StringUtils.replaceNRTtoUnderline(loginUser.getUserName());
String tenantCodeReplace = StringUtils.replaceNRTtoUnderline(tenantCode);
String descReplace = StringUtils.replaceNRTtoUnderline(description);
logger.info("login user {}, create tenant, tenantCode: {}, queueId: {}, desc: {}", userReplace, tenantCodeReplace, queueId, descReplace);
Map<String, Object> result = tenantService.updateTenant(loginUser, id, tenantCode, queueId, description);
return returnDataList(result);
}

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java

@ -166,7 +166,7 @@ public class UsersController extends BaseController {
@RequestParam(value = "state", required = false) int state) throws Exception {
logger.info("login user {}, updateProcessInstance user, userName: {}, email: {}, tenantId: {}, userPassword: {}, phone: {}, user queue: {}, state: {}",
loginUser.getUserName(), userName, email, tenantId, Constants.PASSWORD_DEFAULT, phone, queue, state);
Map<String, Object> result = usersService.updateUser(id, userName, userPassword, email, tenantId, phone, queue, state);
Map<String, Object> result = usersService.updateUser(loginUser, id, userName, userPassword, email, tenantId, phone, queue, state);
return returnDataList(result);
}

28
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CommandStateCount.java

@ -57,4 +57,32 @@ public class CommandStateCount {
public void setCommandState(CommandType commandState) {
this.commandState = commandState;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
CommandStateCount that = (CommandStateCount) o;
if (errorCount != that.errorCount) {
return false;
}
if (normalCount != that.normalCount) {
return false;
}
return commandState == that.commandState;
}
@Override
public int hashCode() {
int result = errorCount;
result = 31 * result + normalCount;
result = 31 * result + (commandState != null ? commandState.hashCode() : 0);
return result;
}
}

81
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java

@ -19,8 +19,10 @@ package org.apache.dolphinscheduler.api.dto;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* task count dto
@ -43,77 +45,16 @@ public class TaskCountDto {
}
private void countTaskDtos(List<ExecuteStatusCount> taskInstanceStateCounts) {
int submittedSuccess = 0;
int runningExecution = 0;
int delayExecution = 0;
int readyPause = 0;
int pause = 0;
int readyStop = 0;
int stop = 0;
int failure = 0;
int success = 0;
int needFaultTolerance = 0;
int kill = 0;
int waittingThread = 0;
Map<ExecutionStatus, Integer> statusCountMap = taskInstanceStateCounts.stream()
.collect(Collectors.toMap(ExecuteStatusCount::getExecutionStatus, ExecuteStatusCount::getCount, Integer::sum));
for (ExecuteStatusCount taskInstanceStateCount : taskInstanceStateCounts) {
ExecutionStatus status = taskInstanceStateCount.getExecutionStatus();
totalCount += taskInstanceStateCount.getCount();
switch (status) {
case SUBMITTED_SUCCESS:
submittedSuccess += taskInstanceStateCount.getCount();
break;
case RUNNING_EXECUTION:
runningExecution += taskInstanceStateCount.getCount();
break;
case DELAY_EXECUTION:
delayExecution += taskInstanceStateCount.getCount();
break;
case READY_PAUSE:
readyPause += taskInstanceStateCount.getCount();
break;
case PAUSE:
pause += taskInstanceStateCount.getCount();
break;
case READY_STOP:
readyStop += taskInstanceStateCount.getCount();
break;
case STOP:
stop += taskInstanceStateCount.getCount();
break;
case FAILURE:
failure += taskInstanceStateCount.getCount();
break;
case SUCCESS:
success += taskInstanceStateCount.getCount();
break;
case NEED_FAULT_TOLERANCE:
needFaultTolerance += taskInstanceStateCount.getCount();
break;
case KILL:
kill += taskInstanceStateCount.getCount();
break;
case WAITTING_THREAD:
waittingThread += taskInstanceStateCount.getCount();
break;
taskCountDtos = Arrays.stream(ExecutionStatus.values())
.map(status -> new TaskStateCount(status, statusCountMap.getOrDefault(status, 0)))
.collect(Collectors.toList());
default:
break;
}
}
this.taskCountDtos = new ArrayList<>();
this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.SUBMITTED_SUCCESS, submittedSuccess));
this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.RUNNING_EXECUTION, runningExecution));
this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.DELAY_EXECUTION, delayExecution));
this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.READY_PAUSE, readyPause));
this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.PAUSE, pause));
this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.READY_STOP, readyStop));
this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.STOP, stop));
this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.FAILURE, failure));
this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.SUCCESS, success));
this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.NEED_FAULT_TOLERANCE, needFaultTolerance));
this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.KILL, kill));
this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.WAITTING_THREAD, waittingThread));
totalCount = taskCountDtos.stream()
.mapToInt(TaskStateCount::getCount)
.sum();
}
public List<TaskStateCount> getTaskCountDtos() {

24
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskStateCount.java

@ -47,4 +47,28 @@ public class TaskStateCount {
public void setTaskStateType(ExecutionStatus taskStateType) {
this.taskStateType = taskStateType;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
TaskStateCount that = (TaskStateCount) o;
if (count != that.count) {
return false;
}
return taskStateType == that.taskStateType;
}
@Override
public int hashCode() {
int result = count;
result = 31 * result + (taskStateType != null ? taskStateType.hashCode() : 0);
return result;
}
}

8
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java

@ -36,7 +36,7 @@ public enum Status {
USER_NAME_NULL(10004, "user name is null", "用户名不能为空"),
HDFS_OPERATION_ERROR(10006, "hdfs operation error", "hdfs操作错误"),
TASK_INSTANCE_NOT_FOUND(10008, "task instance not found", "任务实例不存在"),
TENANT_NAME_EXIST(10009, "tenant code {0} already exists", "租户编码[{0}]已存在"),
TENANT_CODE_EXIST(10009, "tenant code {0} already exists", "租户编码[{0}]已存在"),
USER_NOT_EXIST(10010, "user {0} not exists", "用户[{0}]不存在"),
ALERT_GROUP_NOT_EXIST(10011, "alarm group not found", "告警组不存在"),
ALERT_GROUP_EXIST(10012, "alarm group already exists", "告警组名称已存在"),
@ -133,7 +133,7 @@ public enum Status {
QUERY_TASK_INSTANCE_LOG_ERROR(10103, "view task instance log error", "查询任务实例日志错误"),
DOWNLOAD_TASK_INSTANCE_LOG_FILE_ERROR(10104, "download task instance log file error", "下载任务日志文件错误"),
CREATE_PROCESS_DEFINITION(10105, "create process definition", "创建工作流错误"),
VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR(10106, "verify process definition name unique error", "工作流名称已存在"),
VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR(10106, "verify process definition name unique error", "工作流定义名称已存在"),
UPDATE_PROCESS_DEFINITION_ERROR(10107, "update process definition error", "更新工作流定义错误"),
RELEASE_PROCESS_DEFINITION_ERROR(10108, "release process definition error", "上线工作流错误"),
QUERY_DATAIL_OF_PROCESS_DEFINITION_ERROR(10109, "query datail of process definition error", "查询工作流详细信息错误"),
@ -193,7 +193,9 @@ public enum Status {
BATCH_COPY_PROCESS_DEFINITION_ERROR(10159, "batch copy process definition error", "复制工作流错误"),
BATCH_MOVE_PROCESS_DEFINITION_ERROR(10160, "batch move process definition error", "移动工作流错误"),
QUERY_WORKFLOW_LINEAGE_ERROR(10161, "query workflow lineage error", "查询血缘失败"),
DELETE_PROCESS_DEFINITION_BY_ID_FAIL(10162,"delete process definition by id fail, for there are {0} process instances in executing using it", "删除工作流定义失败,有[{0}]个运行中的工作流实例正在使用"),
QUERY_AUTHORIZED_AND_USER_CREATED_PROJECT_ERROR(10162, "query authorized and user created project error error", "查询授权的和用户创建的项目错误"),
DELETE_PROCESS_DEFINITION_BY_ID_FAIL(10163,"delete process definition by id fail, for there are {0} process instances in executing using it", "删除工作流定义失败,有[{0}]个运行中的工作流实例正在使用"),
CHECK_TENANT_CODE_ERROR(10164, "Please enter the English tenant code", "请输入英文租户编码"),
UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found", "UDF函数不存在"),
UDF_FUNCTION_EXISTS(20002, "UDF function already exists", "UDF函数已存在"),

7
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java

@ -44,7 +44,8 @@ public interface AccessTokenService {
* @param token token string
* @return create result code
*/
Map<String, Object> createToken(int userId, String expireTime, String token);
Map<String, Object> createToken(User loginUser, int userId, String expireTime, String token);
/**
* generate token
@ -53,7 +54,7 @@ public interface AccessTokenService {
* @param expireTime token expire time
* @return token string
*/
Map<String, Object> generateToken(int userId, String expireTime);
Map<String, Object> generateToken(User loginUser, int userId, String expireTime);
/**
* delete access token
@ -73,5 +74,5 @@ public interface AccessTokenService {
* @param token token string
* @return update result code
*/
Map<String, Object> updateToken(int id, int userId, String expireTime, String token);
Map<String, Object> updateToken(User loginUser, int id, int userId, String expireTime, String token);
}

19
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java

@ -95,6 +95,25 @@ public class BaseService {
}
/**
* check
*
* @param result result
* @param bool bool
* @param userNoOperationPerm status
* @return check result
*/
protected boolean check(Map<String, Object> result, boolean bool, Status userNoOperationPerm) {
//only admin can operate
if (bool) {
result.put(Constants.STATUS, userNoOperationPerm);
result.put(Constants.MSG, userNoOperationPerm.getMsg());
return true;
}
return false;
}
/**
* get cookie info by name
*

30
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java

@ -499,7 +499,13 @@ public class DataSourceService extends BaseService {
String address = buildAddress(type, host, port, connectType);
Map<String, Object> parameterMap = new LinkedHashMap<String, Object>(6);
String jdbcUrl = address + "/" + database;
String jdbcUrl;
if (DbType.SQLSERVER == type) {
jdbcUrl = address + ";databaseName=" + database;
} else {
jdbcUrl = address + "/" + database;
}
if (Constants.ORACLE.equals(type.name())) {
parameterMap.put(Constants.ORACLE_DB_CONNECT_TYPE, connectType);
}
@ -533,19 +539,17 @@ public class DataSourceService extends BaseService {
(type == DbType.HIVE || type == DbType.SPARK)) {
parameterMap.put(Constants.PRINCIPAL, principal);
}
if (other != null && !"".equals(other)) {
Map<String, String> map = JSONUtils.toMap(other);
if (map.size() > 0) {
StringBuilder otherSb = new StringBuilder();
for (Map.Entry<String, String> entry : map.entrySet()) {
otherSb.append(String.format("%s=%s%s", entry.getKey(), entry.getValue(), separator));
}
if (!Constants.DB2.equals(type.name())) {
otherSb.deleteCharAt(otherSb.length() - 1);
}
parameterMap.put(Constants.OTHER, otherSb);
}
Map<String, String> map = JSONUtils.toMap(other);
if (map != null) {
StringBuilder otherSb = new StringBuilder();
for (Map.Entry<String, String> entry: map.entrySet()) {
otherSb.append(String.format("%s=%s%s", entry.getKey(), entry.getValue(), separator));
}
if (!Constants.DB2.equals(type.name())) {
otherSb.deleteCharAt(otherSb.length() - 1);
}
parameterMap.put(Constants.OTHER, otherSb);
}
if (logger.isDebugEnabled()) {

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java

@ -305,10 +305,10 @@ public class ProcessInstanceService extends BaseService {
private void addDependResultForTaskList(List<TaskInstance> taskInstanceList) throws IOException {
for (TaskInstance taskInstance : taskInstanceList) {
if (taskInstance.getTaskType().equalsIgnoreCase(TaskType.DEPENDENT.toString())) {
Result logResult = loggerService.queryLog(
Result<String> logResult = loggerService.queryLog(
taskInstance.getId(), 0, 4098);
if (logResult.getCode() == Status.SUCCESS.ordinal()) {
String log = (String) logResult.getData();
String log = logResult.getData();
Map<String, DependResult> resultMap = parseLogForDependentResult(log);
taskInstance.setDependentResult(JSONUtils.toJsonString(resultMap));
}

7
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectService.java

@ -120,4 +120,11 @@ public interface ProjectService {
*/
Map<String, Object> queryAllProjectList();
/**
* query authorized and user create project list by user id
* @param loginUser
* @return
*/
Map<String, Object> queryProjectCreatedAndAuthorizedByUser(User loginUser);
}

181
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java

@ -29,6 +29,7 @@ import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.utils.*;
import org.apache.dolphinscheduler.dao.entity.*;
@ -87,7 +88,7 @@ public class ResourcesService extends BaseService {
* @param currentDir current directory
* @return create directory result
*/
@Transactional(rollbackFor = RuntimeException.class)
@Transactional(rollbackFor = Exception.class)
public Result createDirectory(User loginUser,
String name,
String description,
@ -101,8 +102,11 @@ public class ResourcesService extends BaseService {
putMsg(result, Status.HDFS_NOT_STARTUP);
return result;
}
String fullName = "/".equals(currentDir) ? String.format("%s%s",currentDir,name):String.format("%s/%s",currentDir,name);
String fullName = currentDir.equals("/") ? String.format("%s%s",currentDir,name):String.format("%s/%s",currentDir,name);
result = verifyResourceName(fullName,type,loginUser);
if (!result.getCode().equals(Status.SUCCESS.getCode())) {
return result;
}
if (pid != -1) {
Resource parentResource = resourcesMapper.selectById(pid);
@ -165,7 +169,7 @@ public class ResourcesService extends BaseService {
* @param currentDir current directory
* @return create result code
*/
@Transactional(rollbackFor = RuntimeException.class)
@Transactional(rollbackFor = Exception.class)
public Result createResource(User loginUser,
String name,
String desc,
@ -230,7 +234,7 @@ public class ResourcesService extends BaseService {
}
// check resoure name exists
String fullName = "/".equals(currentDir) ? String.format("%s%s",currentDir,name):String.format("%s/%s",currentDir,name);
String fullName = currentDir.equals("/") ? String.format("%s%s",currentDir,name):String.format("%s/%s",currentDir,name);
if (checkResourceExists(fullName, 0, type.ordinal())) {
logger.error("resource {} has exist, can't recreate", name);
putMsg(result, Status.RESOURCE_EXIST);
@ -288,14 +292,16 @@ public class ResourcesService extends BaseService {
* @param name name
* @param desc description
* @param type resource type
* @param file resource file
* @return update result code
*/
@Transactional(rollbackFor = RuntimeException.class)
@Transactional(rollbackFor = Exception.class)
public Result updateResource(User loginUser,
int resourceId,
String name,
String desc,
ResourceType type) {
ResourceType type,
MultipartFile file) {
Result result = new Result();
// if resource upload startup
@ -315,7 +321,7 @@ public class ResourcesService extends BaseService {
return result;
}
if (name.equals(resource.getAlias()) && desc.equals(resource.getDescription())) {
if (file == null && name.equals(resource.getAlias()) && desc.equals(resource.getDescription())) {
putMsg(result, Status.SUCCESS);
return result;
}
@ -331,6 +337,42 @@ public class ResourcesService extends BaseService {
return result;
}
if (file != null) {
// file is empty
if (file.isEmpty()) {
logger.error("file is empty: {}", file.getOriginalFilename());
putMsg(result, Status.RESOURCE_FILE_IS_EMPTY);
return result;
}
// file suffix
String fileSuffix = FileUtils.suffix(file.getOriginalFilename());
String nameSuffix = FileUtils.suffix(name);
// determine file suffix
if (!(StringUtils.isNotEmpty(fileSuffix) && fileSuffix.equalsIgnoreCase(nameSuffix))) {
/**
* rename file suffix and original suffix must be consistent
*/
logger.error("rename file suffix and original suffix must be consistent: {}", file.getOriginalFilename());
putMsg(result, Status.RESOURCE_SUFFIX_FORBID_CHANGE);
return result;
}
//If resource type is UDF, only jar packages are allowed to be uploaded, and the suffix must be .jar
if (Constants.UDF.equals(type.name()) && !JAR.equalsIgnoreCase(FileUtils.suffix(originFullName))) {
logger.error(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg());
putMsg(result, Status.UDF_RESOURCE_SUFFIX_NOT_JAR);
return result;
}
if (file.getSize() > Constants.MAX_FILE_SIZE) {
logger.error("file size is too large: {}", file.getOriginalFilename());
putMsg(result, Status.RESOURCE_SIZE_EXCEED_LIMIT);
return result;
}
}
// query tenant by user id
String tenantCode = getTenantCode(resource.getUserId(),result);
if (StringUtils.isEmpty(tenantCode)){
@ -380,31 +422,61 @@ public class ResourcesService extends BaseService {
}
// updateResource data
List<Integer> childrenResource = listAllChildren(resource,false);
Date now = new Date();
resource.setAlias(name);
resource.setFullName(fullName);
resource.setDescription(desc);
resource.setUpdateTime(now);
if (file != null) {
resource.setFileName(file.getOriginalFilename());
resource.setSize(file.getSize());
}
try {
resourcesMapper.updateById(resource);
if (resource.isDirectory() && CollectionUtils.isNotEmpty(childrenResource)) {
String matcherFullName = Matcher.quoteReplacement(fullName);
List<Resource> childResourceList = new ArrayList<>();
List<Resource> resourceList = resourcesMapper.listResourceByIds(childrenResource.toArray(new Integer[childrenResource.size()]));
childResourceList = resourceList.stream().map(t -> {
t.setFullName(t.getFullName().replaceFirst(originFullName, matcherFullName));
t.setUpdateTime(now);
return t;
}).collect(Collectors.toList());
resourcesMapper.batchUpdateResource(childResourceList);
if (resource.isDirectory()) {
List<Integer> childrenResource = listAllChildren(resource,false);
if (CollectionUtils.isNotEmpty(childrenResource)) {
String matcherFullName = Matcher.quoteReplacement(fullName);
List<Resource> childResourceList = new ArrayList<>();
Integer[] childResIdArray = childrenResource.toArray(new Integer[childrenResource.size()]);
List<Resource> resourceList = resourcesMapper.listResourceByIds(childResIdArray);
childResourceList = resourceList.stream().map(t -> {
t.setFullName(t.getFullName().replaceFirst(originFullName, matcherFullName));
t.setUpdateTime(now);
return t;
}).collect(Collectors.toList());
resourcesMapper.batchUpdateResource(childResourceList);
if (ResourceType.UDF.equals(resource.getType())) {
List<UdfFunc> udfFuncs = udfFunctionMapper.listUdfByResourceId(childResIdArray);
if (CollectionUtils.isNotEmpty(udfFuncs)) {
udfFuncs = udfFuncs.stream().map(t -> {
t.setResourceName(t.getResourceName().replaceFirst(originFullName, matcherFullName));
t.setUpdateTime(now);
return t;
}).collect(Collectors.toList());
udfFunctionMapper.batchUpdateUdfFunc(udfFuncs);
}
}
}
} else if (ResourceType.UDF.equals(resource.getType())) {
List<UdfFunc> udfFuncs = udfFunctionMapper.listUdfByResourceId(new Integer[]{resourceId});
if (CollectionUtils.isNotEmpty(udfFuncs)) {
udfFuncs = udfFuncs.stream().map(t -> {
t.setResourceName(fullName);
t.setUpdateTime(now);
return t;
}).collect(Collectors.toList());
udfFunctionMapper.batchUpdateUdfFunc(udfFuncs);
}
}
putMsg(result, Status.SUCCESS);
Map<Object, Object> dataMap = new BeanMap(resource);
Map<String, Object> resultMap = new HashMap<>();
Map<String, Object> resultMap = new HashMap<>(5);
for (Map.Entry<Object, Object> entry: dataMap.entrySet()) {
if (!Constants.CLASS.equalsIgnoreCase(entry.getKey().toString())) {
resultMap.put(entry.getKey().toString(), entry.getValue());
@ -415,11 +487,31 @@ public class ResourcesService extends BaseService {
logger.error(Status.UPDATE_RESOURCE_ERROR.getMsg(), e);
throw new ServiceException(Status.UPDATE_RESOURCE_ERROR);
}
// if name unchanged, return directly without moving on HDFS
if (originResourceName.equals(name)) {
if (originResourceName.equals(name) && file == null) {
return result;
}
if (file != null) {
// fail upload
if (!upload(loginUser, fullName, file, type)) {
logger.error("upload resource: {} file: {} failed.", name, file.getOriginalFilename());
putMsg(result, Status.HDFS_OPERATION_ERROR);
throw new RuntimeException(String.format("upload resource: %s file: %s failed.", name, file.getOriginalFilename()));
}
if (!fullName.equals(originFullName)) {
try {
HadoopUtils.getInstance().delete(originHdfsFileName,false);
} catch (IOException e) {
logger.error(e.getMessage(),e);
throw new RuntimeException(String.format("delete resource: %s failed.", originFullName));
}
}
return result;
}
// get the path of dest file in hdfs
String destHdfsFileName = HadoopUtils.getHdfsFileName(resource.getType(),tenantCode,fullName);
@ -449,7 +541,7 @@ public class ResourcesService extends BaseService {
*/
public Map<String, Object> queryResourceListPaging(User loginUser, int direcotryId, ResourceType type, String searchVal, Integer pageNo, Integer pageSize) {
HashMap<String, Object> result = new HashMap<>();
HashMap<String, Object> result = new HashMap<>(5);
Page<Resource> page = new Page(pageNo, pageSize);
int userId = loginUser.getId();
if (isAdmin(loginUser)) {
@ -550,7 +642,7 @@ public class ResourcesService extends BaseService {
*/
public Map<String, Object> queryResourceList(User loginUser, ResourceType type) {
Map<String, Object> result = new HashMap<>();
Map<String, Object> result = new HashMap<>(5);
int userId = loginUser.getId();
if(isAdmin(loginUser)){
@ -565,21 +657,33 @@ public class ResourcesService extends BaseService {
}
/**
* query resource list
* query resource list by program type
*
* @param loginUser login user
* @param type resource type
* @return resource list
*/
public Map<String, Object> queryResourceJarList(User loginUser, ResourceType type) {
public Map<String, Object> queryResourceByProgramType(User loginUser, ResourceType type, ProgramType programType) {
Map<String, Object> result = new HashMap<>();
Map<String, Object> result = new HashMap<>(5);
String suffix = ".jar";
int userId = loginUser.getId();
if(isAdmin(loginUser)){
userId = 0;
}
if (programType != null) {
switch (programType) {
case JAVA:
break;
case SCALA:
break;
case PYTHON:
suffix = ".py";
break;
}
}
List<Resource> allResourceList = resourcesMapper.queryResourceListAuthored(userId, type.ordinal(),0);
List<Resource> resources = new ResourceFilter(".jar",new ArrayList<>(allResourceList)).filter();
List<Resource> resources = new ResourceFilter(suffix,new ArrayList<>(allResourceList)).filter();
Visitor resourceTreeVisitor = new ResourceTreeVisitor(resources);
result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren());
putMsg(result,Status.SUCCESS);
@ -829,7 +933,7 @@ public class ResourcesService extends BaseService {
* @param content content
* @return create result code
*/
@Transactional(rollbackFor = RuntimeException.class)
@Transactional(rollbackFor = Exception.class)
public Result onlineCreateResource(User loginUser, ResourceType type, String fileName, String fileSuffix, String desc, String content,int pid,String currentDirectory) {
Result result = new Result();
// if resource upload startup
@ -852,12 +956,25 @@ public class ResourcesService extends BaseService {
}
String name = fileName.trim() + "." + nameSuffix;
String fullName = "/".equals(currentDirectory) ? String.format("%s%s",currentDirectory,name):String.format("%s/%s",currentDirectory,name);
String fullName = currentDirectory.equals("/") ? String.format("%s%s",currentDirectory,name):String.format("%s/%s",currentDirectory,name);
result = verifyResourceName(fullName,type,loginUser);
if (!result.getCode().equals(Status.SUCCESS.getCode())) {
return result;
}
if (pid != -1) {
Resource parentResource = resourcesMapper.selectById(pid);
if (parentResource == null) {
putMsg(result, Status.PARENT_RESOURCE_NOT_EXIST);
return result;
}
if (!hasPerm(loginUser, parentResource.getUserId())) {
putMsg(result, Status.USER_NO_OPERATION_PERM);
return result;
}
}
// save data
Date now = new Date();
@ -891,7 +1008,7 @@ public class ResourcesService extends BaseService {
* @param content content
* @return update result cod
*/
@Transactional(rollbackFor = RuntimeException.class)
@Transactional(rollbackFor = Exception.class)
public Result updateResourceContent(int resourceId, String content) {
Result result = new Result();
@ -1096,7 +1213,7 @@ public class ResourcesService extends BaseService {
* @return unauthorized result code
*/
public Map<String, Object> unauthorizedUDFFunction(User loginUser, Integer userId) {
Map<String, Object> result = new HashMap<>();
Map<String, Object> result = new HashMap<>(5);
//only admin can operate
if (checkAdmin(loginUser, result)) {
return result;
@ -1148,7 +1265,7 @@ public class ResourcesService extends BaseService {
* @return authorized result
*/
public Map<String, Object> authorizedFile(User loginUser, Integer userId) {
Map<String, Object> result = new HashMap<>();
Map<String, Object> result = new HashMap<>(5);
if (checkAdmin(loginUser, result)){
return result;
}

7
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java

@ -14,8 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo;
@ -69,7 +69,6 @@ public class TaskInstanceService extends BaseService {
@Autowired
UsersService usersService;
/**
* query task list by project, process instance, task name, task start time, task end time, task status, keyword paging
*
@ -87,7 +86,7 @@ public class TaskInstanceService extends BaseService {
* @return task list page
*/
public Map<String, Object> queryTaskListPaging(User loginUser, String projectName,
Integer processInstanceId, String taskName, String executorName, String startDate,
Integer processInstanceId, String processInstanceName, String taskName, String executorName, String startDate,
String endDate, String searchVal, ExecutionStatus stateType, String host,
Integer pageNo, Integer pageSize) {
Map<String, Object> result = new HashMap<>();
@ -124,7 +123,7 @@ public class TaskInstanceService extends BaseService {
int executorId = usersService.getUserIdByName(executorName);
IPage<TaskInstance> taskInstanceIPage = taskInstanceMapper.queryTaskInstanceListPaging(
page, project.getId(), processInstanceId, searchVal, taskName, executorId, statusArray, host, start, end
page, project.getId(), processInstanceId, processInstanceName, searchVal, taskName, executorId, statusArray, host, start, end
);
Set<String> exclusionSet = new HashSet<>();
exclusionSet.add(Constants.CLASS);

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java

@ -32,7 +32,6 @@ public interface TenantService {
*
* @param loginUser login user
* @param tenantCode tenant code
* @param tenantName tenant name
* @param queueId queue id
* @param desc description
* @return create result code
@ -40,7 +39,6 @@ public interface TenantService {
*/
Map<String, Object> createTenant(User loginUser,
String tenantCode,
String tenantName,
int queueId,
String desc) throws Exception;
@ -61,13 +59,12 @@ public interface TenantService {
* @param loginUser login user
* @param id tennat id
* @param tenantCode tennat code
* @param tenantName tennat name
* @param queueId queue id
* @param desc description
* @return update result code
* @throws Exception exception
*/
Map<String, Object> updateTenant(User loginUser, int id, String tenantCode, String tenantName, int queueId,
Map<String, Object> updateTenant(User loginUser, int id, String tenantCode, int queueId,
String desc) throws Exception;
/**

20
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java

@ -148,7 +148,7 @@ public class UdfFuncService extends BaseService{
*/
public Map<String, Object> queryUdfFuncDetail(int id) {
Map<String, Object> result = new HashMap<>();
Map<String, Object> result = new HashMap<>(5);
UdfFunc udfFunc = udfFuncMapper.selectById(id);
if (udfFunc == null) {
putMsg(result, Status.RESOURCE_NOT_EXIST);
@ -244,7 +244,7 @@ public class UdfFuncService extends BaseService{
* @return udf function list page
*/
public Map<String, Object> queryUdfFuncListPaging(User loginUser, String searchVal, Integer pageNo, Integer pageSize) {
Map<String, Object> result = new HashMap<>();
Map<String, Object> result = new HashMap<>(5);
PageInfo pageInfo = new PageInfo<Resource>(pageNo, pageSize);
@ -276,15 +276,19 @@ public class UdfFuncService extends BaseService{
}
/**
* query data resource by type
* query udf list
*
* @param loginUser login user
* @param type resource type
* @return resource list
* @param type udf type
* @return udf func list
*/
public Map<String, Object> queryResourceList(User loginUser, Integer type) {
Map<String, Object> result = new HashMap<>();
List<UdfFunc> udfFuncList = udfFuncMapper.getUdfFuncByType(loginUser.getId(), type);
public Map<String, Object> queryUdfFuncList(User loginUser, Integer type) {
Map<String, Object> result = new HashMap<>(5);
int userId = loginUser.getId();
if (isAdmin(loginUser)) {
userId = 0;
}
List<UdfFunc> udfFuncList = udfFuncMapper.getUdfFuncByType(userId, type);
result.put(Constants.DATA_LIST, udfFuncList);
putMsg(result, Status.SUCCESS);

51
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java

@ -103,7 +103,7 @@ public class UsersService extends BaseService {
String queue,
int state) throws Exception {
Map<String, Object> result = new HashMap<>();
Map<String, Object> result = new HashMap<>(5);
//check all user params
String msg = this.checkUserParams(userName, userPassword, email, phone);
@ -231,7 +231,7 @@ public class UsersService extends BaseService {
* @return user list page
*/
public Map<String, Object> queryUserList(User loginUser, String searchVal, Integer pageNo, Integer pageSize) {
Map<String, Object> result = new HashMap<>();
Map<String, Object> result = new HashMap<>(5);
if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) {
return result;
@ -253,6 +253,8 @@ public class UsersService extends BaseService {
/**
* updateProcessInstance user
*
*
* @param loginUser
* @param userId user id
* @param userName user name
* @param userPassword user password
@ -263,7 +265,7 @@ public class UsersService extends BaseService {
* @return update result code
* @throws Exception exception
*/
public Map<String, Object> updateUser(int userId,
public Map<String, Object> updateUser(User loginUser, int userId,
String userName,
String userPassword,
String email,
@ -271,16 +273,17 @@ public class UsersService extends BaseService {
String phone,
String queue,
int state) throws Exception {
Map<String, Object> result = new HashMap<>();
Map<String, Object> result = new HashMap<>(5);
result.put(Constants.STATUS, false);
if (check(result, !hasPerm(loginUser, userId), Status.USER_NO_OPERATION_PERM)) {
return result;
}
User user = userMapper.selectById(userId);
if (user == null) {
putMsg(result, Status.USER_NOT_EXIST, userId);
return result;
}
if (StringUtils.isNotEmpty(userName)) {
if (!CheckUtils.checkUserName(userName)){
@ -394,7 +397,7 @@ public class UsersService extends BaseService {
* @throws Exception exception when operate hdfs
*/
public Map<String, Object> deleteUserById(User loginUser, int id) throws Exception {
Map<String, Object> result = new HashMap<>();
Map<String, Object> result = new HashMap<>(5);
//only admin can operate
if (!isAdmin(loginUser)) {
putMsg(result, Status.USER_NO_OPERATION_PERM, id);
@ -434,7 +437,7 @@ public class UsersService extends BaseService {
*/
@Transactional(rollbackFor = RuntimeException.class)
public Map<String, Object> grantProject(User loginUser, int userId, String projectIds) {
Map<String, Object> result = new HashMap<>();
Map<String, Object> result = new HashMap<>(5);
result.put(Constants.STATUS, false);
//only admin can operate
@ -484,7 +487,7 @@ public class UsersService extends BaseService {
*/
@Transactional(rollbackFor = RuntimeException.class)
public Map<String, Object> grantResources(User loginUser, int userId, String resourceIds) {
Map<String, Object> result = new HashMap<>();
Map<String, Object> result = new HashMap<>(5);
//only admin can operate
if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) {
return result;
@ -581,7 +584,7 @@ public class UsersService extends BaseService {
*/
@Transactional(rollbackFor = RuntimeException.class)
public Map<String, Object> grantUDFFunction(User loginUser, int userId, String udfIds) {
Map<String, Object> result = new HashMap<>();
Map<String, Object> result = new HashMap<>(5);
//only admin can operate
if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) {
@ -628,7 +631,7 @@ public class UsersService extends BaseService {
*/
@Transactional(rollbackFor = RuntimeException.class)
public Map<String, Object> grantDataSource(User loginUser, int userId, String datasourceIds) {
Map<String, Object> result = new HashMap<>();
Map<String, Object> result = new HashMap<>(5);
result.put(Constants.STATUS, false);
//only admin can operate
@ -708,7 +711,7 @@ public class UsersService extends BaseService {
* @return user list
*/
public Map<String, Object> queryAllGeneralUsers(User loginUser) {
Map<String, Object> result = new HashMap<>();
Map<String, Object> result = new HashMap<>(5);
//only admin can operate
if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) {
return result;
@ -729,7 +732,7 @@ public class UsersService extends BaseService {
* @return user list
*/
public Map<String, Object> queryUserList(User loginUser) {
Map<String, Object> result = new HashMap<>();
Map<String, Object> result = new HashMap<>(5);
//only admin can operate
if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) {
return result;
@ -773,7 +776,7 @@ public class UsersService extends BaseService {
*/
public Map<String, Object> unauthorizedUser(User loginUser, Integer alertgroupId) {
Map<String, Object> result = new HashMap<>();
Map<String, Object> result = new HashMap<>(5);
//only admin can operate
if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) {
return result;
@ -809,7 +812,7 @@ public class UsersService extends BaseService {
* @return authorized result code
*/
public Map<String, Object> authorizedUser(User loginUser, Integer alertgroupId) {
Map<String, Object> result = new HashMap<>();
Map<String, Object> result = new HashMap<>(5);
//only admin can operate
if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) {
return result;
@ -821,24 +824,6 @@ public class UsersService extends BaseService {
return result;
}
/**
* check
*
* @param result result
* @param bool bool
* @param userNoOperationPerm status
* @return check result
*/
private boolean check(Map<String, Object> result, boolean bool, Status userNoOperationPerm) {
//only admin can operate
if (bool) {
result.put(Constants.STATUS, userNoOperationPerm);
result.put(Constants.MSG, userNoOperationPerm.getMsg());
return true;
}
return false;
}
/**
* @param tenantId tenant id
* @return true if tenant exists, otherwise return false

24
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java

@ -86,9 +86,14 @@ public class AccessTokenServiceImpl extends BaseService implements AccessTokenSe
* @param token token string
* @return create result code
*/
public Map<String, Object> createToken(int userId, String expireTime, String token) {
public Map<String, Object> createToken(User loginUser, int userId, String expireTime, String token) {
Map<String, Object> result = new HashMap<>(5);
if (!hasPerm(loginUser,userId)){
putMsg(result, Status.USER_NO_OPERATION_PERM);
return result;
}
if (userId <= 0) {
throw new IllegalArgumentException("User id should not less than or equals to 0.");
}
@ -118,8 +123,12 @@ public class AccessTokenServiceImpl extends BaseService implements AccessTokenSe
* @param expireTime token expire time
* @return token string
*/
public Map<String, Object> generateToken(int userId, String expireTime) {
public Map<String, Object> generateToken(User loginUser, int userId, String expireTime) {
Map<String, Object> result = new HashMap<>(5);
if (!hasPerm(loginUser,userId)){
putMsg(result, Status.USER_NO_OPERATION_PERM);
return result;
}
String token = EncryptionUtils.getMd5(userId + expireTime + System.currentTimeMillis());
result.put(Constants.DATA_LIST, token);
putMsg(result, Status.SUCCESS);
@ -144,8 +153,8 @@ public class AccessTokenServiceImpl extends BaseService implements AccessTokenSe
return result;
}
if (loginUser.getId() != accessToken.getUserId() &&
loginUser.getUserType() != UserType.ADMIN_USER) {
if (!hasPerm(loginUser,accessToken.getUserId())){
putMsg(result, Status.USER_NO_OPERATION_PERM);
return result;
}
@ -164,9 +173,12 @@ public class AccessTokenServiceImpl extends BaseService implements AccessTokenSe
* @param token token string
* @return update result code
*/
public Map<String, Object> updateToken(int id, int userId, String expireTime, String token) {
public Map<String, Object> updateToken(User loginUser, int id, int userId, String expireTime, String token) {
Map<String, Object> result = new HashMap<>(5);
if (!hasPerm(loginUser,userId)){
putMsg(result, Status.USER_NO_OPERATION_PERM);
return result;
}
AccessToken accessToken = accessTokenMapper.selectById(id);
if (accessToken == null) {
logger.error("access token not exist, access token id {}", id);

169
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java

@ -16,7 +16,6 @@
*/
package org.apache.dolphinscheduler.api.service.impl;
import org.apache.dolphinscheduler.api.dto.CommandStateCount;
import org.apache.dolphinscheduler.api.dto.DefineUserDto;
import org.apache.dolphinscheduler.api.dto.TaskCountDto;
@ -45,14 +44,14 @@ import org.apache.dolphinscheduler.service.process.ProcessService;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@ -62,8 +61,6 @@ import org.springframework.stereotype.Service;
@Service
public class DataAnalysisServiceImpl extends BaseService implements DataAnalysisService {
private static final Logger logger = LoggerFactory.getLogger(DataAnalysisServiceImpl.class);
@Autowired
private ProjectMapper projectMapper;
@ -88,10 +85,6 @@ public class DataAnalysisServiceImpl extends BaseService implements DataAnalysis
@Autowired
private ProcessService processService;
private static final String COMMAND_STATE = "commandState";
private static final String ERROR_COMMAND_STATE = "errorCommandState";
/**
* statistical task instance status data
*
@ -137,16 +130,17 @@ public class DataAnalysisServiceImpl extends BaseService implements DataAnalysis
return result;
}
Date start;
Date end;
try {
Date start = null;
Date end = null;
if (StringUtils.isNotEmpty(startDate) && StringUtils.isNotEmpty(endDate)) {
start = DateUtils.getScheduleDate(startDate);
end = DateUtils.getScheduleDate(endDate);
} catch (Exception e) {
logger.error(e.getMessage(), e);
putErrorRequestParamsMsg(result);
return result;
if (Objects.isNull(start) || Objects.isNull(end)) {
putErrorRequestParamsMsg(result);
return result;
}
}
Integer[] projectIdArray = getProjectIdsArrays(loginUser, projectId);
List<ExecuteStatusCount> processInstanceStateCounts =
instanceStateCounter.apply(start, end, projectIdArray);
@ -204,79 +198,39 @@ public class DataAnalysisServiceImpl extends BaseService implements DataAnalysis
* statistics based on task status execution, failure, completion, wait, total
*/
Date start = null;
if (StringUtils.isNotEmpty(startDate)) {
start = DateUtils.getScheduleDate(startDate);
if (Objects.isNull(start)) {
putErrorRequestParamsMsg(result);
return result;
}
}
Date end = null;
if (startDate != null && endDate != null) {
try {
start = DateUtils.getScheduleDate(startDate);
end = DateUtils.getScheduleDate(endDate);
} catch (Exception e) {
logger.error(e.getMessage(), e);
if (StringUtils.isNotEmpty(endDate)) {
end = DateUtils.getScheduleDate(endDate);
if (Objects.isNull(end)) {
putErrorRequestParamsMsg(result);
return result;
}
}
Integer[] projectIdArray = getProjectIdsArrays(loginUser, projectId);
// count command state
List<CommandCount> commandStateCounts =
commandMapper.countCommandState(
loginUser.getId(),
start,
end,
projectIdArray);
// count normal command state
Map<CommandType, Integer> normalCountCommandCounts = commandMapper.countCommandState(loginUser.getId(), start, end, projectIdArray)
.stream()
.collect(Collectors.toMap(CommandCount::getCommandType, CommandCount::getCount));
// count error command state
List<CommandCount> errorCommandStateCounts =
errorCommandMapper.countCommandState(
start, end, projectIdArray);
// enumMap
Map<CommandType, Map<String, Integer>> dataMap = new EnumMap<>(CommandType.class);
Map<String, Integer> commonCommand = new HashMap<>();
commonCommand.put(COMMAND_STATE, 0);
commonCommand.put(ERROR_COMMAND_STATE, 0);
// init data map
/**
* START_PROCESS, START_CURRENT_TASK_PROCESS, RECOVER_TOLERANCE_FAULT_PROCESS, RECOVER_SUSPENDED_PROCESS,
START_FAILURE_TASK_PROCESS,COMPLEMENT_DATA,SCHEDULER, REPEAT_RUNNING,PAUSE,STOP,RECOVER_WAITTING_THREAD;
*/
dataMap.put(CommandType.START_PROCESS, commonCommand);
dataMap.put(CommandType.START_CURRENT_TASK_PROCESS, commonCommand);
dataMap.put(CommandType.RECOVER_TOLERANCE_FAULT_PROCESS, commonCommand);
dataMap.put(CommandType.RECOVER_SUSPENDED_PROCESS, commonCommand);
dataMap.put(CommandType.START_FAILURE_TASK_PROCESS, commonCommand);
dataMap.put(CommandType.COMPLEMENT_DATA, commonCommand);
dataMap.put(CommandType.SCHEDULER, commonCommand);
dataMap.put(CommandType.REPEAT_RUNNING, commonCommand);
dataMap.put(CommandType.PAUSE, commonCommand);
dataMap.put(CommandType.STOP, commonCommand);
dataMap.put(CommandType.RECOVER_WAITTING_THREAD, commonCommand);
// put command state
for (CommandCount executeStatusCount : commandStateCounts) {
Map<String, Integer> commandStateCountsMap = new HashMap<>(dataMap.get(executeStatusCount.getCommandType()));
commandStateCountsMap.put(COMMAND_STATE, executeStatusCount.getCount());
dataMap.put(executeStatusCount.getCommandType(), commandStateCountsMap);
}
Map<CommandType, Integer> errorCommandCounts = errorCommandMapper.countCommandState(start, end, projectIdArray)
.stream()
.collect(Collectors.toMap(CommandCount::getCommandType, CommandCount::getCount));
// put error command state
for (CommandCount errorExecutionStatus : errorCommandStateCounts) {
Map<String, Integer> errorCommandStateCountsMap = new HashMap<>(dataMap.get(errorExecutionStatus.getCommandType()));
errorCommandStateCountsMap.put(ERROR_COMMAND_STATE, errorExecutionStatus.getCount());
dataMap.put(errorExecutionStatus.getCommandType(), errorCommandStateCountsMap);
}
List<CommandStateCount> list = new ArrayList<>();
for (Map.Entry<CommandType, Map<String, Integer>> next : dataMap.entrySet()) {
CommandStateCount commandStateCount = new CommandStateCount(next.getValue().get(ERROR_COMMAND_STATE),
next.getValue().get(COMMAND_STATE), next.getKey());
list.add(commandStateCount);
}
List<CommandStateCount> list = Arrays.stream(CommandType.values())
.map(commandType -> new CommandStateCount(
errorCommandCounts.getOrDefault(commandType, 0),
normalCountCommandCounts.getOrDefault(commandType, 0),
commandType)
).collect(Collectors.toList());
result.put(Constants.DATA_LIST, list);
putMsg(result, Status.SUCCESS);
@ -311,59 +265,10 @@ public class DataAnalysisServiceImpl extends BaseService implements DataAnalysis
return result;
}
// TODO tasksQueueList and tasksKillList is never updated.
List<String> tasksQueueList = new ArrayList<>();
List<String> tasksKillList = new ArrayList<>();
//TODO need to add detail data info
Map<String, Integer> dataMap = new HashMap<>();
if (loginUser.getUserType() == UserType.ADMIN_USER) {
dataMap.put("taskQueue", tasksQueueList.size());
dataMap.put("taskKill", tasksKillList.size());
result.put(Constants.DATA_LIST, dataMap);
putMsg(result, Status.SUCCESS);
return result;
}
int[] tasksQueueIds = new int[tasksQueueList.size()];
int[] tasksKillIds = new int[tasksKillList.size()];
int i = 0;
for (String taskQueueStr : tasksQueueList) {
if (StringUtils.isNotEmpty(taskQueueStr)) {
String[] splits = taskQueueStr.split("_");
if (splits.length >= 4) {
tasksQueueIds[i++] = Integer.parseInt(splits[3]);
}
}
}
i = 0;
for (String taskKillStr : tasksKillList) {
if (StringUtils.isNotEmpty(taskKillStr)) {
String[] splits = taskKillStr.split("-");
if (splits.length == 2) {
tasksKillIds[i++] = Integer.parseInt(splits[1]);
}
}
}
Integer taskQueueCount = 0;
Integer taskKillCount = 0;
Integer[] projectIds = getProjectIdsArrays(loginUser, projectId);
if (tasksQueueIds.length != 0) {
taskQueueCount = taskInstanceMapper.countTask(
projectIds,
tasksQueueIds);
}
if (tasksKillIds.length != 0) {
taskKillCount = taskInstanceMapper.countTask(projectIds, tasksKillIds);
}
dataMap.put("taskQueue", taskQueueCount);
dataMap.put("taskKill", taskKillCount);
dataMap.put("taskQueue", 0);
dataMap.put("taskKill", 0);
result.put(Constants.DATA_LIST, dataMap);
putMsg(result, Status.SUCCESS);
return result;

1
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java

@ -130,7 +130,6 @@ public class LoggerServiceImpl implements LoggerService {
logClient.getLogBytes(host, Constants.RPC_PORT, taskInstance.getLogPath()));
}
/**
* get host
*

117
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java

@ -48,6 +48,7 @@ import org.apache.dolphinscheduler.common.model.TaskNode;
import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
import org.apache.dolphinscheduler.common.process.ProcessDag;
import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
@ -77,6 +78,7 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
@ -84,6 +86,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
@ -164,7 +167,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
String processDefinitionJson,
String desc,
String locations,
String connects) throws JsonProcessingException {
String connects) {
Map<String, Object> result = new HashMap<>();
Project project = projectMapper.queryByName(projectName);
@ -221,29 +224,38 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
// return processDefinition object with ID
result.put(Constants.DATA_LIST, processDefineMapper.selectById(processDefine.getId()));
putMsg(result, Status.SUCCESS);
result.put("processDefinitionId", processDefine.getId());
result.put(PROCESSDEFINITIONID, processDefine.getId());
return result;
}
/**
* get resource ids
*
* @param processData process data
* @return resource ids
*/
private String getResourceIds(ProcessData processData) {
List<TaskNode> tasks = processData.getTasks();
Set<Integer> resourceIds = new HashSet<>();
StringBuilder sb = new StringBuilder();
if (CollectionUtils.isEmpty(tasks)) {
return sb.toString();
}
for (TaskNode taskNode : tasks) {
String taskParameter = taskNode.getParams();
AbstractParameters params = TaskParametersUtils.getParameters(taskNode.getType(), taskParameter);
if (params == null) {
continue;
}
if (CollectionUtils.isNotEmpty(params.getResourceFilesList())) {
Set<Integer> tempSet = params.getResourceFilesList().stream().map(t -> t.getId()).collect(Collectors.toSet());
Set<Integer> tempSet = params.getResourceFilesList().
stream()
.filter(t -> t.getId() != 0)
.map(ResourceInfo::getId)
.collect(Collectors.toSet());
resourceIds.addAll(tempSet);
}
}
StringBuilder sb = new StringBuilder();
for (int i : resourceIds) {
if (sb.length() > 0) {
sb.append(",");
@ -252,7 +264,6 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
}
return sb.toString();
}
/**
* query process definition list
*
@ -262,7 +273,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
*/
public Map<String, Object> queryProcessDefinitionList(User loginUser, String projectName) {
HashMap<String, Object> result = new HashMap<>();
HashMap<String, Object> result = new HashMap<>(5);
Project project = projectMapper.queryByName(projectName);
Map<String, Object> checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName);
@ -355,10 +366,15 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
* @param connects connects for nodes
* @return update result code
*/
public Map<String, Object> updateProcessDefinition(User loginUser, String projectName, int id, String name,
String processDefinitionJson, String desc,
String locations, String connects) {
Map<String, Object> result = new HashMap<>();
public Map<String, Object> updateProcessDefinition(User loginUser,
String projectName,
int id,
String name,
String processDefinitionJson,
String desc,
String locations,
String connects) {
Map<String, Object> result = new HashMap<>(5);
Project project = projectMapper.queryByName(projectName);
Map<String, Object> checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName);
@ -373,16 +389,24 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
return checkProcessJson;
}
ProcessDefinition processDefine = processService.findProcessDefineById(id);
// check process definition exists
if (processDefine == null) {
// check process definition exists
putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, id);
return result;
} else if (processDefine.getReleaseState() == ReleaseState.ONLINE) {
}
if (processDefine.getReleaseState() == ReleaseState.ONLINE) {
// online can not permit edit
putMsg(result, Status.PROCESS_DEFINE_NOT_ALLOWED_EDIT, processDefine.getName());
return result;
} else {
putMsg(result, Status.SUCCESS);
}
if (!name.equals(processDefine.getName())) {
// check whether the new process define name exist
ProcessDefinition definition = processDefineMapper.verifyByDefineName(project.getId(), name);
if (definition != null) {
putMsg(result, Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR, name);
return result;
}
}
Date now = new Date();
@ -441,11 +465,11 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
if (resultEnum != Status.SUCCESS) {
return checkResult;
}
ProcessDefinition processDefinition = processDefineMapper.queryByDefineName(project.getId(), name);
ProcessDefinition processDefinition = processDefineMapper.verifyByDefineName(project.getId(), name);
if (processDefinition == null) {
putMsg(result, Status.SUCCESS);
} else {
putMsg(result, Status.PROCESS_INSTANCE_EXIST, name);
putMsg(result, Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR, name);
}
return result;
}
@ -461,7 +485,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
@Transactional(rollbackFor = RuntimeException.class)
public Map<String, Object> deleteProcessDefinitionById(User loginUser, String projectName, Integer processDefinitionId) {
Map<String, Object> result = new HashMap<>();
Map<String, Object> result = new HashMap<>(5);
Project project = projectMapper.queryByName(projectName);
Map<String, Object> checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName);
@ -673,6 +697,17 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
}
}
/**
* get export process metadata string
* @param processDefinitionId process definition id
* @param processDefinition process definition
* @return export process metadata string
*/
public String exportProcessMetaDataStr(Integer processDefinitionId, ProcessDefinition processDefinition) {
//create workflow json file
return JSONUtils.toJsonString(exportProcessMetaData(processDefinitionId, processDefinition));
}
/**
* get export process metadata string
*
@ -690,6 +725,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
exportProcessMeta.setProjectName(processDefinition.getProjectName());
exportProcessMeta.setProcessDefinitionName(processDefinition.getName());
exportProcessMeta.setProcessDefinitionJson(processDefinition.getProcessDefinitionJson());
exportProcessMeta.setProcessDefinitionDescription(processDefinition.getDescription());
exportProcessMeta.setProcessDefinitionLocations(processDefinition.getLocations());
exportProcessMeta.setProcessDefinitionConnects(processDefinition.getConnects());
@ -756,7 +792,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
*/
@Transactional(rollbackFor = RuntimeException.class)
public Map<String, Object> importProcessDefinition(User loginUser, MultipartFile file, String currentProjectName) {
Map<String, Object> result = new HashMap<>();
Map<String, Object> result = new HashMap<>(5);
String processMetaJson = FileUtils.file2String(file);
List<ProcessMeta> processMetaList = JSONUtils.toList(processMetaJson, ProcessMeta.class);
@ -851,7 +887,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
processMeta.getProcessDefinitionLocations(),
processMeta.getProcessDefinitionConnects());
putMsg(result, Status.SUCCESS);
} catch (JsonProcessingException e) {
} catch (Exception e) {
logger.error("import process meta json data: {}", e.getMessage(), e);
putMsg(result, Status.IMPORT_PROCESS_DEFINE_ERROR);
}
@ -925,7 +961,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
}
//recursive sub-process parameter correction map key for old process id value for new process id
Map<Integer, Integer> subProcessIdMap = new HashMap<>();
Map<Integer, Integer> subProcessIdMap = new HashMap<>(20);
List<Object> subProcessList = StreamUtils.asStream(jsonArray.elements())
.filter(elem -> checkTaskHasSubProcess(JSONUtils.parseObject(elem.toString()).path("type").asText()))
@ -1213,7 +1249,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
*/
public Map<String, Object> queryProcessDefinitionAllByProjectId(Integer projectId) {
HashMap<String, Object> result = new HashMap<>();
HashMap<String, Object> result = new HashMap<>(5);
List<ProcessDefinition> resourceList = processDefineMapper.queryAllDefinitionList(projectId);
result.put(Constants.DATA_LIST, resourceList);
@ -1423,7 +1459,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
Integer processId,
Project targetProject) throws JsonProcessingException {
Map<String, Object> result = new HashMap<>();
Map<String, Object> result = new HashMap<>(5);
ProcessDefinition processDefinition = processDefineMapper.selectById(processId);
if (processDefinition == null) {
@ -1442,6 +1478,41 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
}
}
/**
* copy process definition
*
* @param loginUser login user
* @param projectName project name
* @param processId process definition id
* @return copy result code
*/
public Map<String, Object> copyProcessDefinition(User loginUser, String projectName, Integer processId) {
Map<String, Object> result = new HashMap<>(5);
Project project = projectMapper.queryByName(projectName);
Map<String, Object> checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName);
Status resultStatus = (Status) checkResult.get(Constants.STATUS);
if (resultStatus != Status.SUCCESS) {
return checkResult;
}
ProcessDefinition processDefinition = processDefineMapper.selectById(processId);
if (processDefinition == null) {
putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, processId);
return result;
} else {
return createProcessDefinition(
loginUser,
projectName,
processDefinition.getName() + "_copy_" + System.currentTimeMillis(),
processDefinition.getProcessDefinitionJson(),
processDefinition.getDescription(),
processDefinition.getLocations(),
processDefinition.getConnects());
}
}
/**
* batch copy process definition
*

22
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java

@ -375,6 +375,28 @@ public class ProjectServiceImpl extends BaseService implements ProjectService {
return result;
}
/**
* query authorized and user create project list by user
*
* @param loginUser login user
* @return
*/
public Map<String, Object> queryProjectCreatedAndAuthorizedByUser(User loginUser) {
Map<String, Object> result = new HashMap<>();
List<Project> projects = null;
if (loginUser.getUserType() == UserType.ADMIN_USER) {
projects = projectMapper.selectList(null);
} else {
projects = projectMapper.queryProjectCreatedAndAuthorizedByUserId(loginUser.getId());
}
result.put(Constants.DATA_LIST, projects);
putMsg(result, Status.SUCCESS);
return result;
}
/**
* check whether have read permission
*

58
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java

@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.BaseService;
import org.apache.dolphinscheduler.api.service.TenantService;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.RegexUtils;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
@ -73,18 +74,16 @@ public class TenantServiceImpl extends BaseService implements TenantService {
/**
* create tenant
*
* @param loginUser login user
* @param loginUser login user
* @param tenantCode tenant code
* @param tenantName tenant name
* @param queueId queue id
* @param desc description
* @param queueId queue id
* @param desc description
* @return create result code
* @throws Exception exception
*/
@Transactional(rollbackFor = Exception.class)
public Map<String, Object> createTenant(User loginUser,
String tenantCode,
String tenantName,
int queueId,
String desc) throws Exception {
@ -94,6 +93,11 @@ public class TenantServiceImpl extends BaseService implements TenantService {
return result;
}
if (RegexUtils.isNumeric(tenantCode)) {
putMsg(result, Status.CHECK_TENANT_CODE_ERROR);
return result;
}
if (checkTenantExists(tenantCode)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, tenantCode);
return result;
@ -107,7 +111,6 @@ public class TenantServiceImpl extends BaseService implements TenantService {
return result;
}
tenant.setTenantCode(tenantCode);
tenant.setTenantName(tenantName);
tenant.setQueueId(queueId);
tenant.setDescription(desc);
tenant.setCreateTime(now);
@ -131,8 +134,8 @@ public class TenantServiceImpl extends BaseService implements TenantService {
*
* @param loginUser login user
* @param searchVal search value
* @param pageNo page number
* @param pageSize page size
* @param pageNo page number
* @param pageSize page size
* @return tenant list page
*/
public Map<String, Object> queryTenantList(User loginUser, String searchVal, Integer pageNo, Integer pageSize) {
@ -157,16 +160,15 @@ public class TenantServiceImpl extends BaseService implements TenantService {
/**
* updateProcessInstance tenant
*
* @param loginUser login user
* @param id tennat id
* @param loginUser login user
* @param id tennat id
* @param tenantCode tennat code
* @param tenantName tennat name
* @param queueId queue id
* @param desc description
* @param queueId queue id
* @param desc description
* @return update result code
* @throws Exception exception
*/
public Map<String, Object> updateTenant(User loginUser, int id, String tenantCode, String tenantName, int queueId,
public Map<String, Object> updateTenant(User loginUser, int id, String tenantCode, int queueId,
String desc) throws Exception {
Map<String, Object> result = new HashMap<>(5);
@ -209,10 +211,6 @@ public class TenantServiceImpl extends BaseService implements TenantService {
tenant.setTenantCode(tenantCode);
}
if (StringUtils.isNotEmpty(tenantName)) {
tenant.setTenantName(tenantName);
}
if (queueId != 0) {
tenant.setQueueId(queueId);
}
@ -229,7 +227,7 @@ public class TenantServiceImpl extends BaseService implements TenantService {
* delete tenant
*
* @param loginUser login user
* @param id tenant id
* @param id tenant id
* @return delete result code
* @throws Exception exception
*/
@ -285,6 +283,26 @@ public class TenantServiceImpl extends BaseService implements TenantService {
return processInstanceMapper.queryByTenantIdAndStatus(tenant.getId(), Constants.NOT_TERMINATED_STATES);
}
/**
* query tenant list
*
* @param tenantCode tenant code
* @return tenant list
*/
public Map<String, Object> queryTenantList(String tenantCode) {
Map<String, Object> result = new HashMap<>(5);
List<Tenant> resourceList = tenantMapper.queryByTenantCode(tenantCode);
if (CollectionUtils.isNotEmpty(resourceList)) {
result.put(Constants.DATA_LIST, resourceList);
putMsg(result, Status.SUCCESS);
} else {
putMsg(result, Status.TENANT_NOT_EXIST);
}
return result;
}
/**
* query tenant list
*
@ -311,7 +329,7 @@ public class TenantServiceImpl extends BaseService implements TenantService {
public Result verifyTenantCode(String tenantCode) {
Result result = new Result();
if (checkTenantExists(tenantCode)) {
putMsg(result, Status.TENANT_NAME_EXIST, tenantCode);
putMsg(result, Status.TENANT_CODE_EXIST, tenantCode);
} else {
putMsg(result, Status.SUCCESS);
}

47
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegexUtils.java

@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.utils;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* This is Regex expression utils.
*/
public class RegexUtils {
/**
* check number regex expression
*/
private static final String CHECK_NUMBER = "^-?\\d+(\\.\\d+)?$";
private RegexUtils() {
}
/**
* check if the input is number
*
* @param str input
* @return
*/
public static boolean isNumeric(String str) {
Pattern pattern = Pattern.compile(CHECK_NUMBER);
Matcher isNum = pattern.matcher(str);
return isNum.matches();
}
}

2
dolphinscheduler-api/src/main/resources/i18n/messages.properties

@ -117,7 +117,6 @@ VERIFY_QUEUE_NOTES=verify queue
TENANT_TAG=tenant related operation
CREATE_TENANT_NOTES=create tenant
TENANT_CODE=tenant code
TENANT_NAME=tenant name
QUEUE_NAME=queue name
PASSWORD=password
DATA_SOURCE_OTHER=jdbc connection params, format:{"key1":"value1",...}
@ -254,6 +253,7 @@ QUERY_ALERT_GROUP_LIST_PAGING_NOTES=query alert group list paging
EXPORT_PROCESS_DEFINITION_BY_ID_NOTES=export process definition by id
BATCH_EXPORT_PROCESS_DEFINITION_BY_IDS_NOTES= batch export process definition by ids
QUERY_USER_CREATED_PROJECT_NOTES= query user created project
QUERY_AUTHORIZED_AND_USER_CREATED_PROJECT_NOTES= query authorized and user created project
COPY_PROCESS_DEFINITION_NOTES= copy process definition notes
MOVE_PROCESS_DEFINITION_NOTES= move process definition notes
TARGET_PROJECT_ID= target project id

2
dolphinscheduler-api/src/main/resources/i18n/messages_en_US.properties

@ -117,7 +117,6 @@ VERIFY_QUEUE_NOTES=verify queue
TENANT_TAG=tenant related operation
CREATE_TENANT_NOTES=create tenant
TENANT_CODE=tenant code
TENANT_NAME=tenant name
QUEUE_NAME=queue name
PASSWORD=password
DATA_SOURCE_OTHER=jdbc connection params, format:{"key1":"value1",...}
@ -254,6 +253,7 @@ QUERY_ALERT_GROUP_LIST_PAGING_NOTES=query alert group list paging
EXPORT_PROCESS_DEFINITION_BY_ID_NOTES=export process definition by id
BATCH_EXPORT_PROCESS_DEFINITION_BY_IDS_NOTES= batch export process definition by ids
QUERY_USER_CREATED_PROJECT_NOTES= query user created project
QUERY_AUTHORIZED_AND_USER_CREATED_PROJECT_NOTES= query authorized and user created project
COPY_PROCESS_DEFINITION_NOTES= copy process definition notes
MOVE_PROCESS_DEFINITION_NOTES= move process definition notes
TARGET_PROJECT_ID= target project id

2
dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties

@ -116,7 +116,6 @@ VERIFY_QUEUE_NOTES=验证队列
TENANT_TAG=租户相关操作
CREATE_TENANT_NOTES=创建租户
TENANT_CODE=租户编码
TENANT_NAME=租户名称
QUEUE_NAME=队列名
PASSWORD=密码
DATA_SOURCE_OTHER=jdbc连接参数,格式为:{"key1":"value1",...}
@ -252,6 +251,7 @@ QUERY_ALERT_GROUP_LIST_PAGING_NOTES=分页查询告警组列表
EXPORT_PROCESS_DEFINITION_BY_ID_NOTES=通过工作流ID导出工作流定义
BATCH_EXPORT_PROCESS_DEFINITION_BY_IDS_NOTES=批量导出工作流定义
QUERY_USER_CREATED_PROJECT_NOTES= 查询用户创建的项目
QUERY_AUTHORIZED_AND_USER_CREATED_PROJECT_NOTES= 查询授权和用户创建的项目
COPY_PROCESS_DEFINITION_NOTES= 复制工作流定义
MOVE_PROCESS_DEFINITION_NOTES= 移动工作流定义
TARGET_PROJECT_ID= 目标项目ID

1
dolphinscheduler-api/src/main/resources/logback-api.xml

@ -55,7 +55,6 @@
<root level="INFO">
<appender-ref ref="STDOUT"/>
<appender-ref ref="APILOGFILE"/>
</root>

4
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java

@ -117,14 +117,14 @@ public class ProcessDefinitionControllerTest {
public void testVerifyProcessDefinitionName() throws Exception {
Map<String, Object> result = new HashMap<>();
putMsg(result, Status.PROCESS_INSTANCE_EXIST);
putMsg(result, Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR);
String projectName = "test";
String name = "dag_test";
Mockito.when(processDefinitionService.verifyProcessDefinitionName(user, projectName, name)).thenReturn(result);
Result response = processDefinitionController.verifyProcessDefinitionName(user, projectName, name);
Assert.assertEquals(Status.PROCESS_INSTANCE_EXIST.getCode(), response.getCode().intValue());
Assert.assertEquals(Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR.getCode(), response.getCode().intValue());
}

72
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java

@ -14,52 +14,60 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.controller;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.TaskInstanceService;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.utils.*;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import java.util.HashMap;
import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.MediaType;
import org.springframework.test.web.servlet.MvcResult;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
/**
* task instance controller test
*/
public class TaskInstanceControllerTest extends AbstractControllerTest{
private static Logger logger = LoggerFactory.getLogger(TaskInstanceControllerTest.class);
@RunWith(MockitoJUnitRunner.Silent.class)
public class TaskInstanceControllerTest {
@InjectMocks
private TaskInstanceController taskInstanceController;
@Mock
private TaskInstanceService taskInstanceService;
@Test
public void testQueryTaskListPaging() throws Exception {
public void testQueryTaskListPaging() {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
//paramsMap.add("processInstanceId","1380");
paramsMap.add("searchVal","");
paramsMap.add("taskName","");
//paramsMap.add("stateType","");
paramsMap.add("startDate","2019-02-26 19:48:00");
paramsMap.add("endDate","2019-02-26 19:48:22");
paramsMap.add("pageNo","1");
paramsMap.add("pageSize","20");
Map<String,Object> result = new HashMap<>();
Integer pageNo = 1;
Integer pageSize = 20;
PageInfo pageInfo = new PageInfo<TaskInstance>(pageNo, pageSize);
result.put(Constants.DATA_LIST, pageInfo);
result.put(Constants.STATUS, Status.SUCCESS);
MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/task-instance/list-paging","cxc_1113")
.header(SESSION_ID, sessionId)
.params(paramsMap))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andReturn();
when(taskInstanceService.queryTaskListPaging(any(), eq(""), eq(1), eq(""), eq(""), eq(""),any(), any(),
eq(""), Mockito.any(), eq("192.168.xx.xx"), any(), any())).thenReturn(result);
Result taskResult = taskInstanceController.queryTaskListPaging(null, "", 1, "", "",
"", "", ExecutionStatus.SUCCESS,"192.168.xx.xx", "2020-01-01 00:00:00", "2020-01-02 00:00:00",pageNo, pageSize);
Assert.assertEquals(Integer.valueOf(Status.SUCCESS.getCode()), taskResult.getCode());
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
}

8
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java

@ -43,7 +43,6 @@ public class TenantControllerTest extends AbstractControllerTest{
public void testCreateTenant() throws Exception {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("tenantCode","tenantCode");
paramsMap.add("tenantName","tenantName");
paramsMap.add("queueId","1");
paramsMap.add("description","tenant description");
@ -84,7 +83,6 @@ public class TenantControllerTest extends AbstractControllerTest{
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("id","9");
paramsMap.add("tenantCode","cxc_te");
paramsMap.add("tenantName","tenant_update_2");
paramsMap.add("queueId","1");
paramsMap.add("description","tenant description");
@ -96,7 +94,7 @@ public class TenantControllerTest extends AbstractControllerTest{
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
Assert.assertEquals(Status.TENANT_NOT_EXIST.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@ -133,7 +131,7 @@ public class TenantControllerTest extends AbstractControllerTest{
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assert.assertEquals(Status.TENANT_NAME_EXIST.getCode(), result.getCode().intValue());
Assert.assertEquals(Status.TENANT_CODE_EXIST.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@ -165,7 +163,7 @@ public class TenantControllerTest extends AbstractControllerTest{
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
Assert.assertEquals(Status.TENANT_NOT_EXIST.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
}

16
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java

@ -81,7 +81,7 @@ public class AccessTokenServiceTest {
public void testCreateToken() {
when(accessTokenMapper.insert(any(AccessToken.class))).thenReturn(2);
Map<String, Object> result = accessTokenService.createToken(1, getDate(), "AccessTokenServiceTest");
Map<String, Object> result = accessTokenService.createToken(getLoginUser(), 1, getDate(), "AccessTokenServiceTest");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
}
@ -89,7 +89,7 @@ public class AccessTokenServiceTest {
@Test
public void testGenerateToken() {
Map<String, Object> result = accessTokenService.generateToken(Integer.MAX_VALUE, getDate());
Map<String, Object> result = accessTokenService.generateToken(getLoginUser(), Integer.MAX_VALUE,getDate());
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
String token = (String) result.get(Constants.DATA_LIST);
@ -121,16 +121,24 @@ public class AccessTokenServiceTest {
public void testUpdateToken() {
when(accessTokenMapper.selectById(1)).thenReturn(getEntity());
Map<String, Object> result = accessTokenService.updateToken(1, Integer.MAX_VALUE, getDate(), "token");
Map<String, Object> result = accessTokenService.updateToken(getLoginUser(), 1,Integer.MAX_VALUE,getDate(),"token");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
// not exist
result = accessTokenService.updateToken(2, Integer.MAX_VALUE, getDate(), "token");
result = accessTokenService.updateToken(getLoginUser(), 2,Integer.MAX_VALUE,getDate(),"token");
logger.info(result.toString());
Assert.assertEquals(Status.ACCESS_TOKEN_NOT_EXIST, result.get(Constants.STATUS));
}
private User getLoginUser(){
User loginUser = new User();
loginUser.setId(1);
loginUser.setUserType(UserType.ADMIN_USER);
return loginUser;
}
/**
* create entity
*/

137
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java

@ -16,12 +16,19 @@
*/
package org.apache.dolphinscheduler.api.service;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import org.apache.dolphinscheduler.api.dto.CommandStateCount;
import org.apache.dolphinscheduler.api.dto.TaskStateCount;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.impl.DataAnalysisServiceImpl;
import org.apache.dolphinscheduler.api.service.impl.ProjectServiceImpl;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.dao.entity.CommandCount;
import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount;
@ -36,6 +43,7 @@ import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import org.apache.dolphinscheduler.service.process.ProcessService;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -123,6 +131,74 @@ public class DataAnalysisServiceTest {
result = dataAnalysisService.countTaskStateByProject(user, 1, startDate, endDate);
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
// when date in illegal format then return error message
String startDate2 = "illegalDateString";
String endDate2 = "illegalDateString";
result = dataAnalysisService.countTaskStateByProject(user, 0, startDate2, endDate2);
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS));
// when one of date in illegal format then return error message
String startDate3 = "2020-08-28 14:13:40";
String endDate3 = "illegalDateString";
result = dataAnalysisService.countTaskStateByProject(user, 0, startDate3, endDate3);
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS));
// when one of date in illegal format then return error message
String startDate4 = "illegalDateString";
String endDate4 = "2020-08-28 14:13:40";
result = dataAnalysisService.countTaskStateByProject(user, 0, startDate4, endDate4);
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS));
// when counting general user's task status then return user's task status count
user.setUserType(UserType.GENERAL_USER);
Mockito.when(processService.getProjectIdListHavePerm(anyInt()))
.thenReturn(Collections.singletonList(123));
ExecuteStatusCount executeStatusCount = new ExecuteStatusCount();
executeStatusCount.setExecutionStatus(ExecutionStatus.RUNNING_EXECUTION);
executeStatusCount.setCount(10);
Mockito.when(taskInstanceMapper.countTaskInstanceStateByUser(any(), any(), any()))
.thenReturn(Collections.singletonList(executeStatusCount));
result = dataAnalysisService.countTaskStateByProject(user, 0, startDate, null);
assertThat(result.get(Constants.DATA_LIST)).extracting("taskCountDtos").first().asList()
.hasSameSizeAs(ExecutionStatus.values());
assertThat(result.get(Constants.DATA_LIST)).extracting("totalCount").first().isEqualTo(10);
TaskStateCount taskStateCount = new TaskStateCount(ExecutionStatus.RUNNING_EXECUTION, 10);
assertThat(result.get(Constants.DATA_LIST)).extracting("taskCountDtos").first().asList().containsOnlyOnce(taskStateCount);
// when general user doesn't have any task then return all count are 0
user.setUserType(UserType.GENERAL_USER);
Mockito.when(processService.getProjectIdListHavePerm(anyInt()))
.thenReturn(new ArrayList<>());
Mockito.when(taskInstanceMapper.countTaskInstanceStateByUser(any(), any(), any()))
.thenReturn(Collections.emptyList());
result = dataAnalysisService.countTaskStateByProject(user, 0, null, null);
assertThat(result.get(Constants.DATA_LIST)).extracting("totalCount").first().isEqualTo(0);
assertThat(result.get(Constants.DATA_LIST)).extracting("taskCountDtos").first().asList()
.hasSameSizeAs(ExecutionStatus.values());
assertThat(result.get(Constants.DATA_LIST)).extracting("taskCountDtos").first().asList()
.extracting("count").allMatch(count -> count.equals(0));
// when general user doesn't have any task then return all count are 0
user.setUserType(UserType.GENERAL_USER);
Mockito.when(processService.getProjectIdListHavePerm(anyInt()))
.thenReturn(new ArrayList<>());
Mockito.when(taskInstanceMapper.countTaskInstanceStateByUser(any(), any(), any()))
.thenReturn(Collections.emptyList());
result = dataAnalysisService.countTaskStateByProject(user, 0, null, null);
assertThat(result.get(Constants.DATA_LIST)).extracting("totalCount").first().isEqualTo(0);
assertThat(result.get(Constants.DATA_LIST)).extracting("taskCountDtos").first().asList()
.hasSameSizeAs(ExecutionStatus.values());
assertThat(result.get(Constants.DATA_LIST)).extracting("taskCountDtos").first().asList()
.extracting("count").allMatch(count -> count.equals(0));
// when instanceStateCounter return null, then return nothing
user.setUserType(UserType.GENERAL_USER);
Mockito.when(processService.getProjectIdListHavePerm(anyInt()))
.thenReturn(new ArrayList<>());
Mockito.when(taskInstanceMapper.countTaskInstanceStateByUser(any(), any(), any()))
.thenReturn(null);
result = dataAnalysisService.countTaskStateByProject(user, 0, null, null);
assertThat(result).isEmpty();
}
@Test
@ -169,6 +245,67 @@ public class DataAnalysisServiceTest {
result = dataAnalysisService.countCommandState(user, 1, startDate, endDate);
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
// when project check fail then return nothing
Map<String, Object> result1 = dataAnalysisService.countCommandState(user, 2, null, null);
Assert.assertTrue(result1.isEmpty());
// when all date in illegal format then return error message
String startDate2 = "illegalDateString";
String endDate2 = "illegalDateString";
Map<String, Object> result2 = dataAnalysisService.countCommandState(user, 0, startDate2, endDate2);
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result2.get(Constants.STATUS));
// when one of date in illegal format then return error message
String startDate3 = "2020-08-22 09:23:10";
String endDate3 = "illegalDateString";
Map<String, Object> result3 = dataAnalysisService.countCommandState(user, 0, startDate3, endDate3);
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result3.get(Constants.STATUS));
// when one of date in illegal format then return error message
String startDate4 = "illegalDateString";
String endDate4 = "2020-08-22 09:23:10";
Map<String, Object> result4 = dataAnalysisService.countCommandState(user, 0, startDate4, endDate4);
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result4.get(Constants.STATUS));
// when no command found then return all count are 0
Mockito.when(commandMapper.countCommandState(anyInt(), any(), any(), any())).thenReturn(Collections.emptyList());
Mockito.when(errorCommandMapper.countCommandState(any(), any(), any())).thenReturn(Collections.emptyList());
Map<String, Object> result5 = dataAnalysisService.countCommandState(user, 0, startDate, null);
assertThat(result5).containsEntry(Constants.STATUS, Status.SUCCESS);
assertThat(result5.get(Constants.DATA_LIST)).asList().extracting("errorCount").allMatch(count -> count.equals(0));
assertThat(result5.get(Constants.DATA_LIST)).asList().extracting("normalCount").allMatch(count -> count.equals(0));
// when command found then return combination result
CommandCount normalCommandCount = new CommandCount();
normalCommandCount.setCommandType(CommandType.START_PROCESS);
normalCommandCount.setCount(10);
CommandCount errorCommandCount = new CommandCount();
errorCommandCount.setCommandType(CommandType.START_PROCESS);
errorCommandCount.setCount(5);
Mockito.when(commandMapper.countCommandState(anyInt(), any(), any(), any())).thenReturn(Collections.singletonList(normalCommandCount));
Mockito.when(errorCommandMapper.countCommandState(any(), any(), any())).thenReturn(Collections.singletonList(errorCommandCount));
Map<String, Object> result6 = dataAnalysisService.countCommandState(user, 0, null, null);
assertThat(result6).containsEntry(Constants.STATUS, Status.SUCCESS);
CommandStateCount commandStateCount = new CommandStateCount();
commandStateCount.setCommandState(CommandType.START_PROCESS);
commandStateCount.setNormalCount(10);
commandStateCount.setErrorCount(5);
assertThat(result6.get(Constants.DATA_LIST)).asList().containsOnlyOnce(commandStateCount);
}
@Test
public void testCountQueueState() {
// when project check fail then return nothing
Map<String, Object> result1 = dataAnalysisService.countQueueState(user, 2);
Assert.assertTrue(result1.isEmpty());
// when project check success when return all count are 0
Map<String, Object> result2 = dataAnalysisService.countQueueState(user, 1);
assertThat(result2.get(Constants.DATA_LIST)).extracting("taskQueue", "taskKill")
.isNotEmpty()
.allMatch(count -> count.equals(0));
}
/**

90
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java

@ -17,6 +17,8 @@
package org.apache.dolphinscheduler.api.service;
import static org.assertj.core.api.Assertions.assertThat;
import org.apache.dolphinscheduler.api.dto.ProcessMeta;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.impl.ProcessDefinitionServiceImpl;
@ -28,6 +30,9 @@ import org.apache.dolphinscheduler.common.enums.Priority;
import org.apache.dolphinscheduler.common.enums.ReleaseState;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.enums.WarningType;
import org.apache.dolphinscheduler.common.model.TaskNode;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.shell.ShellParameters;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
@ -51,8 +56,11 @@ import org.apache.http.entity.ContentType;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.lang.reflect.Method;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@ -66,6 +74,7 @@ import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.util.ReflectionUtils;
import org.springframework.web.multipart.MultipartFile;
import com.baomidou.mybatisplus.core.metadata.IPage;
@ -609,16 +618,16 @@ public class ProcessDefinitionServiceTest {
//project check auth success, process not exist
putMsg(result, Status.SUCCESS, projectName);
Mockito.when(processDefineMapper.queryByDefineName(project.getId(), "test_pdf")).thenReturn(null);
Mockito.when(processDefineMapper.verifyByDefineName(project.getId(), "test_pdf")).thenReturn(null);
Map<String, Object> processNotExistRes = processDefinitionService.verifyProcessDefinitionName(loginUser,
"project_test1", "test_pdf");
Assert.assertEquals(Status.SUCCESS, processNotExistRes.get(Constants.STATUS));
//process exist
Mockito.when(processDefineMapper.queryByDefineName(project.getId(), "test_pdf")).thenReturn(getProcessDefinition());
Mockito.when(processDefineMapper.verifyByDefineName(project.getId(), "test_pdf")).thenReturn(getProcessDefinition());
Map<String, Object> processExistRes = processDefinitionService.verifyProcessDefinitionName(loginUser,
"project_test1", "test_pdf");
Assert.assertEquals(Status.PROCESS_INSTANCE_EXIST, processExistRes.get(Constants.STATUS));
Assert.assertEquals(Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR, processExistRes.get(Constants.STATUS));
}
@Test
@ -984,6 +993,81 @@ public class ProcessDefinitionServiceTest {
loginUser, projectName, "1", null);
}
@Test
public void testGetResourceIds() throws Exception {
// set up
Method testMethod = ReflectionUtils.findMethod(ProcessDefinitionServiceImpl.class, "getResourceIds", ProcessData.class);
assertThat(testMethod).isNotNull();
testMethod.setAccessible(true);
// when processData has empty task, then return empty string
ProcessData input1 = new ProcessData();
input1.setTasks(Collections.emptyList());
String output1 = (String) testMethod.invoke(processDefinitionService, input1);
assertThat(output1).isEmpty();
// when task is null, then return empty string
ProcessData input2 = new ProcessData();
input2.setTasks(null);
String output2 = (String) testMethod.invoke(processDefinitionService, input2);
assertThat(output2).isEmpty();
// when task type is incorrect mapping, then return empty string
ProcessData input3 = new ProcessData();
TaskNode taskNode3 = new TaskNode();
taskNode3.setType("notExistType");
input3.setTasks(Collections.singletonList(taskNode3));
String output3 = (String) testMethod.invoke(processDefinitionService, input3);
assertThat(output3).isEmpty();
// when task parameter list is null, then return empty string
ProcessData input4 = new ProcessData();
TaskNode taskNode4 = new TaskNode();
taskNode4.setType("SHELL");
taskNode4.setParams(null);
input4.setTasks(Collections.singletonList(taskNode4));
String output4 = (String) testMethod.invoke(processDefinitionService, input4);
assertThat(output4).isEmpty();
// when resource id list is 0 1, then return 0,1
ProcessData input5 = new ProcessData();
TaskNode taskNode5 = new TaskNode();
taskNode5.setType("SHELL");
ShellParameters shellParameters5 = new ShellParameters();
ResourceInfo resourceInfo5A = new ResourceInfo();
resourceInfo5A.setId(1);
ResourceInfo resourceInfo5B = new ResourceInfo();
resourceInfo5B.setId(2);
shellParameters5.setResourceList(Arrays.asList(resourceInfo5A, resourceInfo5B));
taskNode5.setParams(JSONUtils.toJsonString(shellParameters5));
input5.setTasks(Collections.singletonList(taskNode5));
String output5 = (String) testMethod.invoke(processDefinitionService, input5);
assertThat(output5.split(",")).hasSize(2)
.containsExactlyInAnyOrder("1", "2");
// when resource id list is 0 1 1 2, then return 0,1,2
ProcessData input6 = new ProcessData();
TaskNode taskNode6 = new TaskNode();
taskNode6.setType("SHELL");
ShellParameters shellParameters6 = new ShellParameters();
ResourceInfo resourceInfo6A = new ResourceInfo();
resourceInfo6A.setId(3);
ResourceInfo resourceInfo6B = new ResourceInfo();
resourceInfo6B.setId(1);
ResourceInfo resourceInfo6C = new ResourceInfo();
resourceInfo6C.setId(1);
ResourceInfo resourceInfo6D = new ResourceInfo();
resourceInfo6D.setId(2);
shellParameters6.setResourceList(Arrays.asList(resourceInfo6A, resourceInfo6B, resourceInfo6C, resourceInfo6D));
taskNode6.setParams(JSONUtils.toJsonString(shellParameters6));
input6.setTasks(Collections.singletonList(taskNode6));
String output6 = (String) testMethod.invoke(processDefinitionService, input6);
assertThat(output6.split(",")).hasSize(3)
.containsExactlyInAnyOrder("3", "1", "2");
}
/**
* get mock datasource
*

24
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java

@ -295,6 +295,28 @@ public class ProjectServiceTest {
}
@Test
public void testQueryProjectCreatedAndAuthorizedByUser() {
Map<String, Object> result = null;
User loginUser = getLoginUser();
// not admin user
Mockito.when(projectMapper.queryProjectCreatedAndAuthorizedByUserId(1)).thenReturn(getList());
result = projectService.queryProjectCreatedAndAuthorizedByUser(loginUser);
List<Project> notAdminUserResult = (List<Project>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(notAdminUserResult));
//admin user
loginUser.setUserType(UserType.ADMIN_USER);
Mockito.when(projectMapper.selectList(null)).thenReturn(getList());
result = projectService.queryProjectCreatedAndAuthorizedByUser(loginUser);
List<Project> projects = (List<Project>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(projects));
}
@Test
public void testQueryAllProjectList() {
@ -340,13 +362,11 @@ public class ProjectServiceTest {
* create admin user
*/
private User getLoginUser() {
User loginUser = new User();
loginUser.setUserType(UserType.GENERAL_USER);
loginUser.setUserName(userName);
loginUser.setId(1);
return loginUser;
}
/**

312
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java

@ -14,12 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
@ -33,7 +31,18 @@ import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.*;
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@ -49,14 +58,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.mock.web.MockMultipartFile;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
@RunWith(PowerMockRunner.class)
@PowerMockIgnore({"sun.security.*", "javax.net.*"})
@PrepareForTest({HadoopUtils.class,PropertyUtils.class, FileUtils.class,org.apache.dolphinscheduler.api.utils.FileUtils.class})
@PrepareForTest({HadoopUtils.class, PropertyUtils.class, FileUtils.class, org.apache.dolphinscheduler.api.utils.FileUtils.class})
public class ResourcesServiceTest {
private static final Logger logger = LoggerFactory.getLogger(ResourcesServiceTest.class);
@ -94,100 +101,103 @@ public class ResourcesServiceTest {
}
@Test
public void testCreateResource(){
public void testCreateResource() {
PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
User user = new User();
//HDFS_NOT_STARTUP
Result result = resourcesService.createResource(user,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE,null,-1,"/");
Result result = resourcesService.createResource(user, "ResourcesServiceTest", "ResourcesServiceTest", ResourceType.FILE, null, -1, "/");
logger.info(result.toString());
Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg());
Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg());
//RESOURCE_FILE_IS_EMPTY
MockMultipartFile mockMultipartFile = new MockMultipartFile("test.pdf",new String().getBytes());
MockMultipartFile mockMultipartFile = new MockMultipartFile("test.pdf", "".getBytes());
PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
result = resourcesService.createResource(user,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE,mockMultipartFile,-1,"/");
result = resourcesService.createResource(user, "ResourcesServiceTest", "ResourcesServiceTest", ResourceType.FILE, mockMultipartFile, -1, "/");
logger.info(result.toString());
Assert.assertEquals(Status.RESOURCE_FILE_IS_EMPTY.getMsg(),result.getMsg());
Assert.assertEquals(Status.RESOURCE_FILE_IS_EMPTY.getMsg(), result.getMsg());
//RESOURCE_SUFFIX_FORBID_CHANGE
mockMultipartFile = new MockMultipartFile("test.pdf","test.pdf","pdf",new String("test").getBytes());
mockMultipartFile = new MockMultipartFile("test.pdf", "test.pdf", "pdf", "test".getBytes());
PowerMockito.when(FileUtils.suffix("test.pdf")).thenReturn("pdf");
PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.jar")).thenReturn("jar");
result = resourcesService.createResource(user,"ResourcesServiceTest.jar","ResourcesServiceTest",ResourceType.FILE,mockMultipartFile,-1,"/");
result = resourcesService.createResource(user, "ResourcesServiceTest.jar", "ResourcesServiceTest", ResourceType.FILE, mockMultipartFile, -1, "/");
logger.info(result.toString());
Assert.assertEquals(Status.RESOURCE_SUFFIX_FORBID_CHANGE.getMsg(),result.getMsg());
Assert.assertEquals(Status.RESOURCE_SUFFIX_FORBID_CHANGE.getMsg(), result.getMsg());
//UDF_RESOURCE_SUFFIX_NOT_JAR
mockMultipartFile = new MockMultipartFile("ResourcesServiceTest.pdf","ResourcesServiceTest.pdf","pdf",new String("test").getBytes());
mockMultipartFile = new MockMultipartFile("ResourcesServiceTest.pdf", "ResourcesServiceTest.pdf", "pdf", "test".getBytes());
PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.pdf")).thenReturn("pdf");
result = resourcesService.createResource(user,"ResourcesServiceTest.pdf","ResourcesServiceTest",ResourceType.UDF,mockMultipartFile,-1,"/");
result = resourcesService.createResource(user, "ResourcesServiceTest.pdf", "ResourcesServiceTest", ResourceType.UDF, mockMultipartFile, -1, "/");
logger.info(result.toString());
Assert.assertEquals(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg(),result.getMsg());
Assert.assertEquals(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg(), result.getMsg());
}
@Test
public void testCreateDirecotry(){
public void testCreateDirecotry() {
PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
User user = new User();
//HDFS_NOT_STARTUP
Result result = resourcesService.createDirectory(user,"directoryTest","directory test",ResourceType.FILE,-1,"/");
Result result = resourcesService.createDirectory(user, "directoryTest", "directory test", ResourceType.FILE, -1, "/");
logger.info(result.toString());
Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg());
Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg());
//PARENT_RESOURCE_NOT_EXIST
user.setId(1);
user.setTenantId(1);
Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
Mockito.when(resourcesMapper.selectById(Mockito.anyInt())).thenReturn(null);
result = resourcesService.createDirectory(user,"directoryTest","directory test",ResourceType.FILE,1,"/");
result = resourcesService.createDirectory(user, "directoryTest", "directory test", ResourceType.FILE, 1, "/");
logger.info(result.toString());
Assert.assertEquals(Status.PARENT_RESOURCE_NOT_EXIST.getMsg(),result.getMsg());
Assert.assertEquals(Status.PARENT_RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
//RESOURCE_EXIST
PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
Mockito.when(resourcesMapper.queryResourceList("/directoryTest", 0, 0)).thenReturn(getResourceList());
result = resourcesService.createDirectory(user,"directoryTest","directory test",ResourceType.FILE,-1,"/");
result = resourcesService.createDirectory(user, "directoryTest", "directory test", ResourceType.FILE, -1, "/");
logger.info(result.toString());
Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(),result.getMsg());
Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg());
}
@Test
public void testUpdateResource(){
public void testUpdateResource() {
PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
User user = new User();
//HDFS_NOT_STARTUP
Result result = resourcesService.updateResource(user,1,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE);
Result result = resourcesService.updateResource(user, 1, "ResourcesServiceTest", "ResourcesServiceTest", ResourceType.FILE, null);
logger.info(result.toString());
Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg());
Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg());
//RESOURCE_NOT_EXIST
Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource());
PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
result = resourcesService.updateResource(user,0,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE);
result = resourcesService.updateResource(user, 0, "ResourcesServiceTest", "ResourcesServiceTest", ResourceType.FILE, null);
logger.info(result.toString());
Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg());
Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
//USER_NO_OPERATION_PERM
result = resourcesService.updateResource(user,1,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE);
result = resourcesService.updateResource(user, 1, "ResourcesServiceTest", "ResourcesServiceTest", ResourceType.FILE, null);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM.getMsg(),result.getMsg());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM.getMsg(), result.getMsg());
//RESOURCE_NOT_EXIST
user.setId(1);
Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
PowerMockito.when(HadoopUtils.getHdfsFileName(Mockito.any(), Mockito.any(),Mockito.anyString())).thenReturn("test1");
PowerMockito.when(HadoopUtils.getHdfsFileName(Mockito.any(), Mockito.any(), Mockito.anyString())).thenReturn("test1");
try {
Mockito.when(HadoopUtils.getInstance().exists(Mockito.any())).thenReturn(false);
} catch (IOException e) {
logger.error(e.getMessage(),e);
logger.error(e.getMessage(), e);
}
result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest", ResourceType.UDF);
Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg());
result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest", ResourceType.UDF, null);
Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
//SUCCESS
user.setId(1);
@ -196,57 +206,56 @@ public class ResourcesServiceTest {
try {
Mockito.when(HadoopUtils.getInstance().exists(Mockito.any())).thenReturn(true);
} catch (IOException e) {
logger.error(e.getMessage(),e);
logger.error(e.getMessage(), e);
}
result = resourcesService.updateResource(user,1,"ResourcesServiceTest.jar","ResourcesServiceTest",ResourceType.FILE);
result = resourcesService.updateResource(user, 1, "ResourcesServiceTest.jar", "ResourcesServiceTest", ResourceType.FILE, null);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg());
Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
//RESOURCE_EXIST
Mockito.when(resourcesMapper.queryResourceList("/ResourcesServiceTest1.jar", 0, 0)).thenReturn(getResourceList());
result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest",ResourceType.FILE);
result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest", ResourceType.FILE, null);
logger.info(result.toString());
Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(),result.getMsg());
Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg());
//USER_NOT_EXIST
Mockito.when(userMapper.selectById(Mockito.anyInt())).thenReturn(null);
result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest",ResourceType.UDF);
result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest", ResourceType.UDF, null);
logger.info(result.toString());
Assert.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode());
//TENANT_NOT_EXIST
Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
Mockito.when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(null);
result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest",ResourceType.UDF);
result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest", ResourceType.UDF, null);
logger.info(result.toString());
Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(),result.getMsg());
Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(), result.getMsg());
//SUCCESS
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
PowerMockito.when(HadoopUtils.getHdfsResourceFileName(Mockito.any(), Mockito.any())).thenReturn("test");
try {
PowerMockito.when(HadoopUtils.getInstance().copy(Mockito.anyString(),Mockito.anyString(),true,true)).thenReturn(true);
PowerMockito.when(HadoopUtils.getInstance().copy(Mockito.anyString(), Mockito.anyString(), true, true)).thenReturn(true);
} catch (Exception e) {
logger.error(e.getMessage(),e);
logger.error(e.getMessage(), e);
}
result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest1.jar",ResourceType.UDF);
result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest1.jar", ResourceType.UDF, null);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg());
Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
}
@Test
public void testQueryResourceListPaging(){
public void testQueryResourceListPaging() {
User loginUser = new User();
loginUser.setUserType(UserType.ADMIN_USER);
IPage<Resource> resourcePage = new Page<>(1,10);
IPage<Resource> resourcePage = new Page<>(1, 10);
resourcePage.setTotal(1);
resourcePage.setRecords(getResourceList());
Mockito.when(resourcesMapper.queryResourcePaging(Mockito.any(Page.class),
Mockito.eq(0),Mockito.eq(-1), Mockito.eq(0), Mockito.eq("test"))).thenReturn(resourcePage);
Map<String, Object> result = resourcesService.queryResourceListPaging(loginUser,-1,ResourceType.FILE,"test",1,10);
Mockito.eq(0), Mockito.eq(-1), Mockito.eq(0), Mockito.eq("test"))).thenReturn(resourcePage);
Map<String, Object> result = resourcesService.queryResourceListPaging(loginUser, -1, ResourceType.FILE, "test", 1, 10);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST);
@ -255,11 +264,11 @@ public class ResourcesServiceTest {
}
@Test
public void testQueryResourceList(){
public void testQueryResourceList() {
User loginUser = new User();
loginUser.setId(0);
loginUser.setUserType(UserType.ADMIN_USER);
Mockito.when(resourcesMapper.queryResourceListAuthored(0, 0,0)).thenReturn(getResourceList());
Mockito.when(resourcesMapper.queryResourceListAuthored(0, 0, 0)).thenReturn(getResourceList());
Map<String, Object> result = resourcesService.queryResourceList(loginUser, ResourceType.FILE);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
@ -268,7 +277,7 @@ public class ResourcesServiceTest {
}
@Test
public void testDelete(){
public void testDelete() {
User loginUser = new User();
loginUser.setId(0);
@ -277,19 +286,19 @@ public class ResourcesServiceTest {
try {
// HDFS_NOT_STARTUP
Result result = resourcesService.delete(loginUser,1);
Result result = resourcesService.delete(loginUser, 1);
logger.info(result.toString());
Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg());
//RESOURCE_NOT_EXIST
PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource());
result = resourcesService.delete(loginUser,2);
result = resourcesService.delete(loginUser, 2);
logger.info(result.toString());
Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
// USER_NO_OPERATION_PERM
result = resourcesService.delete(loginUser,2);
result = resourcesService.delete(loginUser, 2);
logger.info(result.toString());
Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
@ -297,30 +306,30 @@ public class ResourcesServiceTest {
loginUser.setUserType(UserType.ADMIN_USER);
loginUser.setTenantId(2);
Mockito.when(userMapper.selectById(Mockito.anyInt())).thenReturn(loginUser);
result = resourcesService.delete(loginUser,1);
result = resourcesService.delete(loginUser, 1);
logger.info(result.toString());
Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(), result.getMsg());
//SUCCESS
loginUser.setTenantId(1);
Mockito.when(hadoopUtils.delete(Mockito.anyString(), Mockito.anyBoolean())).thenReturn(true);
result = resourcesService.delete(loginUser,1);
result = resourcesService.delete(loginUser, 1);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
} catch (Exception e) {
logger.error("delete error",e);
logger.error("delete error", e);
Assert.assertTrue(false);
}
}
@Test
public void testVerifyResourceName(){
public void testVerifyResourceName() {
User user = new User();
user.setId(1);
Mockito.when(resourcesMapper.queryResourceList("/ResourcesServiceTest.jar", 0, 0)).thenReturn(getResourceList());
Result result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar",ResourceType.FILE,user);
Result result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar", ResourceType.FILE, user);
logger.info(result.toString());
Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg());
@ -330,94 +339,89 @@ public class ResourcesServiceTest {
try {
Mockito.when(hadoopUtils.exists(unExistFullName)).thenReturn(false);
} catch (IOException e) {
logger.error("hadoop error",e);
logger.error("hadoop error", e);
}
result = resourcesService.verifyResourceName("/test.jar",ResourceType.FILE,user);
result = resourcesService.verifyResourceName("/test.jar", ResourceType.FILE, user);
logger.info(result.toString());
Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(), result.getMsg());
//RESOURCE_FILE_EXIST
user.setTenantId(1);
try {
Mockito.when(hadoopUtils.exists("test")).thenReturn(true);
} catch (IOException e) {
logger.error("hadoop error",e);
logger.error("hadoop error", e);
}
PowerMockito.when(HadoopUtils.getHdfsResourceFileName("123", "test1")).thenReturn("test");
result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar",ResourceType.FILE,user);
result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar", ResourceType.FILE, user);
logger.info(result.toString());
Assert.assertTrue(Status.RESOURCE_EXIST.getCode()==result.getCode());
Assert.assertTrue(Status.RESOURCE_EXIST.getCode() == result.getCode());
//SUCCESS
result = resourcesService.verifyResourceName("test2",ResourceType.FILE,user);
result = resourcesService.verifyResourceName("test2", ResourceType.FILE, user);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
}
@Test
public void testReadResource(){
public void testReadResource() {
PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
//HDFS_NOT_STARTUP
Result result = resourcesService.readResource(1,1,10);
Result result = resourcesService.readResource(1, 1, 10);
logger.info(result.toString());
Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg());
Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg());
//RESOURCE_NOT_EXIST
Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource());
PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
result = resourcesService.readResource(2,1,10);
result = resourcesService.readResource(2, 1, 10);
logger.info(result.toString());
Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg());
Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
//RESOURCE_SUFFIX_NOT_SUPPORT_VIEW
PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("class");
PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
result = resourcesService.readResource(1,1,10);
result = resourcesService.readResource(1, 1, 10);
logger.info(result.toString());
Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(),result.getMsg());
Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg());
//USER_NOT_EXIST
PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("jar");
PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.jar")).thenReturn("jar");
result = resourcesService.readResource(1,1,10);
result = resourcesService.readResource(1, 1, 10);
logger.info(result.toString());
Assert.assertTrue(Status.USER_NOT_EXIST.getCode()==result.getCode());
Assert.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode());
//TENANT_NOT_EXIST
Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
result = resourcesService.readResource(1,1,10);
result = resourcesService.readResource(1, 1, 10);
logger.info(result.toString());
Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(),result.getMsg());
Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(), result.getMsg());
//RESOURCE_FILE_NOT_EXIST
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
try {
Mockito.when(hadoopUtils.exists(Mockito.anyString())).thenReturn(false);
} catch (IOException e) {
logger.error("hadoop error",e);
logger.error("hadoop error", e);
}
result = resourcesService.readResource(1,1,10);
result = resourcesService.readResource(1, 1, 10);
logger.info(result.toString());
Assert.assertTrue(Status.RESOURCE_FILE_NOT_EXIST.getCode()==result.getCode());
Assert.assertTrue(Status.RESOURCE_FILE_NOT_EXIST.getCode() == result.getCode());
//SUCCESS
try {
Mockito.when(hadoopUtils.exists(null)).thenReturn(true);
Mockito.when(hadoopUtils.catFile(null,1,10)).thenReturn(getContent());
Mockito.when(hadoopUtils.catFile(null, 1, 10)).thenReturn(getContent());
} catch (IOException e) {
logger.error("hadoop error",e);
logger.error("hadoop error", e);
}
result = resourcesService.readResource(1,1,10);
result = resourcesService.readResource(1, 1, 10);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg());
Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
}
@ -429,23 +433,23 @@ public class ResourcesServiceTest {
PowerMockito.when(HadoopUtils.getHdfsUdfDir("udfDir")).thenReturn("udfDir");
User user = getUser();
//HDFS_NOT_STARTUP
Result result = resourcesService.onlineCreateResource(user,ResourceType.FILE,"test","jar","desc","content",-1,"/");
Result result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content", -1, "/");
logger.info(result.toString());
Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg());
Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg());
//RESOURCE_SUFFIX_NOT_SUPPORT_VIEW
PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("class");
result = resourcesService.onlineCreateResource(user,ResourceType.FILE,"test","jar","desc","content",-1,"/");
result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content", -1, "/");
logger.info(result.toString());
Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(),result.getMsg());
Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg());
//RuntimeException
try {
PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("jar");
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content",-1,"/");
}catch (RuntimeException ex){
result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content", -1, "/");
} catch (RuntimeException ex) {
logger.info(result.toString());
Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), ex.getMessage());
}
@ -453,50 +457,48 @@ public class ResourcesServiceTest {
//SUCCESS
Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test");
PowerMockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true);
result = resourcesService.onlineCreateResource(user,ResourceType.FILE,"test","jar","desc","content",-1,"/");
result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content", -1, "/");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg());
Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
}
@Test
public void testUpdateResourceContent(){
public void testUpdateResourceContent() {
User loginUser = new User();
loginUser.setId(0);
PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
// HDFS_NOT_STARTUP
Result result = resourcesService.updateResourceContent(1,"content");
Result result = resourcesService.updateResourceContent(1, "content");
logger.info(result.toString());
Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg());
//RESOURCE_NOT_EXIST
PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource());
result = resourcesService.updateResourceContent(2,"content");
result = resourcesService.updateResourceContent(2, "content");
logger.info(result.toString());
Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
//RESOURCE_SUFFIX_NOT_SUPPORT_VIEW
PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("class");
result = resourcesService.updateResourceContent(1,"content");
result = resourcesService.updateResourceContent(1, "content");
logger.info(result.toString());
Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(),result.getMsg());
Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg());
//USER_NOT_EXIST
PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("jar");
PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.jar")).thenReturn("jar");
result = resourcesService.updateResourceContent(1,"content");
result = resourcesService.updateResourceContent(1, "content");
logger.info(result.toString());
Assert.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode());
//TENANT_NOT_EXIST
Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
result = resourcesService.updateResourceContent(1,"content");
result = resourcesService.updateResourceContent(1, "content");
logger.info(result.toString());
Assert.assertTrue(Status.TENANT_NOT_EXIST.getCode() == result.getCode());
@ -504,13 +506,13 @@ public class ResourcesServiceTest {
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test");
PowerMockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true);
result = resourcesService.updateResourceContent(1,"content");
result = resourcesService.updateResourceContent(1, "content");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
}
@Test
public void testDownloadResource(){
public void testDownloadResource() {
PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
@ -526,102 +528,117 @@ public class ResourcesServiceTest {
resource = resourcesService.downloadResource(1);
Assert.assertNotNull(resource);
} catch (Exception e) {
logger.error("DownloadResource error",e);
logger.error("DownloadResource error", e);
Assert.assertTrue(false);
}
}
@Test
public void testUnauthorizedFile(){
public void testUnauthorizedFile() {
User user = getUser();
//USER_NO_OPERATION_PERM
Map<String, Object> result = resourcesService.unauthorizedFile(user,1);
Map<String, Object> result = resourcesService.unauthorizedFile(user, 1);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS));
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//SUCCESS
user.setUserType(UserType.ADMIN_USER);
Mockito.when(resourcesMapper.queryResourceExceptUserId(1)).thenReturn(getResourceList());
result = resourcesService.unauthorizedFile(user,1);
result = resourcesService.unauthorizedFile(user, 1);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
List<Resource> resources = (List<Resource>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(resources));
}
@Test
public void testUnauthorizedUDFFunction(){
public void testUnauthorizedUDFFunction() {
User user = getUser();
//USER_NO_OPERATION_PERM
Map<String, Object> result = resourcesService.unauthorizedUDFFunction(user,1);
Map<String, Object> result = resourcesService.unauthorizedUDFFunction(user, 1);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS));
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//SUCCESS
user.setUserType(UserType.ADMIN_USER);
Mockito.when(udfFunctionMapper.queryUdfFuncExceptUserId(1)).thenReturn(getUdfFuncList());
result = resourcesService.unauthorizedUDFFunction(user,1);
result = resourcesService.unauthorizedUDFFunction(user, 1);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
List<UdfFunc> udfFuncs = (List<UdfFunc>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(udfFuncs));
}
@Test
public void testAuthorizedUDFFunction(){
public void testAuthorizedUDFFunction() {
User user = getUser();
//USER_NO_OPERATION_PERM
Map<String, Object> result = resourcesService.authorizedUDFFunction(user,1);
Map<String, Object> result = resourcesService.authorizedUDFFunction(user, 1);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS));
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//SUCCESS
user.setUserType(UserType.ADMIN_USER);
Mockito.when(udfFunctionMapper.queryAuthedUdfFunc(1)).thenReturn(getUdfFuncList());
result = resourcesService.authorizedUDFFunction(user,1);
result = resourcesService.authorizedUDFFunction(user, 1);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
List<UdfFunc> udfFuncs = (List<UdfFunc>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(udfFuncs));
}
@Test
public void testAuthorizedFile(){
public void testAuthorizedFile() {
User user = getUser();
//USER_NO_OPERATION_PERM
Map<String, Object> result = resourcesService.authorizedFile(user,1);
Map<String, Object> result = resourcesService.authorizedFile(user, 1);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS));
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//SUCCESS
user.setUserType(UserType.ADMIN_USER);
Mockito.when(resourcesMapper.queryAuthorizedResourceList(1)).thenReturn(getResourceList());
result = resourcesService.authorizedFile(user,1);
result = resourcesService.authorizedFile(user, 1);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
List<Resource> resources = (List<Resource>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(resources));
}
@Test
public void testCatFile() {
PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
//SUCCESS
try {
Mockito.when(hadoopUtils.exists(null)).thenReturn(true);
Mockito.when(hadoopUtils.catFile(null, 1, 10)).thenReturn(getContent());
List<String> list = hadoopUtils.catFile(null, 1, 10);
Assert.assertNotNull(list);
} catch (IOException e) {
logger.error("hadoop error", e);
}
}
private List<Resource> getResourceList(){
private List<Resource> getResourceList() {
List<Resource> resources = new ArrayList<>();
List<Resource> resources = new ArrayList<>();
resources.add(getResource());
return resources;
}
private Tenant getTenant() {
Tenant tenant = new Tenant();
tenant.setTenantCode("123");
return tenant;
}
private Resource getResource(){
private Resource getResource() {
Resource resource = new Resource();
resource.setPid(-1);
@ -633,7 +650,7 @@ public class ResourcesServiceTest {
return resource;
}
private Resource getUdfResource(){
private Resource getUdfResource() {
Resource resource = new Resource();
resource.setUserId(1);
@ -644,27 +661,28 @@ public class ResourcesServiceTest {
return resource;
}
private UdfFunc getUdfFunc(){
private UdfFunc getUdfFunc() {
UdfFunc udfFunc = new UdfFunc();
udfFunc.setId(1);
return udfFunc;
}
private List<UdfFunc> getUdfFuncList(){
private List<UdfFunc> getUdfFuncList() {
List<UdfFunc> udfFuncs = new ArrayList<>();
udfFuncs.add(getUdfFunc());
return udfFuncs;
}
private User getUser(){
private User getUser() {
User user = new User();
user.setId(1);
user.setTenantId(1);
return user;
}
private List<String> getContent(){
private List<String> getContent() {
List<String> contentList = new ArrayList<>();
contentList.add("test");
return contentList;

23
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskInstanceServiceTest.java

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import static org.mockito.ArgumentMatchers.any;
@ -88,7 +89,7 @@ public class TaskInstanceServiceTest {
//project auth fail
when(projectMapper.queryByName(projectName)).thenReturn(null);
when(projectService.checkProjectAndAuth(loginUser, null, projectName)).thenReturn(result);
Map<String, Object> proejctAuthFailRes = taskInstanceService.queryTaskListPaging(loginUser, "project_test1", 0, "",
Map<String, Object> proejctAuthFailRes = taskInstanceService.queryTaskListPaging(loginUser, "project_test1", 0, "", "",
"test_user", "2019-02-26 19:48:00", "2019-02-26 19:48:22", "", null, "", 1, 20);
Assert.assertEquals(Status.PROJECT_NOT_FOUNT, proejctAuthFailRes.get(Constants.STATUS));
@ -107,43 +108,43 @@ public class TaskInstanceServiceTest {
when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result);
when(usersService.queryUser(loginUser.getId())).thenReturn(loginUser);
when(usersService.getUserIdByName(loginUser.getUserName())).thenReturn(loginUser.getId());
when(taskInstanceMapper.queryTaskInstanceListPaging(Mockito.any(Page.class), eq(project.getId()), eq(1), eq(""), eq(""),
when(taskInstanceMapper.queryTaskInstanceListPaging(Mockito.any(Page.class), eq(project.getId()), eq(1), eq(""), eq(""), eq(""),
eq(0), Mockito.any(), eq("192.168.xx.xx"), eq(start), eq(end))).thenReturn(pageReturn);
when(usersService.queryUser(processInstance.getExecutorId())).thenReturn(loginUser);
when(processService.findProcessInstanceDetailById(taskInstance.getProcessInstanceId())).thenReturn(processInstance);
Map<String, Object> successRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "",
Map<String, Object> successRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "", "",
"test_user", "2020-01-01 00:00:00", "2020-01-02 00:00:00", "", ExecutionStatus.SUCCESS, "192.168.xx.xx", 1, 20);
Assert.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS));
//executor name empty
when(taskInstanceMapper.queryTaskInstanceListPaging(Mockito.any(Page.class), eq(project.getId()), eq(1), eq(""), eq(""),
when(taskInstanceMapper.queryTaskInstanceListPaging(Mockito.any(Page.class), eq(project.getId()), eq(1), eq(""), eq(""), eq(""),
eq(0), Mockito.any(), eq("192.168.xx.xx"), eq(start), eq(end))).thenReturn(pageReturn);
Map<String, Object> executorEmptyRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "",
Map<String, Object> executorEmptyRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "", "",
"", "2020-01-01 00:00:00", "2020-01-02 00:00:00", "", ExecutionStatus.SUCCESS, "192.168.xx.xx", 1, 20);
Assert.assertEquals(Status.SUCCESS, executorEmptyRes.get(Constants.STATUS));
//executor null
when(usersService.queryUser(loginUser.getId())).thenReturn(null);
when(usersService.getUserIdByName(loginUser.getUserName())).thenReturn(-1);
Map<String, Object> executorNullRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "",
Map<String, Object> executorNullRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "", "",
"test_user", "2020-01-01 00:00:00", "2020-01-02 00:00:00", "", ExecutionStatus.SUCCESS, "192.168.xx.xx", 1, 20);
Assert.assertEquals(Status.SUCCESS, executorNullRes.get(Constants.STATUS));
//start/end date null
when(taskInstanceMapper.queryTaskInstanceListPaging(Mockito.any(Page.class), eq(project.getId()), eq(1), eq(""), eq(""),
when(taskInstanceMapper.queryTaskInstanceListPaging(Mockito.any(Page.class), eq(project.getId()), eq(1), eq(""), eq(""), eq(""),
eq(0), Mockito.any(), eq("192.168.xx.xx"), any(), any())).thenReturn(pageReturn);
Map<String, Object> executorNullDateRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "",
Map<String, Object> executorNullDateRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "", "",
"", null, null, "", ExecutionStatus.SUCCESS, "192.168.xx.xx", 1, 20);
Assert.assertEquals(Status.SUCCESS, executorNullDateRes.get(Constants.STATUS));
//start date error format
when(taskInstanceMapper.queryTaskInstanceListPaging(Mockito.any(Page.class), eq(project.getId()), eq(1), eq(""), eq(""),
when(taskInstanceMapper.queryTaskInstanceListPaging(Mockito.any(Page.class), eq(project.getId()), eq(1), eq(""), eq(""), eq(""),
eq(0), Mockito.any(), eq("192.168.xx.xx"), any(), any())).thenReturn(pageReturn);
Map<String, Object> executorErrorStartDateRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "",
Map<String, Object> executorErrorStartDateRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "", "",
"", "error date", null, "", ExecutionStatus.SUCCESS, "192.168.xx.xx", 1, 20);
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, executorErrorStartDateRes.get(Constants.STATUS));
Map<String, Object> executorErrorEndDateRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "",
Map<String, Object> executorErrorEndDateRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "", "",
"", null, "error date", "", ExecutionStatus.SUCCESS, "192.168.xx.xx", 1, 20);
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, executorErrorEndDateRes.get(Constants.STATUS));
}

13
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java

@ -73,8 +73,6 @@ public class TenantServiceTest {
private static final String tenantCode = "TenantServiceTest";
private static final String tenantName = "TenantServiceTest";
@Test
public void testCreateTenant() {
@ -83,17 +81,17 @@ public class TenantServiceTest {
try {
//check tenantCode
Map<String, Object> result =
tenantService.createTenant(getLoginUser(), "%!1111", tenantName, 1, "TenantServiceTest");
tenantService.createTenant(getLoginUser(), "%!1111", 1, "TenantServiceTest");
logger.info(result.toString());
Assert.assertEquals(Status.VERIFY_TENANT_CODE_ERROR, result.get(Constants.STATUS));
//check exist
result = tenantService.createTenant(loginUser, tenantCode, tenantName, 1, "TenantServiceTest");
result = tenantService.createTenant(loginUser, tenantCode, 1, "TenantServiceTest");
logger.info(result.toString());
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS));
// success
result = tenantService.createTenant(loginUser, "test", "test", 1, "TenantServiceTest");
result = tenantService.createTenant(loginUser, "test", 1, "TenantServiceTest");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
@ -126,11 +124,11 @@ public class TenantServiceTest {
try {
// id not exist
Map<String, Object> result =
tenantService.updateTenant(getLoginUser(), 912222, tenantCode, tenantName, 1, "desc");
tenantService.updateTenant(getLoginUser(), 912222, tenantCode, 1, "desc");
logger.info(result.toString());
// success
Assert.assertEquals(Status.TENANT_NOT_EXIST, result.get(Constants.STATUS));
result = tenantService.updateTenant(getLoginUser(), 1, tenantCode, "TenantServiceTest001", 1, "desc");
result = tenantService.updateTenant(getLoginUser(), 1, tenantCode, 1, "desc");
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
} catch (Exception e) {
@ -236,7 +234,6 @@ public class TenantServiceTest {
Tenant tenant = new Tenant();
tenant.setId(id);
tenant.setTenantCode(tenantCode);
tenant.setTenantName(tenantName);
return tenant;
}

8
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java

@ -149,9 +149,11 @@ public class UdfFuncServiceTest {
}
@Test
public void testQueryResourceList(){
Mockito.when(udfFuncMapper.getUdfFuncByType(1, 1)).thenReturn(getList());
Map<String, Object> result = udfFuncService.queryResourceList(getLoginUser(),1);
public void testQueryUdfFuncList(){
User user = getLoginUser();
user.setUserType(UserType.GENERAL_USER);
Mockito.when(udfFuncMapper.getUdfFuncByType(user.getId(), UdfType.HIVE.ordinal())).thenReturn(getList());
Map<String, Object> result = udfFuncService.queryUdfFuncList(user,UdfType.HIVE.ordinal());
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
List<UdfFunc> udfFuncList = (List<UdfFunc>) result.get(Constants.DATA_LIST);

10
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java

@ -225,13 +225,13 @@ public class UsersServiceTest {
String userPassword = "userTest0001";
try {
//user not exist
Map<String, Object> result = usersService.updateUser(0,userName,userPassword,"3443@qq.com",1,"13457864543","queue", 1);
Map<String, Object> result = usersService.updateUser(getLoginUser(), 0,userName,userPassword,"3443@qq.com",1,"13457864543","queue", 1);
Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS));
logger.info(result.toString());
//success
when(userMapper.selectById(1)).thenReturn(getUser());
result = usersService.updateUser(1,userName,userPassword,"32222s@qq.com",1,"13457864543","queue", 1);
result = usersService.updateUser(getLoginUser(), 1,userName,userPassword,"32222s@qq.com",1,"13457864543","queue", 1);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
} catch (Exception e) {
@ -357,6 +357,12 @@ public class UsersServiceTest {
}
private User getLoginUser(){
User loginUser = new User();
loginUser.setId(1);
loginUser.setUserType(UserType.ADMIN_USER);
return loginUser;
}
@Test
public void getUserInfo(){

47
dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/JsonSerializerTest.java → dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/RegexUtilsTest.java

@ -15,44 +15,25 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote;
package org.apache.dolphinscheduler.api.utils;
import org.apache.dolphinscheduler.remote.utils.JsonSerializer;
import org.junit.Assert;
import org.junit.Test;
public class JsonSerializerTest {
/**
* RegexUtils test case
*/
public class RegexUtilsTest {
@Test
public void testSerialize(){
TestObj testObj = new TestObj();
testObj.setAge(12);
byte[] serializeByte = JsonSerializer.serialize(testObj);
//
TestObj deserialize = JsonSerializer.deserialize(serializeByte, TestObj.class);
Assert.assertEquals(testObj.getAge(), deserialize.getAge());
public void testIsNumeric() {
String num1 = "123467854678";
boolean numeric = RegexUtils.isNumeric(num1);
Assert.assertTrue(numeric);
String num2 = "0.0.01";
boolean numeric2 = RegexUtils.isNumeric(num2);
Assert.assertFalse(numeric2);
}
static class TestObj {
private int age;
public int getAge() {
return age;
}
public void setAge(int age) {
this.age = age;
}
@Override
public String toString() {
return "TestObj{" +
"age=" + age +
'}';
}
}
}
}

2
dolphinscheduler-common/pom.xml

@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
<version>1.3.2-SNAPSHOT</version>
<version>1.3.4-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-common</artifactId>
<name>dolphinscheduler-common</name>

47
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@ -221,6 +221,11 @@ public final class Constants {
*/
public static final String COLON = ":";
/**
* SPACE " "
*/
public static final String SPACE = " ";
/**
* SINGLE_SLASH /
*/
@ -231,6 +236,15 @@ public final class Constants {
*/
public static final String DOUBLE_SLASH = "//";
/**
* SINGLE_QUOTES "'"
*/
public static final String SINGLE_QUOTES = "'";
/**
* DOUBLE_QUOTES "\""
*/
public static final String DOUBLE_QUOTES = "\"";
/**
* SEMICOLON ;
*/
@ -523,10 +537,9 @@ public final class Constants {
public static final int HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH = 10;
/**
* hadoop params constant
*/
/**
* hadoop params
* jar
*/
public static final String JAR = "jar";
@ -833,15 +846,15 @@ public final class Constants {
public static final String FLINK_MAIN_CLASS = "-c";
public static final int[] NOT_TERMINATED_STATES = new int[]{
ExecutionStatus.SUBMITTED_SUCCESS.ordinal(),
ExecutionStatus.RUNNING_EXECUTION.ordinal(),
ExecutionStatus.DELAY_EXECUTION.ordinal(),
ExecutionStatus.READY_PAUSE.ordinal(),
ExecutionStatus.READY_STOP.ordinal(),
ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal(),
ExecutionStatus.WAITTING_THREAD.ordinal(),
ExecutionStatus.WAITTING_DEPEND.ordinal()
public static final int[] NOT_TERMINATED_STATES = new int[] {
ExecutionStatus.SUBMITTED_SUCCESS.ordinal(),
ExecutionStatus.RUNNING_EXECUTION.ordinal(),
ExecutionStatus.DELAY_EXECUTION.ordinal(),
ExecutionStatus.READY_PAUSE.ordinal(),
ExecutionStatus.READY_STOP.ordinal(),
ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal(),
ExecutionStatus.WAITTING_THREAD.ordinal(),
ExecutionStatus.WAITTING_DEPEND.ordinal()
};
/**
@ -1009,4 +1022,14 @@ public final class Constants {
* Network IP gets priority, default inner outer
*/
public static final String NETWORK_PRIORITY_STRATEGY = "dolphin.scheduler.network.priority.strategy";
/**
* exec shell scripts
*/
public static final String SH = "sh";
/**
* pstree, get pud and sub pid
*/
public static final String PSTREE = "pstree";
}

18
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/QueryType.java → dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java

@ -14,20 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.enums;
public enum QueryType {
FORM,
SQL;
package org.apache.dolphinscheduler.common.enums;
public static QueryType getEnum(int value){
for (QueryType e:QueryType.values()) {
if(e.ordinal() == value) {
return e;
}
}
//For values out of enum scope
return null;
}
public enum Event {
ACK,
RESULT;
}

41
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SqoopQueryType.java

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.enums;
public enum SqoopQueryType {
FORM(0, "SQOOP_QUERY_FORM"),
SQL(1, "SQOOP_QUERY_SQL");
private final Integer code;
private final String desc;
SqoopQueryType(Integer code, String desc) {
this.code = code;
this.desc = desc;
}
public Integer getCode() {
return code;
}
public String getDesc() {
return desc;
}
}

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ResourceInfo.java

@ -42,6 +42,4 @@ public class ResourceInfo {
public void setRes(String res) {
this.res = res;
}
}

3
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/conditions/ConditionsParameters.java

@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.common.model.DependentTaskModel;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import java.util.ArrayList;
import java.util.List;
public class ConditionsParameters extends AbstractParameters {
@ -43,7 +44,7 @@ public class ConditionsParameters extends AbstractParameters {
@Override
public List<ResourceInfo> getResourceFilesList() {
return null;
return new ArrayList<>();
}
public List<DependentTaskModel> getDependTaskList() {

65
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java

@ -14,15 +14,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.task.datax;
import java.util.ArrayList;
import java.util.List;
package org.apache.dolphinscheduler.common.task.datax;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import java.util.ArrayList;
import java.util.List;
/**
* DataX parameter
@ -89,6 +90,16 @@ public class DataxParameters extends AbstractParameters {
*/
private int jobSpeedRecord;
/**
* Xms memory
*/
private int xms;
/**
* Xmx memory
*/
private int xmx;
public int getCustomConfig() {
return customConfig;
}
@ -185,6 +196,22 @@ public class DataxParameters extends AbstractParameters {
this.jobSpeedRecord = jobSpeedRecord;
}
public int getXms() {
return xms;
}
public void setXms(int xms) {
this.xms = xms;
}
public int getXmx() {
return xmx;
}
public void setXmx(int xmx) {
this.xmx = xmx;
}
@Override
public boolean checkParameters() {
if (customConfig == Flag.NO.ordinal()) {
@ -204,19 +231,21 @@ public class DataxParameters extends AbstractParameters {
@Override
public String toString() {
return "DataxParameters{" +
"customConfig=" + customConfig +
", json='" + json + '\'' +
", dsType='" + dsType + '\'' +
", dataSource=" + dataSource +
", dtType='" + dtType + '\'' +
", dataTarget=" + dataTarget +
", sql='" + sql + '\'' +
", targetTable='" + targetTable + '\'' +
", preStatements=" + preStatements +
", postStatements=" + postStatements +
", jobSpeedByte=" + jobSpeedByte +
", jobSpeedRecord=" + jobSpeedRecord +
'}';
return "DataxParameters{"
+ "customConfig=" + customConfig
+ ", json='" + json + '\''
+ ", dsType='" + dsType + '\''
+ ", dataSource=" + dataSource
+ ", dtType='" + dtType + '\''
+ ", dataTarget=" + dataTarget
+ ", sql='" + sql + '\''
+ ", targetTable='" + targetTable + '\''
+ ", preStatements=" + preStatements
+ ", postStatements=" + postStatements
+ ", jobSpeedByte=" + jobSpeedByte
+ ", jobSpeedRecord=" + jobSpeedRecord
+ ", xms=" + xms
+ ", xmx=" + xmx
+ '}';
}
}

1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java

@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.common.task.flink;
import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import java.util.ArrayList;
import java.util.List;

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/spark/SparkParameters.java

@ -216,7 +216,7 @@ public class SparkParameters extends AbstractParameters {
@Override
public boolean checkParameters() {
return mainJar != null && programType != null && sparkVersion != null;
return mainJar != null && programType != null;
}
@Override

5
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java

@ -30,13 +30,14 @@ import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.charset.UnsupportedCharsetException;
import java.util.Optional;
@ -217,7 +218,7 @@ public class FileUtils {
return false;
}
bufferedReader = new BufferedReader(new StringReader(content));
bufferedWriter = new BufferedWriter(new FileWriter(distFile));
bufferedWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(distFile), StandardCharsets.UTF_8));
char[] buf = new char[1024];
int len;
while ((len = bufferedReader.read(buf)) != -1) {

3
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java

@ -39,6 +39,7 @@ import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.security.PrivilegedExceptionAction;
import java.util.Collections;
@ -263,7 +264,7 @@ public class HadoopUtils implements Closeable {
}
try (FSDataInputStream in = fs.open(new Path(hdfsFilePath))) {
BufferedReader br = new BufferedReader(new InputStreamReader(in));
BufferedReader br = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
Stream<String> stream = br.lines().skip(skipLineNums).limit(limit);
return stream.collect(Collectors.toList());
}

39
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java

@ -17,6 +17,8 @@
package org.apache.dolphinscheduler.common.utils;
import static java.nio.charset.StandardCharsets.UTF_8;
import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT;
import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES;
import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL;
@ -126,6 +128,22 @@ public class JSONUtils {
return null;
}
/**
* deserialize
*
* @param src byte array
* @param clazz class
* @param <T> deserialize type
* @return deserialize type
*/
public static <T> T parseObject(byte[] src, Class<T> clazz) {
if (src == null) {
return null;
}
String json = new String(src, UTF_8);
return parseObject(json, clazz);
}
/**
* json to list
*
@ -253,6 +271,27 @@ public class JSONUtils {
}
}
/**
* serialize to json byte
*
* @param obj object
* @param <T> object type
* @return byte array
*/
public static <T> byte[] toJsonByteArray(T obj) {
if (obj == null) {
return null;
}
String json = "";
try {
json = toJsonString(obj);
} catch (Exception e) {
logger.error("json serialize exception.", e);
}
return json.getBytes(UTF_8);
}
public static ObjectNode parseObject(String text) {
try {
return (ObjectNode) objectMapper.readTree(text);

6
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java

@ -45,6 +45,10 @@ public class StringUtils {
}
public static String replaceNRTtoUnderline(String src) {
return src.replaceAll("[\n|\r|\t]", "_");
if (isBlank(src)) {
return src;
} else {
return src.replaceAll("[\n|\r|\t]", "_");
}
}
}

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save