Browse Source

Merge pull request #21 from apache/dev

update
pull/2/head
samz406 5 years ago committed by GitHub
parent
commit
a1fe80eaa3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 11
      .github/workflows/ci_ut.yml
  2. 16
      README.md
  3. 24
      README_zh_CN.md
  4. 48
      dolphinscheduler-alert/pom.xml
  5. 4
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java
  6. 41
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplate.java
  7. 54
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactory.java
  8. 161
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java
  9. 4
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java
  10. 2
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java
  11. 2
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java
  12. 2
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java
  13. 129
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java
  14. 4
      dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/PropertyUtils.java
  15. 3
      dolphinscheduler-alert/src/main/resources/alert.properties
  16. 17
      dolphinscheduler-alert/src/main/resources/mail_templates/alert_mail_template.ftl
  17. 66
      dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactoryTest.java
  18. 123
      dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplateTest.java
  19. 51
      dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java
  20. 105
      dolphinscheduler-api/pom.xml
  21. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/ServiceModelToSwagger2MapperImpl.java
  22. 6
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/BaseController.java
  23. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java
  24. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java
  25. 35
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoginController.java
  26. 6
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java
  27. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java
  28. 9
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java
  29. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java
  30. 6
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java
  31. 17
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java
  32. 37
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/AuthenticationType.java
  33. 40
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/Authenticator.java
  34. 76
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/PasswordAuthenticator.java
  35. 67
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/SecurityConfig.java
  36. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java
  37. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java
  38. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java
  39. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java
  40. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java
  41. 497
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java
  42. 19
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java
  43. 37
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
  44. 6
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java
  45. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SessionService.java
  46. 9
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java
  47. 22
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java
  48. 57
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java
  49. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java
  50. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java
  51. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java
  52. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitor.java
  53. 5
      dolphinscheduler-api/src/main/resources/application-api.properties
  54. 4
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AbstractControllerTest.java
  55. 71
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptorTest.java
  56. 96
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/security/PasswordAuthenticatorTest.java
  57. 43
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/security/SecurityConfigTest.java
  58. 9
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java
  59. 294
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java
  60. 10
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java
  61. 7
      dolphinscheduler-common/pom.xml
  62. 1
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/ClickHouseDataSource.java
  63. 1
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/DB2ServerDataSource.java
  64. 1
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/HiveDataSource.java
  65. 1
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/MySQLDataSource.java
  66. 1
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/OracleDataSource.java
  67. 1
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/PostgreDataSource.java
  68. 1
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SQLServerDataSource.java
  69. 1
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SparkDataSource.java
  70. 1
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/DateInterval.java
  71. 1
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNodeRelation.java
  72. 37
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java
  73. 26
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ArrayUtils.java
  74. 3
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ConnectionUtils.java
  75. 398
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java
  76. 12
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java
  77. 5
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
  78. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java
  79. 45
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java
  80. 30
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java
  81. 174
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DependentUtilsTest.java
  82. 170
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java
  83. 8
      dolphinscheduler-dao/pom.xml
  84. 2
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java
  85. 12
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java
  86. 2
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java
  87. 43
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AccessToken.java
  88. 1
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Alert.java
  89. 14
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AlertGroup.java
  90. 1
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java
  91. 4
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java
  92. 2
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java
  93. 96
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java
  94. 24
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstanceMap.java
  95. 2
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Project.java
  96. 2
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Schedule.java
  97. 2
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java
  98. 2
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java
  99. 2
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UserAlertGroup.java
  100. 2
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerGroup.java
  101. Some files were not shown because too many files have changed in this diff Show More

11
.github/workflows/ci_ut.yml

@ -15,7 +15,7 @@
# limitations under the License.
#
on: ["pull_request"]
on: [push, "pull_request"]
env:
DOCKER_DIR: ./docker
LOG_DIR: /tmp/dolphinscheduler
@ -49,11 +49,12 @@ jobs:
export MAVEN_OPTS='-Dmaven.repo.local=.m2/repository -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:-UseGCOverheadLimit -Xmx3g'
mvn test -Dmaven.test.skip=false cobertura:cobertura
CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash)
- name: Run SonarCloud analysis
- name: Run SonarCloud Analysis
run: >
mvn clean --batch-mode
verify
org.sonarsource.scanner.maven:sonar-maven-plugin:3.6.1.1688:sonar
mvn verify --batch-mode
org.sonarsource.scanner.maven:sonar-maven-plugin:3.6.1.1688:sonar
-Dsonar.junit.reportPaths=target/cobertura
-Dmaven.test.skip=true
-Dsonar.host.url=https://sonarcloud.io
-Dsonar.organization=apache
-Dsonar.projectKey=apache-dolphinscheduler

16
README.md

@ -58,25 +58,25 @@ Overload processing: Task queue mechanism, the number of schedulable tasks on a
### Document
- <a href="https://dolphinscheduler.apache.org/en-us/docs/user_doc/backend-deployment.html" target="_blank">Backend deployment documentation</a>
- <a href="https://dolphinscheduler.apache.org/en-us/docs/1.2.0/user_doc/backend-deployment.html" target="_blank">Backend deployment documentation</a>
- <a href="https://dolphinscheduler.apache.org/en-us/docs/user_doc/frontend-deployment.html" target="_blank">Front-end deployment documentation</a>
- <a href="https://dolphinscheduler.apache.org/en-us/docs/1.2.0/user_doc/frontend-deployment.html" target="_blank">Front-end deployment documentation</a>
- [**User manual**](https://dolphinscheduler.apache.org/en-us/docs/user_doc/system-manual.html?_blank "System manual")
- [**User manual**](https://dolphinscheduler.apache.org/en-us/docs/1.2.0/user_doc/system-manual.html?_blank "System manual")
- [**Upgrade document**](https://dolphinscheduler.apache.org/en-us/docs/release/upgrade.html?_blank "Upgrade document")
- [**Upgrade document**](https://dolphinscheduler.apache.org/en-us/docs/1.2.0/release/upgrade.html?_blank "Upgrade document")
- <a href="http://106.75.43.194:8888" target="_blank">Online Demo</a>
More documentation please refer to <a href="https://dolphinscheduler.apache.org/en-us/docs/user_doc/quick-start.html" target="_blank">[DolphinScheduler online documentation]</a>
More documentation please refer to <a href="https://dolphinscheduler.apache.org/en-us/docs/1.2.0/user_doc/quick-start.html" target="_blank">[DolphinScheduler online documentation]</a>
### Recent R&D plan
Work plan of Dolphin Scheduler: [R&D plan](https://github.com/apache/incubator-dolphinscheduler/projects/1), Under the `In Develop` card is what is currently being developed, TODO card is to be done (including feature ideas)
### How to contribute code
### How to contribute
Welcome to participate in contributing code, please refer to the process of submitting the code:
[[How to contribute code](https://github.com/apache/incubator-dolphinscheduler/issues/310)]
Welcome to participate in contributing, please refer to the process of submitting the code:
[[How to contribute](https://dolphinscheduler.apache.org/en-us/docs/development/contribute.html)]
### How to Build

24
README_zh_CN.md

@ -44,27 +44,27 @@ Dolphin Scheduler Official Website
### 文档
- <a href="https://dolphinscheduler.apache.org/zh-cn/docs/user_doc/backend-deployment.html" target="_blank">后端部署文档</a>
- <a href="https://dolphinscheduler.apache.org/zh-cn/docs/1.2.0/user_doc/backend-deployment.html" target="_blank">后端部署文档</a>
- <a href="https://dolphinscheduler.apache.org/zh-cn/docs/user_doc/frontend-deployment.html" target="_blank">前端部署文档</a>
- <a href="https://dolphinscheduler.apache.org/zh-cn/docs/1.2.0/user_doc/frontend-deployment.html" target="_blank">前端部署文档</a>
- [**使用手册**](https://dolphinscheduler.apache.org/zh-cn/docs/user_doc/system-manual.html?_blank "系统使用手册")
- [**使用手册**](https://dolphinscheduler.apache.org/zh-cn/docs/1.2.0/user_doc/system-manual.html?_blank "系统使用手册")
- [**升级文档**](https://dolphinscheduler.apache.org/zh-cn/docs/release/upgrade.html?_blank "升级文档")
- [**升级文档**](https://dolphinscheduler.apache.org/zh-cn/docs/1.2.0/release/upgrade.html?_blank "升级文档")
- <a href="http://106.75.43.194:8888" target="_blank">我要体验</a>
更多文档请参考 <a href="https://dolphinscheduler.apache.org/zh-cn/docs/user_doc/quick-start.html" target="_blank">DolphinScheduler中文在线文档</a>
更多文档请参考 <a href="https://dolphinscheduler.apache.org/zh-cn/docs/1.2.0/user_doc/quick-start.html" target="_blank">DolphinScheduler中文在线文档</a>
### 近期研发计划
DolphinScheduler的工作计划:<a href="https://github.com/apache/incubator-dolphinscheduler/projects/1" target="_blank">研发计划</a> ,其中 In Develop卡片下是正在研发的功能,TODO卡片是待做事项(包括 feature ideas)
### 贡献代码
### 参与贡献
非常欢迎大家来参与贡献代码,提交代码流程请参考:
[[How to contribute code](https://github.com/apache/incubator-dolphinscheduler/issues/310)]
非常欢迎大家来参与贡献,贡献流程请参考:
[[参与贡献](https://dolphinscheduler.apache.org/zh-cn/docs/development/contribute.html)]
### How to Build
@ -88,12 +88,12 @@ Dolphin Scheduler使用了很多优秀的开源项目,比如google的guava、g
### 获得帮助
1. Submit an issue
1. Mail to dev-subscribe@dolphinscheduler.apache.org, follow the reply to subscribe the mail list. then you can send mail to dev@dolphinscheduler.apache.org.
1. Contact WeChat group manager, ID 510570367. This is for Mandarin(CN) discussion.
1. 提交issue
1. 先订阅邮件开发列表:[订阅邮件列表](https://dolphinscheduler.apache.org/zh-cn/docs/1.2.0/user_doc/subscribe.html), 订阅成功后发送邮件到dev@dolphinscheduler.apache.org.
1. 联系微信群助手(ID:dailidong66). 微信仅用于中国用户讨论.
### 版权
Please refer to [LICENSE](https://github.com/apache/incubator-dolphinscheduler/blob/dev/LICENSE) file.
请参考 [LICENSE](https://github.com/apache/incubator-dolphinscheduler/blob/dev/LICENSE) 文件.

48
dolphinscheduler-alert/pom.xml

@ -36,15 +36,34 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<type>jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-email</artifactId>
<groupId>org.powermock</groupId>
<artifactId>powermock-module-junit4</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-api-mockito2</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.freemarker</groupId>
<artifactId>freemarker</artifactId>
<groupId>org.apache.commons</groupId>
<artifactId>commons-email</artifactId>
</dependency>
<dependency>
@ -67,21 +86,6 @@
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
@ -92,12 +96,6 @@
<artifactId>logback-classic</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
<!--excel poi-->
<dependency>
<groupId>org.apache.poi</groupId>

4
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java

@ -16,14 +16,14 @@
*/
package org.apache.dolphinscheduler.alert.runner;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.dolphinscheduler.alert.manager.EmailManager;
import org.apache.dolphinscheduler.alert.manager.EnterpriseWeChatManager;
import org.apache.dolphinscheduler.alert.utils.Constants;
import org.apache.dolphinscheduler.alert.utils.EnterpriseWeChatUtils;
import org.apache.dolphinscheduler.common.enums.AlertStatus;
import org.apache.dolphinscheduler.common.enums.AlertType;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.entity.Alert;
import org.apache.dolphinscheduler.dao.entity.User;

41
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplate.java

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.alert.template;
import org.apache.dolphinscheduler.common.enums.ShowType;
/**
* alert message template
*/
public interface AlertTemplate {
/**
* get a message from a specified alert template
* @param content alert message content
* @param showType show type
* @param showAll whether to show all
* @return a message from a specified alert template
*/
String getMessageFromTemplate(String content, ShowType showType,boolean showAll);
/**
* default showAll is true
*/
default String getMessageFromTemplate(String content,ShowType showType){
return getMessageFromTemplate(content,showType,true);
}
}

54
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactory.java

@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.alert.template;
import org.apache.dolphinscheduler.alert.template.impl.DefaultHTMLTemplate;
import org.apache.dolphinscheduler.alert.utils.Constants;
import org.apache.dolphinscheduler.alert.utils.PropertyUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* the alert template factory
*/
public class AlertTemplateFactory {
private static final Logger logger = LoggerFactory.getLogger(AlertTemplateFactory.class);
private static final String alertTemplate = PropertyUtils.getString(Constants.ALERT_TEMPLATE);
private AlertTemplateFactory(){}
/**
* get a template from alert.properties conf file
* @return a template, default is DefaultHTMLTemplate
*/
public static AlertTemplate getMessageTemplate() {
if(StringUtils.isEmpty(alertTemplate)){
return new DefaultHTMLTemplate();
}
switch (alertTemplate){
case "html":
return new DefaultHTMLTemplate();
default:
throw new IllegalArgumentException(String.format("not support alert template: %s",alertTemplate));
}
}
}

161
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplate.java

@ -0,0 +1,161 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.alert.template.impl;
import org.apache.dolphinscheduler.alert.template.AlertTemplate;
import org.apache.dolphinscheduler.alert.utils.Constants;
import org.apache.dolphinscheduler.alert.utils.JSONUtils;
import org.apache.dolphinscheduler.alert.utils.MailUtils;
import org.apache.dolphinscheduler.common.enums.ShowType;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
import static org.apache.dolphinscheduler.common.utils.Preconditions.*;
/**
* the default html alert message template
*/
public class DefaultHTMLTemplate implements AlertTemplate {
public static final Logger logger = LoggerFactory.getLogger(DefaultHTMLTemplate.class);
@Override
public String getMessageFromTemplate(String content, ShowType showType,boolean showAll) {
switch (showType){
case TABLE:
return getTableTypeMessage(content,showAll);
case TEXT:
return getTextTypeMessage(content,showAll);
default:
throw new IllegalArgumentException(String.format("not support showType: %s in DefaultHTMLTemplate",showType));
}
}
/**
* get alert message which type is TABLE
* @param content message content
* @param showAll weather to show all
* @return alert message
*/
private String getTableTypeMessage(String content,boolean showAll){
if (StringUtils.isNotEmpty(content)){
List<LinkedHashMap> mapItemsList = JSONUtils.toList(content, LinkedHashMap.class);
if(!showAll && mapItemsList.size() > Constants.NUMBER_1000){
mapItemsList = mapItemsList.subList(0,Constants.NUMBER_1000);
}
StringBuilder contents = new StringBuilder(200);
boolean flag = true;
String title = "";
for (LinkedHashMap mapItems : mapItemsList){
Set<Map.Entry<String, Object>> entries = mapItems.entrySet();
Iterator<Map.Entry<String, Object>> iterator = entries.iterator();
StringBuilder t = new StringBuilder(Constants.TR);
StringBuilder cs = new StringBuilder(Constants.TR);
while (iterator.hasNext()){
Map.Entry<String, Object> entry = iterator.next();
t.append(Constants.TH).append(entry.getKey()).append(Constants.TH_END);
cs.append(Constants.TD).append(String.valueOf(entry.getValue())).append(Constants.TD_END);
}
t.append(Constants.TR_END);
cs.append(Constants.TR_END);
if (flag){
title = t.toString();
}
flag = false;
contents.append(cs);
}
return getMessageFromHtmlTemplate(title,contents.toString());
}
return content;
}
/**
* get alert message which type is TEXT
* @param content message content
* @param showAll weather to show all
* @return alert message
*/
private String getTextTypeMessage(String content,boolean showAll){
if (StringUtils.isNotEmpty(content)){
List<String> list;
try {
list = JSONUtils.toList(content,String.class);
}catch (Exception e){
logger.error("json format exception",e);
return null;
}
StringBuilder contents = new StringBuilder(100);
for (String str : list){
contents.append(Constants.TR);
contents.append(Constants.TD).append(str).append(Constants.TD_END);
contents.append(Constants.TR_END);
}
return getMessageFromHtmlTemplate(null,contents.toString());
}
return content;
}
/**
* get alert message from a html template
* @param title message title
* @param content message content
* @return alert message which use html template
*/
private String getMessageFromHtmlTemplate(String title,String content){
checkNotNull(content);
String htmlTableThead = StringUtils.isEmpty(title) ? "" : String.format("<thead>%s</thead>\n",title);
return "<html>\n" +
" <head>\n" +
" <title>dolphinscheduler</title>\n" +
" <meta name='Keywords' content=''>\n" +
" <meta name='Description' content=''>\n" +
" <style type=\"text/css\">\n" +
" table {margin-top:0px;padding-top:0px;border:1px solid;font-size: 14px;color: #333333;border-width: 1px;border-color: #666666;border-collapse: collapse;}\n" +
" table th {border-width: 1px;padding: 8px;border-style: solid;border-color: #666666;background-color: #dedede;text-align: right;}\n" +
" table td {border-width: 1px;padding: 8px;border-style: solid;border-color: #666666;background-color: #ffffff;text-align: right;}\n" +
" </style>\n" +
" </head>\n" +
" <body style=\"margin:0;padding:0\">\n" +
" <table border=\"1px\" cellpadding=\"5px\" cellspacing=\"-10px\">\n" + htmlTableThead + content +
" </table>\n" +
" </body>\n" +
"</html>";
}
}

4
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/Constants.java

@ -75,6 +75,8 @@ public class Constants {
public static final int NUMBER_1000 = 1000;
public static final String ALERT_TEMPLATE = "alert.template";
public static final String SPRING_DATASOURCE_DRIVER_CLASS_NAME = "spring.datasource.driver-class-name";
public static final String SPRING_DATASOURCE_URL = "spring.datasource.url";
@ -115,8 +117,6 @@ public class Constants {
public static final String DEVELOPMENT = "development";
public static final String CLASSPATH_MAIL_TEMPLATES_ALERT_MAIL_TEMPLATE_FTL = "classpath:mail_templates/alert_mail_template.ftl";
public static final String TR = "<tr>";
public static final String TD = "<td>";

2
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java

@ -17,11 +17,11 @@
package org.apache.dolphinscheduler.alert.utils;
import org.apache.dolphinscheduler.common.enums.ShowType;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Alert;
import com.alibaba.fastjson.JSON;
import com.google.common.reflect.TypeToken;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;

2
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java

@ -16,7 +16,7 @@
*/
package org.apache.dolphinscheduler.alert.utils;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
public class FuncUtils {

2
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/JSONUtils.java

@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.alert.utils;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.lang3.StringUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

129
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java

@ -16,16 +16,13 @@
*/
package org.apache.dolphinscheduler.alert.utils;
import org.apache.dolphinscheduler.alert.template.AlertTemplate;
import org.apache.dolphinscheduler.alert.template.AlertTemplateFactory;
import org.apache.dolphinscheduler.common.enums.ShowType;
import freemarker.cache.StringTemplateLoader;
import freemarker.template.Configuration;
import freemarker.template.Template;
import freemarker.template.TemplateException;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.mail.EmailException;
import org.apache.commons.mail.HtmlEmail;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.ResourceUtils;
@ -35,7 +32,6 @@ import javax.mail.internet.*;
import java.io.*;
import java.util.*;
import static org.apache.dolphinscheduler.alert.utils.PropertyUtils.getInt;
/**
@ -69,25 +65,7 @@ public class MailUtils {
public static final String sslTrust = PropertyUtils.getString(Constants.MAIL_SMTP_SSL_TRUST);
private static Template MAIL_TEMPLATE;
static {
Configuration cfg = new Configuration(Configuration.VERSION_2_3_21);
cfg.setDefaultEncoding(Constants.UTF_8);
StringTemplateLoader stringTemplateLoader = new StringTemplateLoader();
cfg.setTemplateLoader(stringTemplateLoader);
InputStreamReader isr = null;
try {
isr = new InputStreamReader(new FileInputStream(ResourceUtils.getFile(Constants.CLASSPATH_MAIL_TEMPLATES_ALERT_MAIL_TEMPLATE_FTL)),
Constants.UTF_8);
MAIL_TEMPLATE = new Template("alert_mail_template", isr, cfg);
} catch (Exception e) {
MAIL_TEMPLATE = null;
} finally {
IOUtils.closeQuietly(isr);
}
}
public static final AlertTemplate alertTemplate = AlertTemplateFactory.getMessageTemplate();
/**
@ -174,46 +152,7 @@ public class MailUtils {
* @return the html table form
*/
private static String htmlTable(String content, boolean showAll){
if (StringUtils.isNotEmpty(content)){
List<LinkedHashMap> mapItemsList = JSONUtils.toList(content, LinkedHashMap.class);
if(!showAll && mapItemsList.size() > Constants.NUMBER_1000){
mapItemsList = mapItemsList.subList(0,Constants.NUMBER_1000);
}
StringBuilder contents = new StringBuilder(200);
boolean flag = true;
String title = "";
for (LinkedHashMap mapItems : mapItemsList){
Set<Map.Entry<String, Object>> entries = mapItems.entrySet();
Iterator<Map.Entry<String, Object>> iterator = entries.iterator();
StringBuilder t = new StringBuilder(Constants.TR);
StringBuilder cs = new StringBuilder(Constants.TR);
while (iterator.hasNext()){
Map.Entry<String, Object> entry = iterator.next();
t.append(Constants.TH).append(entry.getKey()).append(Constants.TH_END);
cs.append(Constants.TD).append(String.valueOf(entry.getValue())).append(Constants.TD_END);
}
t.append(Constants.TR_END);
cs.append(Constants.TR_END);
if (flag){
title = t.toString();
}
flag = false;
contents.append(cs);
}
return getTemplateContent(title,contents.toString());
}
return null;
return alertTemplate.getMessageFromTemplate(content,ShowType.TABLE,showAll);
}
/**
@ -231,33 +170,9 @@ public class MailUtils {
* @return text in html form
*/
private static String htmlText(String content){
if (StringUtils.isNotEmpty(content)){
List<String> list;
try {
list = JSONUtils.toList(content,String.class);
}catch (Exception e){
logger.error("json format exception",e);
return null;
}
StringBuilder contents = new StringBuilder(100);
for (String str : list){
contents.append(Constants.TR);
contents.append(Constants.TD).append(str).append(Constants.TD_END);
contents.append(Constants.TR_END);
}
return getTemplateContent(null,contents.toString());
}
return null;
return alertTemplate.getMessageFromTemplate(content,ShowType.TEXT);
}
/**
* send mail as Excel attachment
* @param receivers the receiver list
@ -422,32 +337,8 @@ public class MailUtils {
* @param e the exception
*/
private static void handleException(Collection<String> receivers, Map<String, Object> retMap, Exception e) {
logger.error("Send email to {} failed", StringUtils.join(",", receivers), e);
retMap.put(Constants.MESSAGE, "Send email to {" + StringUtils.join(",", receivers) + "} failed," + e.toString());
logger.error("Send email to {} failed {}", receivers, e);
retMap.put(Constants.MESSAGE, "Send email to {" + StringUtils.join(receivers, ",") + "} failed," + e.toString());
}
/**
* get the content of the template
* @param title the title
* @param content the content to retrieve
* @return the content in the template or null if exception occurs
*/
private static String getTemplateContent(String title,String content){
StringWriter out = new StringWriter();
Map<String,String> map = new HashMap<>();
if(null != title){
map.put(Constants.TITLE,title);
}
map.put(Constants.CONTENT,content);
try {
MAIL_TEMPLATE.process(map, out);
return out.toString();
} catch (TemplateException e) {
logger.error(e.getMessage(),e);
} catch (IOException e) {
logger.error(e.getMessage(),e);
}
return null;
}
}
}

4
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/PropertyUtils.java

@ -16,8 +16,8 @@
*/
package org.apache.dolphinscheduler.alert.utils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.common.utils.IOUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

3
dolphinscheduler-alert/src/main/resources/alert.properties

@ -18,6 +18,9 @@
#alert type is EMAIL/SMS
alert.type=EMAIL
# alter msg template, default is html template
#alert.template=html
# mail server configuration
mail.protocol=SMTP
mail.server.host=xxx.xxx.com

17
dolphinscheduler-alert/src/main/resources/mail_templates/alert_mail_template.ftl

@ -1,17 +0,0 @@
<#--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!DOCTYPE HTML PUBLIC '-//W3C//DTD HTML 4.01 Transitional//EN' 'http://www.w3.org/TR/html4/loose.dtd'><html><head><title> dolphinscheduler</title><meta name='Keywords' content=''><meta name='Description' content=''><style type="text/css">table { margin-top:0px; padding-top:0px; border:1px solid; font-size: 14px; color: #333333; border-width: 1px; border-color: #666666; border-collapse: collapse; } table th { border-width: 1px; padding: 8px; border-style: solid; border-color: #666666; background-color: #dedede; text-align: right;} table td { border-width: 1px; padding: 8px; border-style: solid; border-color: #666666; background-color: #ffffff; text-align: right;}</style></head><body style="margin:0;padding:0"><table border="1px" cellpadding="5px" cellspacing="-10px"><thead><#if title??> ${title}</#if></thead><#if content??> ${content}</#if></table></body></html>

66
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/AlertTemplateFactoryTest.java

@ -0,0 +1,66 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.alert.template;
import org.apache.dolphinscheduler.alert.template.impl.DefaultHTMLTemplate;
import org.apache.dolphinscheduler.alert.utils.Constants;
import org.apache.dolphinscheduler.alert.utils.PropertyUtils;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.mockito.Mockito.*;
import static org.junit.Assert.*;
/**
* test class for AlertTemplateFactory
*/
@RunWith(PowerMockRunner.class)
@PrepareForTest(PropertyUtils.class)
public class AlertTemplateFactoryTest {
private static final Logger logger = LoggerFactory.getLogger(AlertTemplateFactoryTest.class);
/**
* GetMessageTemplate method test
*/
@Test
public void testGetMessageTemplate(){
PowerMockito.mockStatic(PropertyUtils.class);
when(PropertyUtils.getString(Constants.ALERT_TEMPLATE)).thenReturn("html");
AlertTemplate defaultTemplate = AlertTemplateFactory.getMessageTemplate();
assertTrue(defaultTemplate instanceof DefaultHTMLTemplate);
}
/**
* GetMessageTemplate method throw Exception test
*/
@Test
public void testGetMessageTemplateException(){
AlertTemplate defaultTemplate = AlertTemplateFactory.getMessageTemplate();
assertTrue(defaultTemplate instanceof DefaultHTMLTemplate);
}
}

123
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/template/impl/DefaultHTMLTemplateTest.java

@ -0,0 +1,123 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.alert.template.impl;
import org.apache.dolphinscheduler.alert.utils.JSONUtils;
import org.apache.dolphinscheduler.common.enums.ShowType;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import static org.junit.Assert.*;
/**
* test class for DefaultHTMLTemplate
*/
public class DefaultHTMLTemplateTest{
private static final Logger logger = LoggerFactory.getLogger(DefaultHTMLTemplateTest.class);
/**
* only need test method GetMessageFromTemplate
*/
@Test
public void testGetMessageFromTemplate(){
DefaultHTMLTemplate template = new DefaultHTMLTemplate();
String tableTypeMessage = template.getMessageFromTemplate(list2String(), ShowType.TABLE,true);
assertEquals(tableTypeMessage,generateMockTableTypeResultByHand());
String textTypeMessage = template.getMessageFromTemplate(list2String(), ShowType.TEXT,true);
assertEquals(textTypeMessage,generateMockTextTypeResultByHand());
}
/**
* generate some simulation data
*/
private String list2String(){
LinkedHashMap<String, Object> map1 = new LinkedHashMap<>();
map1.put("mysql service name","mysql200");
map1.put("mysql address","192.168.xx.xx");
map1.put("port","3306");
map1.put("no index of number","80");
map1.put("database client connections","190");
LinkedHashMap<String, Object> map2 = new LinkedHashMap<>();
map2.put("mysql service name","mysql210");
map2.put("mysql address","192.168.xx.xx");
map2.put("port","3306");
map2.put("no index of number","10");
map2.put("database client connections","90");
List<LinkedHashMap<String, Object>> maps = new ArrayList<>();
maps.add(0,map1);
maps.add(1,map2);
String mapjson = JSONUtils.toJsonString(maps);
logger.info(mapjson);
return mapjson;
}
private String generateMockTableTypeResultByHand(){
return "<html>\n" +
" <head>\n" +
" <title>dolphinscheduler</title>\n" +
" <meta name='Keywords' content=''>\n" +
" <meta name='Description' content=''>\n" +
" <style type=\"text/css\">\n" +
" table {margin-top:0px;padding-top:0px;border:1px solid;font-size: 14px;color: #333333;border-width: 1px;border-color: #666666;border-collapse: collapse;}\n" +
" table th {border-width: 1px;padding: 8px;border-style: solid;border-color: #666666;background-color: #dedede;text-align: right;}\n" +
" table td {border-width: 1px;padding: 8px;border-style: solid;border-color: #666666;background-color: #ffffff;text-align: right;}\n" +
" </style>\n" +
" </head>\n" +
" <body style=\"margin:0;padding:0\">\n" +
" <table border=\"1px\" cellpadding=\"5px\" cellspacing=\"-10px\">\n" +
"<thead><tr><th>mysql service name</th><th>mysql address</th><th>port</th><th>no index of number</th><th>database client connections</th></tr></thead>\n" +
"<tr><td>mysql200</td><td>192.168.xx.xx</td><td>3306</td><td>80</td><td>190</td></tr><tr><td>mysql210</td><td>192.168.xx.xx</td><td>3306</td><td>10</td><td>90</td></tr> </table>\n" +
" </body>\n" +
"</html>";
}
private String generateMockTextTypeResultByHand(){
return "<html>\n" +
" <head>\n" +
" <title>dolphinscheduler</title>\n" +
" <meta name='Keywords' content=''>\n" +
" <meta name='Description' content=''>\n" +
" <style type=\"text/css\">\n" +
" table {margin-top:0px;padding-top:0px;border:1px solid;font-size: 14px;color: #333333;border-width: 1px;border-color: #666666;border-collapse: collapse;}\n" +
" table th {border-width: 1px;padding: 8px;border-style: solid;border-color: #666666;background-color: #dedede;text-align: right;}\n" +
" table td {border-width: 1px;padding: 8px;border-style: solid;border-color: #666666;background-color: #ffffff;text-align: right;}\n" +
" </style>\n" +
" </head>\n" +
" <body style=\"margin:0;padding:0\">\n" +
" <table border=\"1px\" cellpadding=\"5px\" cellspacing=\"-10px\">\n" +
"<tr><td>{\"mysql service name\":\"mysql200\",\"mysql address\":\"192.168.xx.xx\",\"database client connections\":\"190\",\"port\":\"3306\",\"no index of number\":\"80\"}</td></tr><tr><td>{\"mysql service name\":\"mysql210\",\"mysql address\":\"192.168.xx.xx\",\"database client connections\":\"90\",\"port\":\"3306\",\"no index of number\":\"10\"}</td></tr> </table>\n" +
" </body>\n" +
"</html>";
}
}

51
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/MailUtilsTest.java

@ -23,21 +23,11 @@ import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.DaoFactory;
import org.apache.dolphinscheduler.dao.entity.Alert;
import org.apache.dolphinscheduler.dao.entity.User;
import freemarker.cache.StringTemplateLoader;
import freemarker.template.Configuration;
import freemarker.template.Template;
import freemarker.template.TemplateException;
import org.apache.commons.io.IOUtils;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.ResourceUtils;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StringWriter;
import java.util.*;
@ -48,8 +38,8 @@ public class MailUtilsTest {
private static final Logger logger = LoggerFactory.getLogger(MailUtilsTest.class);
@Test
public void testSendMails() {
String[] receivers = new String[]{"xxx@qq.com"};
String[] receiversCc = new String[]{"xxx@qq.com"};
String[] receivers = new String[]{"347801120@qq.com"};
String[] receiversCc = new String[]{"347801120@qq.com"};
String content ="[\"id:69\"," +
"\"name:UserBehavior-0--1193959466\"," +
@ -114,7 +104,7 @@ public class MailUtilsTest {
@Test
public void testSendTableMail(){
String[] mails = new String[]{"825193156@qq.com"};
String[] mails = new String[]{"347801120@qq.com"};
Alert alert = new Alert();
alert.setTitle("Mysql Exception");
alert.setShowType(ShowType.TABLE);
@ -194,39 +184,4 @@ public class MailUtilsTest {
MailUtils.sendMails(Arrays.asList(mails),"gaojing",alert.getContent(),ShowType.TABLEATTACHMENT);
}
@Test
public void template(){
Template MAIL_TEMPLATE;
Configuration cfg = new Configuration(Configuration.VERSION_2_3_21);
cfg.setDefaultEncoding(Constants.UTF_8);
StringTemplateLoader stringTemplateLoader = new StringTemplateLoader();
cfg.setTemplateLoader(stringTemplateLoader);
InputStreamReader isr = null;
try {
isr = new InputStreamReader(new FileInputStream(ResourceUtils.getFile(Constants.CLASSPATH_MAIL_TEMPLATES_ALERT_MAIL_TEMPLATE_FTL)),
Constants.UTF_8);
MAIL_TEMPLATE = new Template("alert_mail_template", isr, cfg);
} catch (Exception e) {
MAIL_TEMPLATE = null;
} finally {
IOUtils.closeQuietly(isr);
}
StringWriter out = new StringWriter();
Map<String,String> map = new HashMap<>();
map.put(Constants.TITLE,"title_test");
try {
MAIL_TEMPLATE.process(map, out);
logger.info(out.toString());
} catch (TemplateException e) {
logger.error(e.getMessage(),e);
} catch (IOException e) {
logger.error(e.getMessage(),e);
}
}
}

105
dolphinscheduler-api/pom.xml

@ -27,17 +27,13 @@
<name>${project.artifactId}</name>
<packaging>jar</packaging>
<properties>
<jasper-runtime.version>5.5.23</jasper-runtime.version>
<servlet-api.version>2.5</servlet-api.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-alert</artifactId>
</dependency>
<dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-alert</artifactId>
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-server</artifactId>
<exclusions>
@ -96,25 +92,8 @@
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-aop</artifactId>
@ -125,33 +104,12 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
@ -181,19 +139,16 @@
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger2</artifactId>
<version>2.9.2</version>
</dependency>
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger-ui</artifactId>
<version>2.9.2</version>
</dependency>
<dependency>
<groupId>com.github.xiaoymin</groupId>
<artifactId>swagger-bootstrap-ui</artifactId>
<version>1.9.3</version>
</dependency>
<dependency>
@ -201,38 +156,37 @@
<artifactId>dolphinscheduler-rpc</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
</dependency>
<!-- hadoop -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
@ -241,13 +195,38 @@
<dependency>
<groupId>tomcat</groupId>
<artifactId>jasper-runtime</artifactId>
<version>${jasper-runtime.version}</version>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
<version>${servlet-api.version}</version>
</dependency>
<!-- just for test -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/ServiceModelToSwagger2MapperImpl.java

@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.api.configuration;
import com.google.common.collect.Multimap;
import io.swagger.models.*;
import io.swagger.models.parameters.Parameter;
import org.apache.commons.lang3.StringUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication;
import org.springframework.context.MessageSource;
@ -159,7 +159,8 @@ public class ServiceModelToSwagger2MapperImpl extends ServiceModelToSwagger2Mapp
Iterator<String> it = from.getTags().iterator();
while(it.hasNext()){
String tag = it.next();
list.add(StringUtils.isNotBlank(tag) ? messageSource.getMessage(tag, null, tag, locale) : " ");
list.add(
StringUtils.isNotBlank(tag) ? messageSource.getMessage(tag, null, tag, locale) : " ");
}
operation.setTags(list);

6
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/BaseController.java

@ -20,8 +20,8 @@ import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.commons.lang3.StringUtils;
import javax.servlet.http.HttpServletRequest;
import java.text.MessageFormat;
@ -67,7 +67,7 @@ public class BaseController {
public static String getClientIpAddress(HttpServletRequest request) {
String clientIp = request.getHeader(HTTP_X_FORWARDED_FOR);
if (StringUtils.isNotEmpty(clientIp) && !StringUtils.equalsIgnoreCase(HTTP_HEADER_UNKNOWN, clientIp)) {
if (StringUtils.isNotEmpty(clientIp) && !clientIp.equalsIgnoreCase(HTTP_HEADER_UNKNOWN)) {
int index = clientIp.indexOf(COMMA);
if (index != -1) {
return clientIp.substring(0, index);
@ -77,7 +77,7 @@ public class BaseController {
}
clientIp = request.getHeader(HTTP_X_REAL_IP);
if (StringUtils.isNotEmpty(clientIp) && !StringUtils.equalsIgnoreCase(HTTP_HEADER_UNKNOWN, clientIp)) {
if (StringUtils.isNotEmpty(clientIp) && !clientIp.equalsIgnoreCase(HTTP_HEADER_UNKNOWN)) {
return clientIp;
}

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataAnalysisController.java

@ -188,7 +188,7 @@ public class DataAnalysisController extends BaseController{
public Result countQueueState(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value="projectId", required=false, defaultValue = "0") int projectId){
try{
logger.info("count command state, user:{}, start date: {}, end date:{}, project id {}",
logger.info("count command state, user:{}, project id {}",
loginUser.getUserName(), projectId);
Map<String, Object> result = dataAnalysisService.countQueueState(loginUser, projectId);
return returnDataList(result);

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java

@ -461,7 +461,7 @@ public class DataSourceController extends BaseController {
@GetMapping(value="/kerberos-startup-state")
@ResponseStatus(HttpStatus.OK)
public Result getKerberosStartupState(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser){
logger.info("login user {},get kerberos startup state : {}", loginUser.getUserName());
logger.info("login user {}", loginUser.getUserName());
try{
// if upload resource is HDFS and kerberos startup is true , else false
return success(Status.SUCCESS.getMsg(), CommonUtils.getKerberosStartupState());

35
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoginController.java

@ -18,14 +18,14 @@ package org.apache.dolphinscheduler.api.controller;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.security.Authenticator;
import org.apache.dolphinscheduler.api.service.SessionService;
import org.apache.dolphinscheduler.api.service.UsersService;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.User;
import io.swagger.annotations.*;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@ -36,6 +36,8 @@ import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.util.Map;
import static org.apache.dolphinscheduler.api.enums.Status.*;
/**
@ -55,7 +57,7 @@ public class LoginController extends BaseController {
private SessionService sessionService;
@Autowired
private UsersService userService;
private Authenticator authenticator;
/**
@ -94,27 +96,20 @@ public class LoginController extends BaseController {
}
// verify username and password
User user = userService.queryUser(userName, userPassword);
if (user == null) {
return error(Status.USER_NAME_PASSWD_ERROR.getCode(),Status.USER_NAME_PASSWD_ERROR.getMsg()
);
}
// create session
String sessionId = sessionService.createSession(user, ip);
if (sessionId == null) {
return error(Status.LOGIN_SESSION_FAILED.getCode(),
Status.LOGIN_SESSION_FAILED.getMsg()
);
Result<Map<String, String>> result = authenticator.authenticate(userName, userPassword, ip);
if (result.getCode() != Status.SUCCESS.getCode()) {
return result;
}
response.setStatus(HttpStatus.SC_OK);
response.addCookie(new Cookie(Constants.SESSION_ID, sessionId));
Map<String, String> cookieMap = result.getData();
for (Map.Entry<String, String> cookieEntry : cookieMap.entrySet()) {
Cookie cookie = new Cookie(cookieEntry.getKey(), cookieEntry.getValue());
cookie.setHttpOnly(true);
response.addCookie(cookie);
}
logger.info("sessionId : {}" , sessionId);
return success(LOGIN_SUCCESS.getMsg(), sessionId);
return result;
} catch (Exception e) {
logger.error(USER_LOGIN_FAILURE.getMsg(),e);
return error(USER_LOGIN_FAILURE.getCode(), USER_LOGIN_FAILURE.getMsg());

6
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java

@ -16,12 +16,12 @@
*/
package org.apache.dolphinscheduler.api.controller;
import org.apache.commons.lang3.StringUtils;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.ProcessDefinitionService;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.User;
import io.swagger.annotations.*;
import org.slf4j.Logger;
@ -460,8 +460,8 @@ public class ProcessDefinitionController extends BaseController{
}
}
if(deleteFailedIdList.size() > 0){
putMsg(result, Status.BATCH_DELETE_PROCESS_DEFINE_BY_IDS_ERROR,StringUtils.join(deleteFailedIdList.toArray(),","));
if(!deleteFailedIdList.isEmpty()){
putMsg(result, Status.BATCH_DELETE_PROCESS_DEFINE_BY_IDS_ERROR,StringUtils.join(deleteFailedIdList,","));
}else{
putMsg(result, Status.SUCCESS);
}

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java

@ -16,7 +16,6 @@
*/
package org.apache.dolphinscheduler.api.controller;
import org.apache.commons.lang3.StringUtils;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.ProcessInstanceService;
import org.apache.dolphinscheduler.api.utils.Result;
@ -26,6 +25,7 @@ import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.queue.ITaskQueue;
import org.apache.dolphinscheduler.common.queue.TaskQueueFactory;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.User;
import io.swagger.annotations.*;
import org.slf4j.Logger;
@ -390,7 +390,7 @@ public class ProcessInstanceController extends BaseController{
}
}
if(deleteFailedIdList.size() > 0){
putMsg(result, Status.BATCH_DELETE_PROCESS_INSTANCE_BY_IDS_ERROR,StringUtils.join(deleteFailedIdList.toArray(),","));
putMsg(result, Status.BATCH_DELETE_PROCESS_INSTANCE_BY_IDS_ERROR,StringUtils.join(deleteFailedIdList,","));
}else{
putMsg(result, Status.SUCCESS);
}

9
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java

@ -267,11 +267,12 @@ public class ProjectController extends BaseController {
})
@PostMapping(value="/import-definition")
public Result importProcessDefinition(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam("file") MultipartFile file){
@RequestParam("file") MultipartFile file,
@RequestParam("projectName") String projectName){
try{
logger.info("import process definition by id, login user:{}",
loginUser.getUserName());
Map<String, Object> result = processDefinitionService.importProcessDefinition(loginUser,file);
logger.info("import process definition by id, login user:{}, project: {}",
loginUser.getUserName(), projectName);
Map<String, Object> result = processDefinitionService.importProcessDefinition(loginUser, file, projectName);
return returnDataList(result);
}catch (Exception e){
logger.error(IMPORT_PROCESS_DEFINE_ERROR.getMsg(),e);

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java

@ -137,7 +137,7 @@ public class TenantController extends BaseController{
@GetMapping(value="/list")
@ResponseStatus(HttpStatus.OK)
public Result queryTenantlist(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser){
logger.info("login user {}, query tenant list");
logger.info("login user {}, query tenant list", loginUser.getUserName());
try{
Map<String, Object> result = tenantService.queryTenantList(loginUser);
return returnDataList(result);

6
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java

@ -324,7 +324,7 @@ public class UsersController extends BaseController{
@GetMapping(value="/get-user-info")
@ResponseStatus(HttpStatus.OK)
public Result getUserInfo(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser){
logger.info("login user {},get user info : {}", loginUser.getUserName());
logger.info("login user {},get user info", loginUser.getUserName());
try{
Map<String, Object> result = usersService.getUserInfo(loginUser);
return returnDataList(result);
@ -344,7 +344,7 @@ public class UsersController extends BaseController{
@GetMapping(value="/list")
@ResponseStatus(HttpStatus.OK)
public Result listUser(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser){
logger.info("login user {}, user list");
logger.info("login user {}, user list", loginUser.getUserName());
try{
Map<String, Object> result = usersService.queryAllGeneralUsers(loginUser);
return returnDataList(result);
@ -364,7 +364,7 @@ public class UsersController extends BaseController{
@GetMapping(value="/list-all")
@ResponseStatus(HttpStatus.OK)
public Result listAll(@RequestAttribute(value = Constants.SESSION_USER) User loginUser){
logger.info("login user {}, user list");
logger.info("login user {}, user list", loginUser.getUserName());
try{
Map<String, Object> result = usersService.queryUserList(loginUser);
return returnDataList(result);

17
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java

@ -16,9 +16,9 @@
*/
package org.apache.dolphinscheduler.api.interceptor;
import org.apache.dolphinscheduler.api.security.Authenticator;
import org.apache.dolphinscheduler.api.service.SessionService;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.dao.entity.Session;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.commons.httpclient.HttpStatus;
@ -44,6 +44,9 @@ public class LoginHandlerInterceptor implements HandlerInterceptor {
@Autowired
private UserMapper userMapper;
@Autowired
private Authenticator authenticator;
/**
* Intercept the execution of a handler. Called after HandlerMapping determined
* an appropriate handler object, but before HandlerAdapter invokes the handler.
@ -68,17 +71,7 @@ public class LoginHandlerInterceptor implements HandlerInterceptor {
String token = request.getHeader("token");
User user = null;
if (StringUtils.isEmpty(token)){
Session session = sessionService.getSession(request);
if (session == null) {
response.setStatus(HttpStatus.SC_UNAUTHORIZED);
logger.info("session info is null ");
return false;
}
//get user object from session
user = userMapper.selectById(session.getUserId());
user = authenticator.getAuthUser(request);
// if user is null
if (user == null) {
response.setStatus(HttpStatus.SC_UNAUTHORIZED);

37
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/AuthenticationType.java

@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.security;
import com.baomidou.mybatisplus.annotation.EnumValue;
/**
* authentication type
*/
public enum AuthenticationType {
PASSWORD(0, "verify via user name and password"),
;
AuthenticationType(int code, String desc) {
this.code = code;
this.desc = desc;
}
@EnumValue
private final int code;
private final String desc;
}

40
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/Authenticator.java

@ -0,0 +1,40 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.security;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.dao.entity.User;
import javax.servlet.http.HttpServletRequest;
import java.util.Map;
public interface Authenticator {
/**
* Verifying legality via username and password
* @param username user name
* @param password user password
* @param extra extra info
* @return result object
*/
Result<Map<String, String>> authenticate(String username, String password, String extra);
/**
* Get authenticated user
* @param request http servlet request
* @return user
*/
User getAuthUser(HttpServletRequest request);
}

76
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/PasswordAuthenticator.java

@ -0,0 +1,76 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.security;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.SessionService;
import org.apache.dolphinscheduler.api.service.UsersService;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.dao.entity.Session;
import org.apache.dolphinscheduler.dao.entity.User;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import javax.servlet.http.HttpServletRequest;
import java.util.Collections;
import java.util.Map;
public class PasswordAuthenticator implements Authenticator {
private static final Logger logger = LoggerFactory.getLogger(PasswordAuthenticator.class);
@Autowired
private UsersService userService;
@Autowired
private SessionService sessionService;
@Override
public Result<Map<String, String>> authenticate(String username, String password, String extra) {
Result<Map<String, String>> result = new Result<>();
// verify username and password
User user = userService.queryUser(username, password);
if (user == null) {
result.setCode(Status.USER_NAME_PASSWD_ERROR.getCode());
result.setMsg(Status.USER_NAME_PASSWD_ERROR.getMsg());
return result;
}
// create session
String sessionId = sessionService.createSession(user, extra);
if (sessionId == null) {
result.setCode(Status.LOGIN_SESSION_FAILED.getCode());
result.setMsg(Status.LOGIN_SESSION_FAILED.getMsg());
return result;
}
logger.info("sessionId : {}" , sessionId);
result.setData(Collections.singletonMap(Constants.SESSION_ID, sessionId));
result.setCode(Status.SUCCESS.getCode());
result.setMsg(Status.LOGIN_SUCCESS.getMsg());
return result;
}
@Override
public User getAuthUser(HttpServletRequest request) {
Session session = sessionService.getSession(request);
if (session == null) {
logger.info("session info is null ");
return null;
}
//get user object from session
return userService.queryUser(session.getUserId());
}
}

67
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/SecurityConfig.java

@ -0,0 +1,67 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.security;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.config.AutowireCapableBeanFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class SecurityConfig {
private static final Logger logger = LoggerFactory.getLogger(SecurityConfig.class);
@Value("${security.authentication.type:PASSWORD}")
private String type;
private AutowireCapableBeanFactory beanFactory;
private AuthenticationType authenticationType;
@Autowired
public SecurityConfig(AutowireCapableBeanFactory beanFactory) {
this.beanFactory = beanFactory;
}
private void setAuthenticationType(String type) {
if (StringUtils.isBlank(type)) {
logger.info("security.authentication.type configuration is empty, the default value 'PASSWORD'");
this.authenticationType = AuthenticationType.PASSWORD;
return;
}
this.authenticationType = AuthenticationType.valueOf(type);
}
@Bean(name = "authenticator")
public Authenticator authenticator() {
setAuthenticationType(type);
Authenticator authenticator;
switch (authenticationType) {
case PASSWORD:
authenticator = new PasswordAuthenticator();
break;
default:
throw new IllegalStateException("Unexpected value: " + authenticationType);
}
beanFactory.autowireBean(authenticator);
return authenticator;
}
}

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java

@ -21,7 +21,7 @@ import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.AlertType;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.AlertGroup;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.entity.UserAlertGroup;
@ -29,7 +29,6 @@ import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper;
import org.apache.dolphinscheduler.dao.mapper.UserAlertGroupMapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java

@ -21,8 +21,8 @@ import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.commons.lang3.StringUtils;
import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServletRequest;
@ -104,7 +104,7 @@ public class BaseService {
Cookie[] cookies = request.getCookies();
if (cookies != null && cookies.length > 0) {
for (Cookie cookie : cookies) {
if (StringUtils.equalsIgnoreCase(name, cookie.getName())) {
if (StringUtils.isNotEmpty(name) && name.equalsIgnoreCase(cookie.getName())) {
return cookie;
}
}

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java

@ -27,8 +27,8 @@ import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.queue.ITaskQueue;
import org.apache.dolphinscheduler.common.queue.TaskQueueFactory;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.commons.lang3.StringUtils;
import org.apache.dolphinscheduler.dao.entity.*;
import org.apache.dolphinscheduler.dao.mapper.*;
import org.slf4j.Logger;

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java

@ -23,12 +23,12 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.*;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.*;
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java

@ -20,9 +20,9 @@ import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.log.LogClient;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;

497
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java

@ -16,9 +16,16 @@
*/
package org.apache.dolphinscheduler.api.service;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.dolphinscheduler.api.dto.treeview.Instance;
import org.apache.dolphinscheduler.api.dto.treeview.TreeViewDto;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.CheckUtils;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.*;
@ -31,15 +38,8 @@ import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.ProcessDao;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.dolphinscheduler.api.utils.CheckUtils;
import org.apache.dolphinscheduler.dao.entity.*;
import org.apache.dolphinscheduler.dao.mapper.*;
import org.slf4j.Logger;
@ -56,8 +56,10 @@ import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_SUB_PROCESS_DEFINE_ID;
@ -482,50 +484,21 @@ public class ProcessDefinitionService extends BaseDAGService {
* @param response response
*/
public void exportProcessDefinitionById(User loginUser, String projectName, Integer processDefinitionId, HttpServletResponse response) {
//export project info
Project project = projectMapper.queryByName(projectName);
//check user access for project
Map<String, Object> checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName);
Status resultStatus = (Status) checkResult.get(Constants.STATUS);
if (resultStatus == Status.SUCCESS) {
//get workflow info
ProcessDefinition processDefinition = processDefineMapper.queryByDefineId(processDefinitionId);
if (processDefinition != null) {
JSONObject jsonObject = JSONUtils.parseObject(processDefinition.getProcessDefinitionJson());
JSONArray jsonArray = (JSONArray) jsonObject.get("tasks");
for (int i = 0; i < jsonArray.size(); i++) {
JSONObject taskNode = jsonArray.getJSONObject(i);
if (taskNode.get("type") != null && taskNode.get("type") != "") {
String taskType = taskNode.getString("type");
if(taskType.equals(TaskType.SQL.name()) || taskType.equals(TaskType.PROCEDURE.name())){
JSONObject sqlParameters = JSONUtils.parseObject(taskNode.getString("params"));
DataSource dataSource = dataSourceMapper.selectById((Integer) sqlParameters.get("datasource"));
if (dataSource != null) {
sqlParameters.put("datasourceName", dataSource.getName());
}
taskNode.put("params", sqlParameters);
}else if(taskType.equals(TaskType.DEPENDENT.name())){
JSONObject dependentParameters = JSONUtils.parseObject(taskNode.getString("dependence"));
if(dependentParameters != null){
JSONArray dependTaskList = (JSONArray) dependentParameters.get("dependTaskList");
for (int j = 0; j < dependTaskList.size(); j++) {
JSONObject dependentTaskModel = dependTaskList.getJSONObject(j);
JSONArray dependItemList = (JSONArray) dependentTaskModel.get("dependItemList");
for (int k = 0; k < dependItemList.size(); k++) {
JSONObject dependentItem = dependItemList.getJSONObject(k);
int definitionId = dependentItem.getInteger("definitionId");
ProcessDefinition definition = processDefineMapper.queryByDefineId(definitionId);
if(definition != null){
dependentItem.put("projectName",definition.getProjectName());
dependentItem.put("definitionName",definition.getName());
}
}
}
taskNode.put("dependence", dependentParameters);
}
}
}
}
jsonObject.put("tasks", jsonArray);
processDefinition.setProcessDefinitionJson(jsonObject.toString());
if (null != processDefinition) {
//correct task param which has data source or dependent param
String correctProcessDefinitionJson = addTaskNodeSpecialParam(processDefinition.getProcessDefinitionJson());
processDefinition.setProcessDefinitionJson(correctProcessDefinitionJson);
Map<String, Object> row = new LinkedHashMap<>();
row.put("projectName", processDefinition.getProjectName());
@ -535,8 +508,9 @@ public class ProcessDefinitionService extends BaseDAGService {
row.put("processDefinitionLocations", processDefinition.getLocations());
row.put("processDefinitionConnects", processDefinition.getConnects());
//schedule info
List<Schedule> schedules = scheduleMapper.queryByProcessDefinitionId(processDefinitionId);
if (schedules.size() > 0) {
if (!schedules.isEmpty()) {
Schedule schedule = schedules.get(0);
row.put("scheduleWarningType", schedule.getWarningType());
row.put("scheduleWarningGroupId", schedule.getWarningGroupId());
@ -556,6 +530,8 @@ public class ProcessDefinitionService extends BaseDAGService {
}
}
//create workflow json file
String rowsJson = JSONUtils.toJsonString(row);
response.setContentType(MediaType.APPLICATION_JSON_UTF8_VALUE);
response.setHeader("Content-Disposition", "attachment;filename="+processDefinition.getName()+".json");
@ -564,38 +540,136 @@ public class ProcessDefinitionService extends BaseDAGService {
try {
out = response.getOutputStream();
buff = new BufferedOutputStream(out);
buff.write(rowsJson.getBytes("UTF-8"));
buff.write(rowsJson.getBytes(StandardCharsets.UTF_8));
buff.flush();
buff.close();
} catch (IOException e) {
e.printStackTrace();
logger.warn("export process fail", e);
}finally {
try {
buff.close();
out.close();
} catch (Exception e) {
e.printStackTrace();
if (null != buff) {
try {
buff.close();
} catch (Exception e) {
logger.warn("export process buffer not close", e);
}
}
if (null != out) {
try {
out.close();
} catch (Exception e) {
logger.warn("export process output stream not close", e);
}
}
}
}
}
}
/**
* correct task param which has datasource or dependent
* @param processDefinitionJson processDefinitionJson
* @return correct processDefinitionJson
*/
public String addTaskNodeSpecialParam(String processDefinitionJson) {
JSONObject jsonObject = JSONUtils.parseObject(processDefinitionJson);
JSONArray jsonArray = (JSONArray) jsonObject.get("tasks");
for (int i = 0; i < jsonArray.size(); i++) {
JSONObject taskNode = jsonArray.getJSONObject(i);
if (StringUtils.isNotEmpty(taskNode.getString("type"))) {
String taskType = taskNode.getString("type");
if(checkTaskHasDataSource(taskType)){
// add sqlParameters
JSONObject sqlParameters = JSONUtils.parseObject(taskNode.getString("params"));
DataSource dataSource = dataSourceMapper.selectById((Integer) sqlParameters.get("datasource"));
if (null != dataSource) {
sqlParameters.put("datasourceName", dataSource.getName());
}
taskNode.put("params", sqlParameters);
}else if(checkTaskHasDependent(taskType)){
// add dependent param
JSONObject dependentParameters = JSONUtils.parseObject(taskNode.getString("dependence"));
if(null != dependentParameters){
JSONArray dependTaskList = (JSONArray) dependentParameters.get("dependTaskList");
for (int j = 0; j < dependTaskList.size(); j++) {
JSONObject dependentTaskModel = dependTaskList.getJSONObject(j);
JSONArray dependItemList = (JSONArray) dependentTaskModel.get("dependItemList");
for (int k = 0; k < dependItemList.size(); k++) {
JSONObject dependentItem = dependItemList.getJSONObject(k);
int definitionId = dependentItem.getInteger("definitionId");
ProcessDefinition definition = processDefineMapper.queryByDefineId(definitionId);
if(null != definition){
dependentItem.put("projectName",definition.getProjectName());
dependentItem.put("definitionName",definition.getName());
}
}
}
taskNode.put("dependence", dependentParameters);
}
}
}
}
jsonObject.put("tasks", jsonArray);
return jsonObject.toString();
}
/**
* check task if has dependent
* @param taskType task type
* @return if task has dependent return true else false
*/
private boolean checkTaskHasDependent(String taskType) {
return taskType.equals(TaskType.DEPENDENT.name());
}
/**
* check task if has data source info
* @param taskType task type
* @return if task has data source return true else false
*/
private boolean checkTaskHasDataSource(String taskType) {
return taskType.equals(TaskType.SQL.name()) || taskType.equals(TaskType.PROCEDURE.name());
}
/**
* check task if has sub process
* @param taskType task type
* @return if task has sub process return true else false
*/
private boolean checkTaskHasSubProcess(String taskType) {
return taskType.equals(TaskType.SUB_PROCESS.name());
}
/**
* import process definition
* @param loginUser login user
* @param file process metadata json file
* @param currentProjectName current project name
* @return
*/
@Transactional(rollbackFor = Exception.class)
public Map<String, Object> importProcessDefinition(User loginUser, MultipartFile file) {
public Map<String, Object> importProcessDefinition(User loginUser, MultipartFile file, String currentProjectName) {
Map<String, Object> result = new HashMap<>(5);
JSONObject json = null;
try(InputStreamReader inputStreamReader = new InputStreamReader( file.getInputStream(), "UTF-8" )) {
JSONObject json;
//read workflow json
try(InputStreamReader inputStreamReader = new InputStreamReader( file.getInputStream(), StandardCharsets.UTF_8)) {
BufferedReader streamReader = new BufferedReader(inputStreamReader);
StringBuilder respomseStrBuilder = new StringBuilder();
String inputStr = "";
String inputStr;
while ((inputStr = streamReader.readLine())!= null){
respomseStrBuilder.append( inputStr );
}
json = JSONObject.parseObject( respomseStrBuilder.toString() );
if(json != null){
String projectName = null;
if(null != json){
String originProjectName = null;
String processDefinitionName = null;
String processDefinitionJson = null;
String processDefinitionDesc = null;
@ -613,151 +687,254 @@ public class ProcessDefinitionService extends BaseDAGService {
String scheduleWorkerGroupId = null;
String scheduleWorkerGroupName = null;
if (ObjectUtils.allNotNull(json.get("projectName"))) {
projectName = json.get("projectName").toString();
if (Objects.nonNull(json.get("projectName"))) {
originProjectName = json.get("projectName").toString();
} else {
putMsg(result, Status.DATA_IS_NULL, "processDefinitionName");
return result;
}
if (ObjectUtils.allNotNull(json.get("processDefinitionName"))) {
if (Objects.nonNull(json.get("processDefinitionName"))) {
processDefinitionName = json.get("processDefinitionName").toString();
} else {
putMsg(result, Status.DATA_IS_NULL, "processDefinitionName");
return result;
}
if (ObjectUtils.allNotNull(json.get("processDefinitionJson"))) {
if (Objects.nonNull(json.get("processDefinitionJson"))) {
processDefinitionJson = json.get("processDefinitionJson").toString();
} else {
putMsg(result, Status.DATA_IS_NULL, "processDefinitionJson");
return result;
}
if (ObjectUtils.allNotNull(json.get("processDefinitionDescription"))) {
if (Objects.nonNull(json.get("processDefinitionDescription"))) {
processDefinitionDesc = json.get("processDefinitionDescription").toString();
}
if (ObjectUtils.allNotNull(json.get("processDefinitionLocations"))) {
if (Objects.nonNull(json.get("processDefinitionLocations"))) {
processDefinitionLocations = json.get("processDefinitionLocations").toString();
}
if (ObjectUtils.allNotNull(json.get("processDefinitionConnects"))) {
if (Objects.nonNull(json.get("processDefinitionConnects"))) {
processDefinitionConnects = json.get("processDefinitionConnects").toString();
}
Project project = projectMapper.queryByName(projectName);
if(project != null){
processDefinitionName = recursionProcessDefinitionName(project.getId(), processDefinitionName, 1);
}
//check user access for org project
Project originProject = projectMapper.queryByName(originProjectName);
Map<String, Object> checkResult = projectService.checkProjectAndAuth(loginUser, originProject, originProjectName);
Status resultStatus = (Status) checkResult.get(Constants.STATUS);
JSONObject jsonObject = JSONUtils.parseObject(processDefinitionJson);
JSONArray jsonArray = (JSONArray) jsonObject.get("tasks");
for (int j = 0; j < jsonArray.size(); j++) {
JSONObject taskNode = jsonArray.getJSONObject(j);
String taskType = taskNode.getString("type");
if(taskType.equals(TaskType.SQL.name()) || taskType.equals(TaskType.PROCEDURE.name())) {
JSONObject sqlParameters = JSONUtils.parseObject(taskNode.getString("params"));
List<DataSource> dataSources = dataSourceMapper.queryDataSourceByName(sqlParameters.getString("datasourceName"));
if (dataSources.size() > 0) {
DataSource dataSource = dataSources.get(0);
sqlParameters.put("datasource", dataSource.getId());
}
taskNode.put("params", sqlParameters);
}else if(taskType.equals(TaskType.DEPENDENT.name())){
JSONObject dependentParameters = JSONUtils.parseObject(taskNode.getString("dependence"));
if(dependentParameters != null){
JSONArray dependTaskList = (JSONArray) dependentParameters.get("dependTaskList");
for (int h = 0; h < dependTaskList.size(); h++) {
JSONObject dependentTaskModel = dependTaskList.getJSONObject(h);
JSONArray dependItemList = (JSONArray) dependentTaskModel.get("dependItemList");
for (int k = 0; k < dependItemList.size(); k++) {
JSONObject dependentItem = dependItemList.getJSONObject(k);
Project dependentItemProject = projectMapper.queryByName(dependentItem.getString("projectName"));
if(dependentItemProject != null){
ProcessDefinition definition = processDefineMapper.queryByDefineName(dependentItemProject.getId(),dependentItem.getString("definitionName"));
if(definition != null){
dependentItem.put("projectId",dependentItemProject.getId());
dependentItem.put("definitionId",definition.getId());
if (resultStatus == Status.SUCCESS) {
//use currentProjectName to query
Project targetProject = projectMapper.queryByName(currentProjectName);
if(null != targetProject){
processDefinitionName = recursionProcessDefinitionName(targetProject.getId(), processDefinitionName, 1);
}
JSONObject jsonObject = JSONUtils.parseObject(processDefinitionJson);
JSONArray jsonArray = (JSONArray) jsonObject.get("tasks");
for (int j = 0; j < jsonArray.size(); j++) {
JSONObject taskNode = jsonArray.getJSONObject(j);
String taskType = taskNode.getString("type");
if(checkTaskHasDataSource(taskType)) {
JSONObject sqlParameters = JSONUtils.parseObject(taskNode.getString("params"));
List<DataSource> dataSources = dataSourceMapper.queryDataSourceByName(sqlParameters.getString("datasourceName"));
if (!dataSources.isEmpty()) {
DataSource dataSource = dataSources.get(0);
sqlParameters.put("datasource", dataSource.getId());
}
taskNode.put("params", sqlParameters);
}else if(checkTaskHasDependent(taskType)){
JSONObject dependentParameters = JSONUtils.parseObject(taskNode.getString("dependence"));
if(dependentParameters != null){
JSONArray dependTaskList = (JSONArray) dependentParameters.get("dependTaskList");
for (int h = 0; h < dependTaskList.size(); h++) {
JSONObject dependentTaskModel = dependTaskList.getJSONObject(h);
JSONArray dependItemList = (JSONArray) dependentTaskModel.get("dependItemList");
for (int k = 0; k < dependItemList.size(); k++) {
JSONObject dependentItem = dependItemList.getJSONObject(k);
Project dependentItemProject = projectMapper.queryByName(dependentItem.getString("projectName"));
if(dependentItemProject != null){
ProcessDefinition definition = processDefineMapper.queryByDefineName(dependentItemProject.getId(),dependentItem.getString("definitionName"));
if(definition != null){
dependentItem.put("projectId",dependentItemProject.getId());
dependentItem.put("definitionId",definition.getId());
}
}
}
}
taskNode.put("dependence", dependentParameters);
}
taskNode.put("dependence", dependentParameters);
}
}
}
jsonObject.put("tasks", jsonArray);
Map<String, Object> createProcessDefinitionResult = createProcessDefinition(loginUser,projectName,processDefinitionName,jsonObject.toString(),processDefinitionDesc,processDefinitionLocations,processDefinitionConnects);
Integer processDefinitionId = null;
if (ObjectUtils.allNotNull(createProcessDefinitionResult.get("processDefinitionId"))) {
processDefinitionId = Integer.parseInt(createProcessDefinitionResult.get("processDefinitionId").toString());
}
if (ObjectUtils.allNotNull(json.get("scheduleCrontab")) && processDefinitionId != null) {
Date now = new Date();
Schedule scheduleObj = new Schedule();
scheduleObj.setProjectName(projectName);
scheduleObj.setProcessDefinitionId(processDefinitionId);
scheduleObj.setProcessDefinitionName(processDefinitionName);
scheduleObj.setCreateTime(now);
scheduleObj.setUpdateTime(now);
scheduleObj.setUserId(loginUser.getId());
scheduleObj.setUserName(loginUser.getUserName());
scheduleCrontab = json.get("scheduleCrontab").toString();
scheduleObj.setCrontab(scheduleCrontab);
if (ObjectUtils.allNotNull(json.get("scheduleStartTime"))) {
scheduleStartTime = json.get("scheduleStartTime").toString();
scheduleObj.setStartTime(DateUtils.stringToDate(scheduleStartTime));
}
if (ObjectUtils.allNotNull(json.get("scheduleEndTime"))) {
scheduleEndTime = json.get("scheduleEndTime").toString();
scheduleObj.setEndTime(DateUtils.stringToDate(scheduleEndTime));
}
if (ObjectUtils.allNotNull(json.get("scheduleWarningType"))) {
scheduleWarningType = json.get("scheduleWarningType").toString();
scheduleObj.setWarningType(WarningType.valueOf(scheduleWarningType));
}
if (ObjectUtils.allNotNull(json.get("scheduleWarningGroupId"))) {
scheduleWarningGroupId = json.get("scheduleWarningGroupId").toString();
scheduleObj.setWarningGroupId(Integer.parseInt(scheduleWarningGroupId));
}
if (ObjectUtils.allNotNull(json.get("scheduleFailureStrategy"))) {
scheduleFailureStrategy = json.get("scheduleFailureStrategy").toString();
scheduleObj.setFailureStrategy(FailureStrategy.valueOf(scheduleFailureStrategy));
}
if (ObjectUtils.allNotNull(json.get("scheduleReleaseState"))) {
scheduleReleaseState = json.get("scheduleReleaseState").toString();
scheduleObj.setReleaseState(ReleaseState.valueOf(scheduleReleaseState));
//recursive sub-process parameter correction map key for old process id value for new process id
Map<Integer, Integer> subProcessIdMap = new HashMap<>(20);
List<Object> subProcessList = jsonArray.stream()
.filter(elem -> checkTaskHasSubProcess(JSONUtils.parseObject(elem.toString()).getString("type")))
.collect(Collectors.toList());
if (!subProcessList.isEmpty()) {
importSubProcess(loginUser, targetProject, jsonArray, subProcessIdMap);
}
if (ObjectUtils.allNotNull(json.get("scheduleProcessInstancePriority"))) {
scheduleProcessInstancePriority = json.get("scheduleProcessInstancePriority").toString();
scheduleObj.setProcessInstancePriority(Priority.valueOf(scheduleProcessInstancePriority));
jsonObject.put("tasks", jsonArray);
Map<String, Object> createProcessDefinitionResult = createProcessDefinition(loginUser,currentProjectName,processDefinitionName,jsonObject.toString(),processDefinitionDesc,processDefinitionLocations,processDefinitionConnects);
Integer processDefinitionId = null;
if (Objects.nonNull(createProcessDefinitionResult.get("processDefinitionId"))) {
processDefinitionId = Integer.parseInt(createProcessDefinitionResult.get("processDefinitionId").toString());
}
if (ObjectUtils.allNotNull(json.get("scheduleWorkerGroupId"))) {
scheduleWorkerGroupId = json.get("scheduleWorkerGroupId").toString();
if(scheduleWorkerGroupId != null){
scheduleObj.setWorkerGroupId(Integer.parseInt(scheduleWorkerGroupId));
}else{
if (ObjectUtils.allNotNull(json.get("scheduleWorkerGroupName"))) {
scheduleWorkerGroupName = json.get("scheduleWorkerGroupName").toString();
List<WorkerGroup> workerGroups = workerGroupMapper.queryWorkerGroupByName(scheduleWorkerGroupName);
if(workerGroups.size() > 0){
scheduleObj.setWorkerGroupId(workerGroups.get(0).getId());
if (Objects.nonNull(json.get("scheduleCrontab")) && processDefinitionId != null) {
Date now = new Date();
Schedule scheduleObj = new Schedule();
scheduleObj.setProjectName(currentProjectName);
scheduleObj.setProcessDefinitionId(processDefinitionId);
scheduleObj.setProcessDefinitionName(processDefinitionName);
scheduleObj.setCreateTime(now);
scheduleObj.setUpdateTime(now);
scheduleObj.setUserId(loginUser.getId());
scheduleObj.setUserName(loginUser.getUserName());
scheduleCrontab = json.get("scheduleCrontab").toString();
scheduleObj.setCrontab(scheduleCrontab);
if (Objects.nonNull(json.get("scheduleStartTime"))) {
scheduleStartTime = json.get("scheduleStartTime").toString();
scheduleObj.setStartTime(DateUtils.stringToDate(scheduleStartTime));
}
if (Objects.nonNull(json.get("scheduleEndTime"))) {
scheduleEndTime = json.get("scheduleEndTime").toString();
scheduleObj.setEndTime(DateUtils.stringToDate(scheduleEndTime));
}
if (Objects.nonNull(json.get("scheduleWarningType"))) {
scheduleWarningType = json.get("scheduleWarningType").toString();
scheduleObj.setWarningType(WarningType.valueOf(scheduleWarningType));
}
if (Objects.nonNull(json.get("scheduleWarningGroupId"))) {
scheduleWarningGroupId = json.get("scheduleWarningGroupId").toString();
scheduleObj.setWarningGroupId(Integer.parseInt(scheduleWarningGroupId));
}
if (Objects.nonNull(json.get("scheduleFailureStrategy"))) {
scheduleFailureStrategy = json.get("scheduleFailureStrategy").toString();
scheduleObj.setFailureStrategy(FailureStrategy.valueOf(scheduleFailureStrategy));
}
if (Objects.nonNull(json.get("scheduleReleaseState"))) {
scheduleReleaseState = json.get("scheduleReleaseState").toString();
scheduleObj.setReleaseState(ReleaseState.valueOf(scheduleReleaseState));
}
if (Objects.nonNull(json.get("scheduleProcessInstancePriority"))) {
scheduleProcessInstancePriority = json.get("scheduleProcessInstancePriority").toString();
scheduleObj.setProcessInstancePriority(Priority.valueOf(scheduleProcessInstancePriority));
}
if (Objects.nonNull(json.get("scheduleWorkerGroupId"))) {
scheduleWorkerGroupId = json.get("scheduleWorkerGroupId").toString();
if(scheduleWorkerGroupId != null){
scheduleObj.setWorkerGroupId(Integer.parseInt(scheduleWorkerGroupId));
}else{
if (Objects.nonNull(json.get("scheduleWorkerGroupName"))) {
scheduleWorkerGroupName = json.get("scheduleWorkerGroupName").toString();
List<WorkerGroup> workerGroups = workerGroupMapper.queryWorkerGroupByName(scheduleWorkerGroupName);
if(!workerGroups.isEmpty()){
scheduleObj.setWorkerGroupId(workerGroups.get(0).getId());
}
}
}
}
scheduleMapper.insert(scheduleObj);
}
scheduleMapper.insert(scheduleObj);
putMsg(result, Status.SUCCESS);
return result;
}
}else{
putMsg(result, Status.EXPORT_PROCESS_DEFINE_BY_ID_ERROR);
putMsg(result, Status.IMPORT_PROCESS_DEFINE_ERROR);
return result;
}
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
}
putMsg(result, Status.SUCCESS);
return result;
}
/**
* check import process has sub process
* recursion create sub process
* @param loginUser login user
* @param targetProject target project
*/
public void importSubProcess(User loginUser, Project targetProject, JSONArray jsonArray, Map<Integer, Integer> subProcessIdMap) {
for (int i = 0; i < jsonArray.size(); i++) {
JSONObject taskNode = jsonArray.getJSONObject(i);
String taskType = taskNode.getString("type");
if (checkTaskHasSubProcess(taskType)) {
//get sub process info
JSONObject subParams = JSONUtils.parseObject(taskNode.getString("params"));
Integer subProcessId = subParams.getInteger("processDefinitionId");
ProcessDefinition subProcess = processDefineMapper.queryByDefineId(subProcessId);
String subProcessJson = subProcess.getProcessDefinitionJson();
//check current project has sub process
ProcessDefinition currentProjectSubProcess = processDefineMapper.queryByDefineName(targetProject.getId(), subProcess.getName());
if (null == currentProjectSubProcess) {
JSONArray subJsonArray = (JSONArray) JSONUtils.parseObject(subProcess.getProcessDefinitionJson()).get("tasks");
List<Object> subProcessList = subJsonArray.stream()
.filter(item -> checkTaskHasSubProcess(JSONUtils.parseObject(item.toString()).getString("type")))
.collect(Collectors.toList());
if (!subProcessList.isEmpty()) {
importSubProcess(loginUser, targetProject, subJsonArray, subProcessIdMap);
//sub process processId correct
if (!subProcessIdMap.isEmpty()) {
for (Map.Entry<Integer, Integer> entry : subProcessIdMap.entrySet()) {
String oldSubProcessId = "\"processDefinitionId\":" + entry.getKey();
String newSubProcessId = "\"processDefinitionId\":" + entry.getValue();
subProcessJson = subProcessJson.replaceAll(oldSubProcessId, newSubProcessId);
}
subProcessIdMap.clear();
}
}
//if sub-process recursion
Date now = new Date();
//create sub process in target project
ProcessDefinition processDefine = new ProcessDefinition();
processDefine.setName(subProcess.getName());
processDefine.setVersion(subProcess.getVersion());
processDefine.setReleaseState(subProcess.getReleaseState());
processDefine.setProjectId(targetProject.getId());
processDefine.setUserId(loginUser.getId());
processDefine.setProcessDefinitionJson(subProcessJson);
processDefine.setDescription(subProcess.getDescription());
processDefine.setLocations(subProcess.getLocations());
processDefine.setConnects(subProcess.getConnects());
processDefine.setTimeout(subProcess.getTimeout());
processDefine.setTenantId(subProcess.getTenantId());
processDefine.setGlobalParams(subProcess.getGlobalParams());
processDefine.setCreateTime(now);
processDefine.setUpdateTime(now);
processDefine.setFlag(subProcess.getFlag());
processDefine.setReceivers(subProcess.getReceivers());
processDefine.setReceiversCc(subProcess.getReceiversCc());
processDefineMapper.insert(processDefine);
logger.info("create sub process, project: {}, process name: {}", targetProject.getName(), processDefine.getName());
//modify task node
ProcessDefinition newSubProcessDefine = processDefineMapper.queryByDefineName(processDefine.getProjectId(),processDefine.getName());
if (null != newSubProcessDefine) {
subProcessIdMap.put(subProcessId, newSubProcessDefine.getId());
subParams.put("processDefinitionId", newSubProcessDefine.getId());
taskNode.put("params", subParams);
}
}
}
}
}
/**

19
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java

@ -37,7 +37,6 @@ import org.apache.dolphinscheduler.dao.ProcessDao;
import com.alibaba.fastjson.JSON;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.commons.lang3.StringUtils;
import org.apache.dolphinscheduler.dao.entity.*;
import org.apache.dolphinscheduler.dao.mapper.*;
import org.slf4j.Logger;
@ -195,12 +194,11 @@ public class ProcessInstanceService extends BaseDAGService {
processInstance.setDuration(DateUtils.differSec(processInstance.getStartTime(),processInstance.getEndTime()));
}
Set<String> exclusionSet = new HashSet<String>(){{
add(Constants.CLASS);
add("locations");
add("connects");
add("processInstanceJson");
}};
Set<String> exclusionSet = new HashSet<String>();
exclusionSet.add(Constants.CLASS);
exclusionSet.add("locations");
exclusionSet.add("connects");
exclusionSet.add("processInstanceJson");
PageInfo pageInfo = new PageInfo<ProcessInstance>(pageNo, pageSize);
pageInfo.setTotalCount((int) processInstanceList.getTotal());
@ -490,13 +488,14 @@ public class ProcessInstanceService extends BaseDAGService {
}
ProcessInstance processInstance = processDao.findProcessInstanceDetailById(processInstanceId);
List<TaskInstance> taskInstanceList = processDao.findValidTaskListByProcessId(processInstanceId);
//process instance priority
int processInstancePriority = processInstance.getProcessInstancePriority().ordinal();
if (processInstance == null) {
if (null == processInstance) {
putMsg(result, Status.PROCESS_INSTANCE_NOT_EXIST, processInstanceId);
return result;
}
//process instance priority
int processInstancePriority = processInstance.getProcessInstancePriority().ordinal();
// delete zk queue
if (CollectionUtils.isNotEmpty(taskInstanceList)){
for (TaskInstance taskInstance : taskInstanceList){

37
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java

@ -16,12 +16,15 @@
*/
package org.apache.dolphinscheduler.api.service;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.commons.collections.BeanMap;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
@ -29,10 +32,6 @@ import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.apache.dolphinscheduler.dao.entity.User;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.commons.collections.BeanMap;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.dao.mapper.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -113,14 +112,12 @@ public class ResourcesService extends BaseService {
putMsg(result, Status.RESOURCE_SUFFIX_FORBID_CHANGE);
return result;
}
//
//If resource type is UDF, only jar packages are allowed to be uploaded, and the suffix must be .jar
if (Constants.UDF.equals(type.name())) {
if (!JAR.equalsIgnoreCase(fileSuffix)) {
logger.error(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg());
putMsg(result, Status.UDF_RESOURCE_SUFFIX_NOT_JAR);
return result;
}
if (Constants.UDF.equals(type.name()) && !JAR.equalsIgnoreCase(fileSuffix)) {
logger.error(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg());
putMsg(result, Status.UDF_RESOURCE_SUFFIX_NOT_JAR);
return result;
}
if (file.getSize() > Constants.maxFileSize) {
logger.error("file size is too large: {}", file.getOriginalFilename());
@ -226,12 +223,16 @@ public class ResourcesService extends BaseService {
}
//check resource aleady exists
if (!resource.getAlias().equals(name)) {
if (checkResourceExists(name, 0, type.ordinal())) {
logger.error("resource {} already exists, can't recreate", name);
putMsg(result, Status.RESOURCE_EXIST);
return result;
}
if (!resource.getAlias().equals(name) && checkResourceExists(name, 0, type.ordinal())) {
logger.error("resource {} already exists, can't recreate", name);
putMsg(result, Status.RESOURCE_EXIST);
return result;
}
// query tenant by user id
String tenantCode = getTenantCode(resource.getUserId(),result);
if (StringUtils.isEmpty(tenantCode)){
return result;
}
// query tenant by user id

6
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java

@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.common.enums.*;
import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.Project;
@ -38,7 +39,6 @@ import org.apache.dolphinscheduler.server.quartz.ProcessScheduleJob;
import org.apache.dolphinscheduler.server.quartz.QuartzExecutors;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.commons.lang3.StringUtils;
import org.quartz.CronExpression;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -374,12 +374,12 @@ public class SchedulerService extends BaseService {
try {
switch (scheduleStatus) {
case ONLINE: {
logger.info("Call master client set schedule online, project id: {}, flow id: {},host: {}, port: {}", project.getId(), processDefinition.getId(), masterServers);
logger.info("Call master client set schedule online, project id: {}, flow id: {},host: {}", project.getId(), processDefinition.getId(), masterServers);
setSchedule(project.getId(), id);
break;
}
case OFFLINE: {
logger.info("Call master client set schedule offline, project id: {}, flow id: {},host: {}, port: {}", project.getId(), processDefinition.getId(), masterServers);
logger.info("Call master client set schedule offline, project id: {}, flow id: {},host: {}", project.getId(), processDefinition.getId(), masterServers);
deleteSchedule(project.getId(), id);
break;
}

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SessionService.java

@ -139,7 +139,6 @@ public class SessionService extends BaseService{
* @param loginUser login user
*/
public void signOut(String ip, User loginUser) {
try {
/**
* query session by user id and ip
*/
@ -147,8 +146,5 @@ public class SessionService extends BaseService{
//delete session
sessionMapper.deleteById(session.getId());
}catch (Exception e){
}
}
}

9
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java

@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.ProcessDao;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.Project;
@ -32,7 +33,6 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@ -116,10 +116,9 @@ public class TaskInstanceService extends BaseService {
page, project.getId(), processInstanceId, searchVal, taskName, statusArray, host, start, end
);
PageInfo pageInfo = new PageInfo<ProcessInstance>(pageNo, pageSize);
Set<String> exclusionSet = new HashSet<String>(){{
add(Constants.CLASS);
add("taskJson");
}};
Set<String> exclusionSet = new HashSet<>();
exclusionSet.add(Constants.CLASS);
exclusionSet.add("taskJson");
List<TaskInstance> taskInstanceList = taskInstanceIPage.getRecords();
for(TaskInstance taskInstance : taskInstanceList){
taskInstance.setDuration(DateUtils.differSec(taskInstance.getStartTime(),

22
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java

@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.api.service;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.commons.lang3.StringUtils;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
@ -26,6 +25,7 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.Tenant;
@ -34,7 +34,6 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.hadoop.fs.FileStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@ -302,6 +301,25 @@ public class TenantService extends BaseService{
return result;
}
/**
* query tenant list via tenant code
* @param tenantCode tenant code
* @return tenant list
*/
public Map<String, Object> queryTenantList(String tenantCode) {
Map<String, Object> result = new HashMap<>(5);
List<Tenant> resourceList = tenantMapper.queryByTenantCode(tenantCode);
if (resourceList != null && resourceList.size() > 0) {
result.put(Constants.DATA_LIST, resourceList);
putMsg(result, Status.SUCCESS);
} else {
putMsg(result, Status.TENANT_NOT_EXIST);
}
return result;
}
/**
* verify tenant code
*

57
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java

@ -29,7 +29,7 @@ import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.commons.lang3.StringUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.*;
import org.apache.dolphinscheduler.dao.mapper.*;
import org.slf4j.Logger;
@ -114,6 +114,31 @@ public class UsersService extends BaseService {
return result;
}
User user = createUser(userName, userPassword, email, tenantId, phone, queue);
Tenant tenant = tenantMapper.queryById(tenantId);
// resource upload startup
if (PropertyUtils.getResUploadStartupState()){
// if tenant not exists
if (!HadoopUtils.getInstance().exists(HadoopUtils.getHdfsTenantDir(tenant.getTenantCode()))){
createTenantDirIfNotExists(tenant.getTenantCode());
}
String userPath = HadoopUtils.getHdfsUserDir(tenant.getTenantCode(),user.getId());
HadoopUtils.getInstance().mkdir(userPath);
}
putMsg(result, Status.SUCCESS);
return result;
}
@Transactional(rollbackFor = Exception.class)
public User createUser(String userName,
String userPassword,
String email,
int tenantId,
String phone,
String queue) throws Exception {
User user = new User();
Date now = new Date();
@ -133,21 +158,25 @@ public class UsersService extends BaseService {
// save user
userMapper.insert(user);
return user;
}
Tenant tenant = tenantMapper.queryById(tenantId);
// resource upload startup
if (PropertyUtils.getResUploadStartupState()){
// if tenant not exists
if (!HadoopUtils.getInstance().exists(HadoopUtils.getHdfsTenantDir(tenant.getTenantCode()))){
createTenantDirIfNotExists(tenant.getTenantCode());
}
String userPath = HadoopUtils.getHdfsUserDir(tenant.getTenantCode(),user.getId());
HadoopUtils.getInstance().mkdir(userPath);
}
putMsg(result, Status.SUCCESS);
return result;
/**
* query user by id
* @param id id
* @return user info
*/
public User queryUser(int id) {
return userMapper.selectById(id);
}
/**
* query user
* @param name name
* @return user info
*/
public User queryUser(String name) {
return userMapper.queryByUserNameAccurately(name);
}
/**

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java

@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
@ -27,7 +28,6 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java

@ -21,8 +21,8 @@ import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.common.utils.TaskParametersUtils;
import org.apache.commons.lang.StringUtils;
import java.text.MessageFormat;
import java.util.HashMap;
@ -148,7 +148,7 @@ public class CheckUtils {
* @return true if regex pattern is right, otherwise return false
*/
private static boolean regexChecks(String str, Pattern pattern) {
if (org.apache.commons.lang3.StringUtils.isEmpty(str)) {
if (StringUtils.isEmpty(str)) {
return false;
}

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java

@ -16,7 +16,7 @@
*/
package org.apache.dolphinscheduler.api.utils;
import org.apache.commons.lang3.StringUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZookeeperMonitor.java

@ -17,10 +17,10 @@
package org.apache.dolphinscheduler.api.utils;
import org.apache.dolphinscheduler.common.enums.ZKNodeType;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.common.zk.AbstractZKClient;
import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.dao.entity.ZookeeperRecord;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;

5
dolphinscheduler-api/src/main/resources/application-api.properties

@ -37,4 +37,9 @@ spring.messages.encoding=UTF-8
#i18n classpath folder , file prefix messages, if have many files, use "," seperator
spring.messages.basename=i18n/messages
# Authentication types (supported types: PASSWORD)
security.authentication.type=PASSWORD

4
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AbstractControllerTest.java

@ -19,8 +19,8 @@ package org.apache.dolphinscheduler.api.controller;
import org.apache.dolphinscheduler.api.ApiApplicationServer;
import org.apache.dolphinscheduler.api.service.SessionService;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.commons.lang3.StringUtils;
import org.junit.*;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
@ -32,8 +32,6 @@ import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.web.context.WebApplicationContext;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
@Ignore
@RunWith(SpringRunner.class)

71
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptorTest.java

@ -0,0 +1,71 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.interceptor;
import org.apache.dolphinscheduler.api.ApiApplicationServer;
import org.apache.dolphinscheduler.api.security.Authenticator;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.context.junit4.SpringRunner;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import static org.mockito.Mockito.when;
@RunWith(SpringRunner.class)
@SpringBootTest(classes = ApiApplicationServer.class)
public class LoginHandlerInterceptorTest {
private static final Logger logger = LoggerFactory.getLogger(LoginHandlerInterceptorTest.class);
@Autowired
LoginHandlerInterceptor interceptor;
@MockBean
private Authenticator authenticator;
@MockBean
private UserMapper userMapper;
@Test
public void testPreHandle() {
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
// test no token and no cookie
Assert.assertFalse(interceptor.preHandle(request, response, null));
User mockUser = new User();
mockUser.setId(1);
mockUser.setUserType(UserType.GENERAL_USER);
// test no token
when(authenticator.getAuthUser(request)).thenReturn(mockUser);
Assert.assertTrue(interceptor.preHandle(request, response, null));
// test token
String token = "123456";
when(request.getHeader("token")).thenReturn(token);
when(userMapper.queryUserByToken(token)).thenReturn(mockUser);
Assert.assertTrue(interceptor.preHandle(request, response, null));
}
}

96
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/security/PasswordAuthenticatorTest.java

@ -0,0 +1,96 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.security;
import org.apache.dolphinscheduler.api.ApiApplicationServer;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.SessionService;
import org.apache.dolphinscheduler.api.service.UsersService;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.dao.entity.Session;
import org.apache.dolphinscheduler.dao.entity.User;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import static org.mockito.Mockito.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.config.AutowireCapableBeanFactory;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.context.junit4.SpringRunner;
import javax.servlet.http.HttpServletRequest;
import java.util.Date;
import java.util.UUID;
@RunWith(SpringRunner.class)
@SpringBootTest(classes = ApiApplicationServer.class)
public class PasswordAuthenticatorTest {
private static Logger logger = LoggerFactory.getLogger(PasswordAuthenticatorTest.class);
@Autowired
private AutowireCapableBeanFactory beanFactory;
@MockBean
private SessionService sessionService;
@MockBean
private UsersService usersService;
private PasswordAuthenticator authenticator;
private User mockUser;
private Session mockSession;
@Before
public void setUp() throws Exception {
authenticator = new PasswordAuthenticator();
beanFactory.autowireBean(authenticator);
mockUser = new User();
mockUser.setUserName("test");
mockUser.setEmail("test@test.com");
mockUser.setUserPassword("test");
mockUser.setId(1);
mockSession = new Session();
mockSession.setId(UUID.randomUUID().toString());
mockSession.setIp("127.0.0.1");
mockSession.setUserId(1);
mockSession.setLastLoginTime(new Date());
}
@Test
public void testAuthenticate() {
when(usersService.queryUser("test", "test")).thenReturn(mockUser);
when(sessionService.createSession(mockUser, "127.0.0.1")).thenReturn(mockSession.getId());
Result result = authenticator.authenticate("test", "test", "127.0.0.1");
Assert.assertEquals(Status.SUCCESS.getCode(), (int) result.getCode());
logger.info(result.toString());
}
@Test
public void testGetAuthUser() {
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
when(usersService.queryUser(mockUser.getId())).thenReturn(mockUser);
when(sessionService.getSession(request)).thenReturn(mockSession);
User user = authenticator.getAuthUser(request);
Assert.assertNotNull(user);
}
}

43
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/security/SecurityConfigTest.java

@ -0,0 +1,43 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.security;
import org.apache.dolphinscheduler.api.ApiApplicationServer;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit4.SpringRunner;
@RunWith(SpringRunner.class)
@SpringBootTest(classes = ApiApplicationServer.class)
@TestPropertySource(properties = {
"security.authentication.type=PASSWORD",
})
public class SecurityConfigTest {
@Autowired
private SecurityConfig securityConfig;
@Test
public void testAuthenticator() {
Authenticator authenticator = securityConfig.authenticator();
Assert.assertNotNull(authenticator);
}
}

9
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java

@ -18,11 +18,12 @@ package org.apache.dolphinscheduler.api.service;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.commons.lang3.time.DateUtils;
import java.util.Calendar;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.dao.entity.AccessToken;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.AccessTokenMapper;
@ -152,7 +153,7 @@ public class AccessTokenServiceTest {
accessToken.setId(1);
accessToken.setUserId(1);
accessToken.setToken("AccessTokenServiceTest");
Date date = DateUtils.addDays(new Date(),30);
Date date = DateUtils.add(new Date(),Calendar.DAY_OF_MONTH, 30);
accessToken.setExpireTime(date);
return accessToken;
}
@ -175,7 +176,7 @@ public class AccessTokenServiceTest {
* @return
*/
private String getDate(){
Date date = DateUtils.addDays(new Date(),30);
return org.apache.dolphinscheduler.common.utils.DateUtils.dateToString(date);
Date date = DateUtils.add(new Date(), Calendar.DAY_OF_MONTH, 30);
return DateUtils.dateToString(date);
}
}

294
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java

@ -16,38 +16,80 @@
*/
package org.apache.dolphinscheduler.api.service;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.apache.dolphinscheduler.api.ApiApplicationServer;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.User;
import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.http.entity.ContentType;
import org.json.JSONException;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.skyscreamer.jsonassert.JSONAssert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.web.multipart.MultipartFile;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.text.MessageFormat;
import java.util.HashMap;
import java.util.Map;
@RunWith(SpringRunner.class)
@RunWith(MockitoJUnitRunner.Silent.class)
@SpringBootTest(classes = ApiApplicationServer.class)
public class ProcessDefinitionServiceTest {
private static final Logger logger = LoggerFactory.getLogger(ProcessDefinitionServiceTest.class);
@Autowired
@InjectMocks
ProcessDefinitionService processDefinitionService;
@Mock
private DataSourceMapper dataSourceMapper;
@Mock
private ProcessDefinitionMapper processDefineMapper;
@Mock
private ProjectMapper projectMapper;
@Mock
private ProjectService projectService;
@Test
public void queryProccessDefinitionList() throws Exception {
String projectName = "project_test1";
Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName));
Project project = getProject(projectName);
User loginUser = new User();
loginUser.setId(-1);
loginUser.setUserType(UserType.GENERAL_USER);
Map<String, Object> result = new HashMap<>(5);
putMsg(result, Status.PROJECT_NOT_FOUNT, projectName);
Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result);
Map<String, Object> map = processDefinitionService.queryProccessDefinitionList(loginUser,"project_test1");
Assert.assertEquals(Status.PROJECT_NOT_FOUNT, map.get(Constants.STATUS));
logger.info(JSON.toJSONString(map));
@ -55,10 +97,20 @@ public class ProcessDefinitionServiceTest {
@Test
public void queryProcessDefinitionListPagingTest() throws Exception {
String projectName = "project_test1";
Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName));
Project project = getProject(projectName);
User loginUser = new User();
loginUser.setId(-1);
loginUser.setUserType(UserType.GENERAL_USER);
Map<String, Object> result = new HashMap<>(5);
putMsg(result, Status.PROJECT_NOT_FOUNT, projectName);
Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result);
Map<String, Object> map = processDefinitionService.queryProcessDefinitionListPaging(loginUser, "project_test1", "",1, 5,0);
Assert.assertEquals(Status.PROJECT_NOT_FOUNT, map.get(Constants.STATUS));
@ -67,13 +119,243 @@ public class ProcessDefinitionServiceTest {
@Test
public void deleteProcessDefinitionByIdTest() throws Exception {
String projectName = "project_test1";
Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName));
Project project = getProject(projectName);
User loginUser = new User();
loginUser.setId(-1);
loginUser.setUserType(UserType.GENERAL_USER);
Map<String, Object> map = processDefinitionService.deleteProcessDefinitionById(loginUser, "li_sql_test", 6);
Map<String, Object> result = new HashMap<>(5);
putMsg(result, Status.PROJECT_NOT_FOUNT, projectName);
Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result);
Map<String, Object> map = processDefinitionService.deleteProcessDefinitionById(loginUser, "project_test1", 6);
Assert.assertEquals(Status.PROJECT_NOT_FOUNT, map.get(Constants.STATUS));
logger.info(JSON.toJSONString(map));
}
/**
* add datasource param and dependent when export process
* @throws JSONException
*/
@Test
public void testAddTaskNodeSpecialParam() throws JSONException {
Mockito.when(dataSourceMapper.selectById(1)).thenReturn(getDataSource());
Mockito.when(processDefineMapper.queryByDefineId(2)).thenReturn(getProcessDefinition());
String sqlDependentJson = "{\"globalParams\":[]," +
"\"tasks\":[{\"type\":\"SQL\",\"id\":\"tasks-27297\",\"name\":\"sql\"," +
"\"params\":{\"type\":\"MYSQL\",\"datasource\":1,\"sql\":\"select * from test\"," +
"\"udfs\":\"\",\"sqlType\":\"1\",\"title\":\"\",\"receivers\":\"\",\"receiversCc\":\"\",\"showType\":\"TABLE\"" +
",\"localParams\":[],\"connParams\":\"\"," +
"\"preStatements\":[],\"postStatements\":[]}," +
"\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\"," +
"\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\"," +
"\"enable\":false},\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1," +
"\"preTasks\":[\"dependent\"]},{\"type\":\"DEPENDENT\",\"id\":\"tasks-33787\"," +
"\"name\":\"dependent\",\"params\":{},\"description\":\"\",\"runFlag\":\"NORMAL\"," +
"\"dependence\":{\"relation\":\"AND\",\"dependTaskList\":[{\"relation\":\"AND\"," +
"\"dependItemList\":[{\"projectId\":2,\"definitionId\":46,\"depTasks\":\"ALL\"," +
"\"cycle\":\"day\",\"dateValue\":\"today\"}]}]},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," +
"\"timeout\":{\"strategy\":\"\",\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," +
"\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}";
String corSqlDependentJson = processDefinitionService.addTaskNodeSpecialParam(sqlDependentJson);
JSONAssert.assertEquals(sqlDependentJson,corSqlDependentJson,false);
}
/**
* import sub process test
*/
@Test
public void testImportSubProcess() {
User loginUser = new User();
loginUser.setId(1);
loginUser.setUserType(UserType.ADMIN_USER);
Project testProject = getProject("test");
//Recursive subprocess sub2 process in sub1 process and sub1process in top process
String topProcessJson = "{\"globalParams\":[]," +
"\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-38634\",\"name\":\"shell1\"," +
"\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"#!/bin/bash\\necho \\\"shell-1\\\"\"}," +
"\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\"," +
"\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false}," +
"\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]}," +
"{\"type\":\"SUB_PROCESS\",\"id\":\"tasks-44207\",\"name\":\"shell-4\"," +
"\"params\":{\"processDefinitionId\":39},\"description\":\"\",\"runFlag\":\"NORMAL\"," +
"\"dependence\":{},\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false}," +
"\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1," +
"\"preTasks\":[\"shell1\"]}],\"tenantId\":1,\"timeout\":0}";
String sub1ProcessJson = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-84090\"," +
"\"name\":\"shell-4\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"#!/bin/bash\\necho \\\"shell-4\\\"\"}," +
"\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\"," +
"\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false}," +
"\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]},{\"type\":\"SUB_PROCESS\"," +
"\"id\":\"tasks-87364\",\"name\":\"shell-5\"," +
"\"params\":{\"processDefinitionId\":46},\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{}," +
"\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," +
"\"workerGroupId\":-1,\"preTasks\":[\"shell-4\"]}],\"tenantId\":1,\"timeout\":0}";
String sub2ProcessJson = "{\"globalParams\":[]," +
"\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-52423\",\"name\":\"shell-5\"," +
"\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo \\\"shell-5\\\"\"},\"description\":\"\"," +
"\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," +
"\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1," +
"\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}";
JSONObject jsonObject = JSONUtils.parseObject(topProcessJson);
JSONArray jsonArray = (JSONArray) jsonObject.get("tasks");
String originSubJson = jsonArray.toString();
Map<Integer, Integer> subProcessIdMap = new HashMap<>(20);
ProcessDefinition shellDefinition1 = new ProcessDefinition();
shellDefinition1.setId(39);
shellDefinition1.setName("shell-4");
shellDefinition1.setProjectId(2);
shellDefinition1.setProcessDefinitionJson(sub1ProcessJson);
ProcessDefinition shellDefinition2 = new ProcessDefinition();
shellDefinition2.setId(46);
shellDefinition2.setName("shell-5");
shellDefinition2.setProjectId(2);
shellDefinition2.setProcessDefinitionJson(sub2ProcessJson);
Mockito.when(processDefineMapper.queryByDefineId(39)).thenReturn(shellDefinition1);
Mockito.when(processDefineMapper.queryByDefineId(46)).thenReturn(shellDefinition2);
Mockito.when(processDefineMapper.queryByDefineName(testProject.getId(), "shell-5")).thenReturn(null);
Mockito.when(processDefineMapper.queryByDefineName(testProject.getId(), "shell-4")).thenReturn(null);
Mockito.when(processDefineMapper.queryByDefineName(testProject.getId(), "testProject")).thenReturn(shellDefinition2);
processDefinitionService.importSubProcess(loginUser,testProject,jsonArray,subProcessIdMap);
String correctSubJson = jsonArray.toString();
Assert.assertEquals(originSubJson, correctSubJson);
}
@Test
public void testImportProcessDefinitionById() throws IOException {
String processJson = "{\"projectName\":\"testProject\",\"processDefinitionName\":\"shell-4\"," +
"\"processDefinitionJson\":\"{\\\"tenantId\\\":1,\\\"globalParams\\\":[]," +
"\\\"tasks\\\":[{\\\"workerGroupId\\\":-1,\\\"description\\\":\\\"\\\",\\\"runFlag\\\":\\\"NORMAL\\\"," +
"\\\"type\\\":\\\"SHELL\\\",\\\"params\\\":{\\\"rawScript\\\":\\\"#!/bin/bash\\\\necho \\\\\\\"shell-4\\\\\\\"\\\"," +
"\\\"localParams\\\":[],\\\"resourceList\\\":[]},\\\"timeout\\\":{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}," +
"\\\"maxRetryTimes\\\":\\\"0\\\",\\\"taskInstancePriority\\\":\\\"MEDIUM\\\",\\\"name\\\":\\\"shell-4\\\"," +
"\\\"dependence\\\":{},\\\"retryInterval\\\":\\\"1\\\",\\\"preTasks\\\":[],\\\"id\\\":\\\"tasks-84090\\\"}," +
"{\\\"taskInstancePriority\\\":\\\"MEDIUM\\\",\\\"name\\\":\\\"shell-5\\\",\\\"workerGroupId\\\":-1," +
"\\\"description\\\":\\\"\\\",\\\"dependence\\\":{},\\\"preTasks\\\":[\\\"shell-4\\\"],\\\"id\\\":\\\"tasks-87364\\\"," +
"\\\"runFlag\\\":\\\"NORMAL\\\",\\\"type\\\":\\\"SUB_PROCESS\\\",\\\"params\\\":{\\\"processDefinitionId\\\":46}," +
"\\\"timeout\\\":{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}}],\\\"timeout\\\":0}\"," +
"\"processDefinitionDescription\":\"\",\"processDefinitionLocations\":\"{\\\"tasks-84090\\\":{\\\"name\\\":\\\"shell-4\\\"," +
"\\\"targetarr\\\":\\\"\\\",\\\"x\\\":128,\\\"y\\\":114},\\\"tasks-87364\\\":{\\\"name\\\":\\\"shell-5\\\"," +
"\\\"targetarr\\\":\\\"tasks-84090\\\",\\\"x\\\":266,\\\"y\\\":115}}\"," +
"\"processDefinitionConnects\":\"[{\\\"endPointSourceId\\\":\\\"tasks-84090\\\"," +
"\\\"endPointTargetId\\\":\\\"tasks-87364\\\"}]\"}";
String subProcessJson = "{\"globalParams\":[]," +
"\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-52423\",\"name\":\"shell-5\"," +
"\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo \\\"shell-5\\\"\"},\"description\":\"\"," +
"\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," +
"\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1," +
"\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}";
FileUtils.writeStringToFile(new File("/tmp/task.json"),processJson);
File file = new File("/tmp/task.json");
FileInputStream fileInputStream = new FileInputStream("/tmp/task.json");
MultipartFile multipartFile = new MockMultipartFile(file.getName(), file.getName(),
ContentType.APPLICATION_OCTET_STREAM.toString(), fileInputStream);
User loginUser = new User();
loginUser.setId(1);
loginUser.setUserType(UserType.ADMIN_USER);
String currentProjectName = "testProject";
Map<String, Object> result = new HashMap<>(5);
putMsg(result, Status.SUCCESS, currentProjectName);
ProcessDefinition shellDefinition2 = new ProcessDefinition();
shellDefinition2.setId(46);
shellDefinition2.setName("shell-5");
shellDefinition2.setProjectId(2);
shellDefinition2.setProcessDefinitionJson(subProcessJson);
Mockito.when(projectMapper.queryByName(currentProjectName)).thenReturn(getProject(currentProjectName));
Mockito.when(projectService.checkProjectAndAuth(loginUser, getProject(currentProjectName), currentProjectName)).thenReturn(result);
Mockito.when(processDefineMapper.queryByDefineId(46)).thenReturn(shellDefinition2);
//import process
Map<String, Object> importProcessResult = processDefinitionService.importProcessDefinition(loginUser, multipartFile, currentProjectName);
Assert.assertEquals(Status.SUCCESS, importProcessResult.get(Constants.STATUS));
boolean delete = file.delete();
Assert.assertTrue(delete);
}
/**
* get mock datasource
* @return DataSource
*/
private DataSource getDataSource(){
DataSource dataSource = new DataSource();
dataSource.setId(2);
dataSource.setName("test");
return dataSource;
}
/**
* get mock processDefinition
* @return ProcessDefinition
*/
private ProcessDefinition getProcessDefinition(){
ProcessDefinition processDefinition = new ProcessDefinition();
processDefinition.setId(46);
processDefinition.setName("testProject");
processDefinition.setProjectId(2);
return processDefinition;
}
/**
* get mock Project
* @param projectName projectName
* @return Project
*/
private Project getProject(String projectName){
Project project = new Project();
project.setId(1);
project.setName(projectName);
project.setUserId(1);
return project;
}
private void putMsg(Map<String, Object> result, Status status, Object... statusParams) {
result.put(Constants.STATUS, status);
if (statusParams != null && statusParams.length > 0) {
result.put(Constants.MSG, MessageFormat.format(status.getMsg(), statusParams));
} else {
result.put(Constants.MSG, status.getMsg());
}
}
}

10
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java

@ -16,10 +16,11 @@
*/
package org.apache.dolphinscheduler.api.service;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import java.util.Calendar;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Session;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.SessionMapper;
@ -121,6 +122,9 @@ public class SessionServiceTest {
String ip = "127.0.0.1";
User user = new User();
user.setId(userId);
Mockito.when(sessionMapper.queryByUserIdAndIp(userId,ip)).thenReturn(getSession());
sessionService.signOut(ip ,user);
}
@ -130,7 +134,7 @@ public class SessionServiceTest {
Session session = new Session();
session.setId(sessionId);
session.setIp("127.0.0.1");
session.setLastLoginTime(DateUtils.addDays(new Date(),40));
session.setLastLoginTime(DateUtils.add(new Date(), Calendar.DAY_OF_MONTH, 40));
session.setUserId(1);
return session;
}

7
dolphinscheduler-common/pom.xml

@ -600,13 +600,6 @@
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${lombok.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>

1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/ClickHouseDataSource.java

@ -66,7 +66,6 @@ public class ClickHouseDataSource extends BaseDataSource {
con.close();
} catch (SQLException e) {
logger.error("ClickHouse datasource try conn close conn error", e);
throw e;
}
}
}

1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/DB2ServerDataSource.java

@ -66,7 +66,6 @@ public class DB2ServerDataSource extends BaseDataSource {
con.close();
} catch (SQLException e) {
logger.error("DB2 Server datasource try conn close conn error", e);
throw e;
}
}
}

1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/HiveDataSource.java

@ -69,7 +69,6 @@ public class HiveDataSource extends BaseDataSource {
con.close();
} catch (SQLException e) {
logger.error("hive datasource try conn close conn error", e);
throw e;
}
}
}

1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/MySQLDataSource.java

@ -64,7 +64,6 @@ public class MySQLDataSource extends BaseDataSource {
con.close();
} catch (SQLException e) {
logger.error("Mysql datasource try conn close conn error", e);
throw e;
}
}
}

1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/OracleDataSource.java

@ -66,7 +66,6 @@ public class OracleDataSource extends BaseDataSource {
con.close();
} catch (SQLException e) {
logger.error("Oracle datasource try conn close conn error", e);
throw e;
}
}
}

1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/PostgreDataSource.java

@ -68,7 +68,6 @@ public class PostgreDataSource extends BaseDataSource {
con.close();
} catch (SQLException e) {
logger.error("Postgre datasource try conn close conn error", e);
throw e;
}
}
}

1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SQLServerDataSource.java

@ -62,7 +62,6 @@ public class SQLServerDataSource extends BaseDataSource {
con.close();
} catch (SQLException e) {
logger.error("SQL Server datasource try conn close conn error", e);
throw e;
}
}
}

1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SparkDataSource.java

@ -71,7 +71,6 @@ public class SparkDataSource extends BaseDataSource {
con.close();
} catch (SQLException e) {
logger.error("Spark datasource try conn close conn error", e);
throw e;
}
}
}

1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/DateInterval.java

@ -59,4 +59,5 @@ public class DateInterval {
public void setEndTime(Date endTime) {
this.endTime = endTime;
}
}

1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNodeRelation.java

@ -53,6 +53,7 @@ public class TaskNodeRelation {
}
@Override
public boolean equals(Object o){
if (!(o instanceof TaskNodeRelation)) {
return false;

37
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/queue/TaskQueueZkImpl.java

@ -37,8 +37,27 @@ public class TaskQueueZkImpl implements ITaskQueue {
private static final Logger logger = LoggerFactory.getLogger(TaskQueueZkImpl.class);
private final ZookeeperOperator zookeeperOperator;
@Autowired
private ZookeeperOperator zookeeperOperator;
public TaskQueueZkImpl(ZookeeperOperator zookeeperOperator) {
this.zookeeperOperator = zookeeperOperator;
try {
String tasksQueuePath = getTasksPath(Constants.DOLPHINSCHEDULER_TASKS_QUEUE);
String tasksKillPath = getTasksPath(Constants.DOLPHINSCHEDULER_TASKS_KILL);
for (String key : new String[]{tasksQueuePath,tasksKillPath}){
if (!zookeeperOperator.isExisted(key)){
zookeeperOperator.persist(key, "");
logger.info("create tasks queue parent node success : {}", key);
}
}
} catch (Exception e) {
logger.error("create tasks queue parent node failure", e);
}
}
/**
* get all tasks from tasks queue
@ -321,20 +340,20 @@ public class TaskQueueZkImpl implements ITaskQueue {
public void delete(){
try {
String tasksQueuePath = getTasksPath(Constants.DOLPHINSCHEDULER_TASKS_QUEUE);
String tasksCancelPath = getTasksPath(Constants.DOLPHINSCHEDULER_TASKS_KILL);
String tasksKillPath = getTasksPath(Constants.DOLPHINSCHEDULER_TASKS_KILL);
for(String taskQueuePath : new String[]{tasksQueuePath,tasksCancelPath}){
if(zookeeperOperator.isExisted(taskQueuePath)){
List<String> list = zookeeperOperator.getChildrenKeys(taskQueuePath);
for (String key : new String[]{tasksQueuePath,tasksKillPath}){
if (zookeeperOperator.isExisted(key)){
List<String> list = zookeeperOperator.getChildrenKeys(key);
for (String task : list) {
zookeeperOperator.remove(taskQueuePath + Constants.SINGLE_SLASH + task);
logger.info("delete task from tasks queue : {}/{} ",taskQueuePath,task);
zookeeperOperator.remove(key + Constants.SINGLE_SLASH + task);
logger.info("delete task from tasks queue : {}/{} ", key, task);
}
}
}
} catch (Exception e) {
logger.error("delete all tasks in tasks queue failure",e);
logger.error("delete all tasks in tasks queue failure", e);
}
}
@ -344,7 +363,7 @@ public class TaskQueueZkImpl implements ITaskQueue {
* @return
*/
public String getTasksPath(String key){
return "/dolphinscheduler" + Constants.SINGLE_SLASH + key;
return zookeeperOperator.getZookeeperConfig().getDsRoot() + Constants.SINGLE_SLASH + key;
}
}

26
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ArrayUtils.java

@ -0,0 +1,26 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
public class ArrayUtils {
public static boolean isEmpty(final int[] array) {
return array == null || array.length == 0;
}
}

3
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ConnectionUtils.java

@ -51,7 +51,6 @@ public class ConnectionUtils {
}
} catch (SQLException e) {
logger.error(e.getMessage(),e);
throw new RuntimeException(e);
} finally {
try {
if (stmt != null) {
@ -60,7 +59,6 @@ public class ConnectionUtils {
}
} catch (SQLException e) {
logger.error(e.getMessage(),e);
throw new RuntimeException(e);
} finally {
try {
if (conn != null) {
@ -69,7 +67,6 @@ public class ConnectionUtils {
}
} catch (SQLException e) {
logger.error(e.getMessage(),e);
throw new RuntimeException(e);
}
}
}

398
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java

File diff suppressed because one or more lines are too long

12
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java

@ -30,4 +30,16 @@ public class EnumUtils {
return null;
}
}
public static <E extends Enum<E>> boolean isValidEnum(final Class<E> enumClass, final String enumName) {
if (enumName == null) {
return false;
}
try {
Enum.valueOf(enumClass, enumName);
return true;
} catch (final IllegalArgumentException ex) {
return false;
}
}
}

5
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java

@ -116,9 +116,9 @@ public class HadoopUtils implements Closeable {
if(StringUtils.isNotBlank(defaultFSProp)){
Map<String, String> fsRelatedProps = PropertyUtils.getPrefixedProperties("fs.");
configuration.set(Constants.FS_DEFAULTFS,defaultFSProp);
fsRelatedProps.entrySet().stream().forEach(entry -> configuration.set(entry.getKey(), entry.getValue()));
fsRelatedProps.forEach((key, value) -> configuration.set(key, value));
}else{
logger.error("property:{} can not to be empty, please set!");
logger.error("property:{} can not to be empty, please set!", Constants.FS_DEFAULTFS );
throw new RuntimeException("property:{} can not to be empty, please set!");
}
}else{
@ -337,7 +337,6 @@ public class HadoopUtils implements Closeable {
* @throws Exception errors
*/
public FileStatus[] listFileStatus(String filePath)throws Exception{
Path path = new Path(filePath);
try {
return fs.listStatus(new Path(filePath));
} catch (IOException e) {

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java

@ -67,7 +67,7 @@ public class HttpUtils {
logger.warn("http entity is null");
}
}else{
logger.error("htt get:{} response status code is not 200!");
logger.error("http get:{} response status code is not 200!", response.getStatusLine().getStatusCode());
}
}catch (Exception e){
logger.error(e.getMessage(),e);

45
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java

@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
public class IOUtils {
public static void closeQuietly(InputStream fis){
if(fis != null){
try {
fis.close();
} catch (IOException ignore) {
}
}
}
public static void closeQuietly(InputStreamReader reader){
if(reader != null){
try {
reader.close();
} catch (IOException ignore) {
}
}
}
}

30
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java

@ -18,6 +18,8 @@ package org.apache.dolphinscheduler.common.utils;
import java.nio.charset.StandardCharsets;
import java.util.Iterator;
import java.util.Objects;
import java.util.regex.Pattern;
@ -122,4 +124,32 @@ public class StringUtils {
}
return false;
}
public static String join(final Iterable<?> iterable, final String separator){
Iterator<?> iterator = iterable.iterator();
if (iterator == null) {
return null;
}
if (!iterator.hasNext()) {
return EMPTY;
}
final Object first = iterator.next();
if (!iterable.iterator().hasNext()) {
return Objects.toString(first, "");
}
final StringBuilder buf = new StringBuilder(64);
if (first != null) {
buf.append(first);
}
while (iterator.hasNext()) {
if (separator != null) {
buf.append(separator);
}
final Object obj = iterator.next();
if (obj != null) {
buf.append(obj);
}
}
return buf.toString();
}
}

174
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DependentUtilsTest.java

@ -34,18 +34,73 @@ public class DependentUtilsTest {
private static final Logger logger = LoggerFactory.getLogger(ShellExecutorTest.class);
@Test
public void getDependResultForRelation() {
//failed
DependentRelation dependentRelation = DependentRelation.AND;
List<DependResult> dependResultList = new ArrayList<>();
dependResultList.add(DependResult.FAILED);
dependResultList.add(DependResult.SUCCESS);
DependResult result = DependentUtils.getDependResultForRelation( dependentRelation, dependResultList);
DependResult result = DependentUtils.getDependResultForRelation(dependentRelation, dependResultList);
Assert.assertEquals(result, DependResult.FAILED);
//waiting
dependResultList = new ArrayList<>();
dependResultList.add(DependResult.WAITING);
dependResultList.add(DependResult.SUCCESS);
result = DependentUtils.getDependResultForRelation(dependentRelation, dependResultList);
Assert.assertEquals(result, DependResult.WAITING);
//success
dependResultList = new ArrayList<>();
dependResultList.add(DependResult.SUCCESS);
dependResultList.add(DependResult.SUCCESS);
result = DependentUtils.getDependResultForRelation(dependentRelation, dependResultList);
Assert.assertEquals(result, DependResult.SUCCESS);
//one success
dependResultList = new ArrayList<>();
dependResultList.add(DependResult.SUCCESS);
result = DependentUtils.getDependResultForRelation(dependentRelation, dependResultList);
Assert.assertEquals(result, DependResult.SUCCESS);
//one failed
dependResultList = new ArrayList<>();
dependResultList.add(DependResult.FAILED);
result = DependentUtils.getDependResultForRelation(dependentRelation, dependResultList);
Assert.assertEquals(result, DependResult.FAILED);
//or success
dependentRelation = DependentRelation.OR;
dependResultList = new ArrayList<>();
dependResultList.add(DependResult.FAILED);
dependResultList.add(DependResult.SUCCESS);
result = DependentUtils.getDependResultForRelation(dependentRelation, dependResultList);
Assert.assertEquals(result, DependResult.SUCCESS);
//waiting
dependResultList = new ArrayList<>();
dependResultList.add(DependResult.WAITING);
dependResultList.add(DependResult.FAILED);
result = DependentUtils.getDependResultForRelation(dependentRelation, dependResultList);
Assert.assertEquals(result, DependResult.WAITING);
Assert.assertEquals(DependentUtils.getDependResultForRelation( dependentRelation, dependResultList),
DependResult.SUCCESS);
//success
dependResultList = new ArrayList<>();
dependResultList.add(DependResult.SUCCESS);
dependResultList.add(DependResult.SUCCESS);
result = DependentUtils.getDependResultForRelation(dependentRelation, dependResultList);
Assert.assertEquals(result, DependResult.SUCCESS);
//one success
dependResultList = new ArrayList<>();
dependResultList.add(DependResult.SUCCESS);
result = DependentUtils.getDependResultForRelation(dependentRelation, dependResultList);
Assert.assertEquals(result, DependResult.SUCCESS);
//one failed
dependResultList = new ArrayList<>();
dependResultList.add(DependResult.FAILED);
result = DependentUtils.getDependResultForRelation(dependentRelation, dependResultList);
Assert.assertEquals(result, DependResult.FAILED);
}
@Test
@ -101,6 +156,115 @@ public class DependentUtilsTest {
Assert.assertEquals(dateIntervals.get(0), monthHead);
Assert.assertEquals(dateIntervals.get(dateIntervals.size() - 1), monthThis);
dateIntervals = DependentUtils.getDateIntervalList(DateUtils.stringToDate("2019-02-04 10:00:00"), "last1Hour");
DateInterval expect = new DateInterval(DateUtils.stringToDate("2019-02-04 09:00:00"),
DateUtils.getEndOfHour(DateUtils.stringToDate("2019-02-04 09:00:00")));
Assert.assertEquals(expect, dateIntervals.get(0));
dateIntervals = DependentUtils.getDateIntervalList(DateUtils.stringToDate("2019-02-04 10:00:00"), "last2Hours");
expect = new DateInterval(DateUtils.stringToDate("2019-02-04 08:00:00"),
DateUtils.getEndOfHour(DateUtils.stringToDate("2019-02-04 08:00:00")));
Assert.assertEquals(expect, dateIntervals.get(0));
dateIntervals = DependentUtils.getDateIntervalList(DateUtils.stringToDate("2019-02-04 10:00:00"), "last3Hours");
expect = new DateInterval(DateUtils.stringToDate("2019-02-04 07:00:00"),
DateUtils.getEndOfHour(DateUtils.stringToDate("2019-02-04 07:00:00")));
Assert.assertEquals(expect, dateIntervals.get(0));
dateValue = "last3Days";
dateIntervals = DependentUtils.getDateIntervalList(DateUtils.stringToDate("2019-02-10 07:00:00"), dateValue);
expect = new DateInterval(DateUtils.stringToDate("2019-02-07 00:00:00"),
DateUtils.getEndOfDay(DateUtils.stringToDate("2019-02-07 00:00:00")));
Assert.assertEquals(expect, dateIntervals.get(0));
dateValue = "last7Days";
dateIntervals = DependentUtils.getDateIntervalList(DateUtils.stringToDate("2019-02-10 07:00:00"), dateValue);
expect = new DateInterval(DateUtils.stringToDate("2019-02-03 00:00:00"),
DateUtils.getEndOfDay(DateUtils.stringToDate("2019-02-03 00:00:00")));
Assert.assertEquals(expect, dateIntervals.get(0));
dateValue = "lastWeek";
dateIntervals = DependentUtils.getDateIntervalList(DateUtils.stringToDate("2019-02-10 07:00:00"), dateValue);
expect = new DateInterval(DateUtils.stringToDate("2019-01-28 00:00:00"),
DateUtils.getEndOfDay(DateUtils.stringToDate("2019-01-28 00:00:00")));
Assert.assertEquals(expect, dateIntervals.get(0));
expect = new DateInterval(DateUtils.stringToDate("2019-02-03 00:00:00"),
DateUtils.getEndOfDay(DateUtils.stringToDate("2019-02-03 00:00:00")));
Assert.assertEquals(expect, dateIntervals.get(6));
Assert.assertEquals(7, dateIntervals.size());
dateValue = "lastMonday";
dateIntervals = DependentUtils.getDateIntervalList(DateUtils.stringToDate("2019-02-10 07:00:00"), dateValue);
expect = new DateInterval(DateUtils.stringToDate("2019-01-28 00:00:00"),
DateUtils.getEndOfDay(DateUtils.stringToDate("2019-01-28 00:00:00")));
Assert.assertEquals(expect, dateIntervals.get(0));
Assert.assertEquals(1, dateIntervals.size());
dateValue = "lastTuesday";
dateIntervals = DependentUtils.getDateIntervalList(DateUtils.stringToDate("2019-02-10 07:00:00"), dateValue);
expect = new DateInterval(DateUtils.stringToDate("2019-01-29 00:00:00"),
DateUtils.getEndOfDay(DateUtils.stringToDate("2019-01-29 00:00:00")));
Assert.assertEquals(expect, dateIntervals.get(0));
Assert.assertEquals(1, dateIntervals.size());
dateValue = "lastWednesday";
dateIntervals = DependentUtils.getDateIntervalList(DateUtils.stringToDate("2019-02-10 07:00:00"), dateValue);
expect = new DateInterval(DateUtils.stringToDate("2019-01-30 00:00:00"),
DateUtils.getEndOfDay(DateUtils.stringToDate("2019-01-30 00:00:00")));
Assert.assertEquals(expect, dateIntervals.get(0));
Assert.assertEquals(1, dateIntervals.size());
dateValue = "lastThursday";
dateIntervals = DependentUtils.getDateIntervalList(DateUtils.stringToDate("2019-02-10 07:00:00"), dateValue);
expect = new DateInterval(DateUtils.stringToDate("2019-01-31 00:00:00"),
DateUtils.getEndOfDay(DateUtils.stringToDate("2019-01-31 00:00:00")));
Assert.assertEquals(expect, dateIntervals.get(0));
Assert.assertEquals(1, dateIntervals.size());
dateValue = "lastFriday";
dateIntervals = DependentUtils.getDateIntervalList(DateUtils.stringToDate("2019-02-10 07:00:00"), dateValue);
expect = new DateInterval(DateUtils.stringToDate("2019-02-01 00:00:00"),
DateUtils.getEndOfDay(DateUtils.stringToDate("2019-02-01 00:00:00")));
Assert.assertEquals(expect, dateIntervals.get(0));
Assert.assertEquals(1, dateIntervals.size());
dateValue = "lastSaturday";
dateIntervals = DependentUtils.getDateIntervalList(DateUtils.stringToDate("2019-02-10 07:00:00"), dateValue);
expect = new DateInterval(DateUtils.stringToDate("2019-02-02 00:00:00"),
DateUtils.getEndOfDay(DateUtils.stringToDate("2019-02-02 00:00:00")));
Assert.assertEquals(expect, dateIntervals.get(0));
Assert.assertEquals(1, dateIntervals.size());
dateValue = "lastSunday";
dateIntervals = DependentUtils.getDateIntervalList(DateUtils.stringToDate("2019-02-10 07:00:00"), dateValue);
expect = new DateInterval(DateUtils.stringToDate("2019-02-03 00:00:00"),
DateUtils.getEndOfDay(DateUtils.stringToDate("2019-02-03 00:00:00")));
Assert.assertEquals(expect, dateIntervals.get(0));
Assert.assertEquals(1, dateIntervals.size());
dateValue = "lastMonth";
dateIntervals = DependentUtils.getDateIntervalList(DateUtils.stringToDate("2019-02-10 07:00:00"), dateValue);
expect = new DateInterval(DateUtils.stringToDate("2019-01-01 00:00:00"),
DateUtils.getEndOfDay(DateUtils.stringToDate("2019-01-01 00:00:00")));
Assert.assertEquals(expect, dateIntervals.get(0));
expect = new DateInterval(DateUtils.stringToDate("2019-01-31 00:00:00"),
DateUtils.getEndOfDay(DateUtils.stringToDate("2019-01-31 00:00:00")));
Assert.assertEquals(expect, dateIntervals.get(30));
Assert.assertEquals(31, dateIntervals.size());
dateValue = "lastMonthBegin";
dateIntervals = DependentUtils.getDateIntervalList(DateUtils.stringToDate("2019-02-10 07:00:00"), dateValue);
expect = new DateInterval(DateUtils.stringToDate("2019-01-01 00:00:00"),
DateUtils.getEndOfDay(DateUtils.stringToDate("2019-01-01 00:00:00")));
Assert.assertEquals(expect, dateIntervals.get(0));
Assert.assertEquals(1, dateIntervals.size());
dateValue = "lastMonthEnd";
dateIntervals = DependentUtils.getDateIntervalList(DateUtils.stringToDate("2019-02-10 07:00:00"), dateValue);
expect = new DateInterval(DateUtils.stringToDate("2019-01-31 00:00:00"),
DateUtils.getEndOfDay(DateUtils.stringToDate("2019-01-31 00:00:00")));
Assert.assertEquals(expect, dateIntervals.get(0));
Assert.assertEquals(1, dateIntervals.size());
}
@Test
@ -197,4 +361,4 @@ public class DependentUtilsTest {
Assert.assertEquals(dateIntervals.get(30), di2);
}
}
}

170
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java

@ -0,0 +1,170 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.hamcrest.core.StringContains.containsString;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
public class PreconditionsTest {
public static final Logger logger = LoggerFactory.getLogger(PreconditionsTest.class);
/**
* Test checkNotNull
*/
@Test
public void testCheckNotNull() throws Exception {
String testReference = "test reference";
//test reference is not null
Assert.assertEquals(testReference,Preconditions.checkNotNull(testReference));
Assert.assertEquals(testReference,Preconditions.checkNotNull(testReference,"reference is null"));
Assert.assertEquals(testReference,Preconditions.checkNotNull(testReference,"%s is null",testReference));
//test reference is null
try {
Preconditions.checkNotNull(null);
} catch (NullPointerException ex) {
assertNull(ex.getMessage());
}
try {
Preconditions.checkNotNull("");
} catch (NullPointerException ex) {
assertNull(ex.getMessage());
}
//test reference is null ,expect contains errorMessage
try {
Preconditions.checkNotNull(null,"reference is null");
} catch (NullPointerException ex) {
assertThat(ex.getMessage(), containsString("reference is null"));
}
try {
Preconditions.checkNotNull("","reference is null");
} catch (NullPointerException ex) {
assertThat(ex.getMessage(), containsString("reference is null"));
}
//test reference is null ,expect contains errorMessageTemplate and errorMessageArgs
try {
Preconditions.checkNotNull(null,"%s is null",testReference);
} catch (NullPointerException ex) {
assertThat(ex.getMessage(), containsString(testReference + " is null"));
}
try {
Preconditions.checkNotNull("","%s is null",testReference);
} catch (NullPointerException ex) {
assertThat(ex.getMessage(), containsString(testReference + " is null"));
}
}
/**
* Test checkArgument
*/
@Test
public void testCheckArgument() throws Exception {
int argument = 100;
//boolean condition is true
Preconditions.checkArgument(argument > 0 && argument < 200);
//boolean condition is false
try {
Preconditions.checkArgument(argument > 0 && argument < 50);
} catch (IllegalArgumentException ex) {
assertNull(ex.getMessage());
}
//boolean condition is false ,expect contains errorMessage
try {
Preconditions.checkArgument(argument > 300, "argument is error");
} catch (IllegalArgumentException ex) {
assertThat(ex.getMessage(), containsString("argument is error"));
}
//boolean condition is false,expect contains errorMessageTemplate and errorMessageArgs
try {
Preconditions.checkArgument(argument > 0 && argument < 99, "argument %s is error",argument);
} catch (IllegalArgumentException ex) {
assertThat(ex.getMessage(), containsString( "argument " + argument + " is error"));
}
}
/**
* Test checkState
*/
@Test
public void testCheckState() throws Exception {
int state = 1;
//boolean condition is true
Preconditions.checkState(state == 1);
Preconditions.checkState(state > -1);
//boolean condition is false
try {
Preconditions.checkState(state > 2);
} catch (IllegalStateException ex) {
assertNull(ex.getMessage());
}
//boolean condition is false ,expect contains errorMessage
try {
Preconditions.checkState(state < 1, "state is error");
} catch (IllegalStateException ex) {
assertThat(ex.getMessage(), containsString("state is error"));
}
//boolean condition is false,expect contains errorMessageTemplate and errorMessageArgs
try {
Preconditions.checkState(state < -1 , "state %s is error",state);
} catch (IllegalStateException ex) {
assertThat(ex.getMessage(), containsString( "state " + state + " is error"));
}
}
/**
* Test checkElementIndex
*/
@Test
public void testCheckElementIndex() throws Exception {
int index = 2;
int size = 30;
//boolean condition is true
Preconditions.checkElementIndex(index, size);
//boolean condition is false
try {
Preconditions.checkElementIndex(-1, 10);
} catch (IndexOutOfBoundsException ex) {
assertThat(ex.getMessage(), containsString("Index: -1, Size: 10"));
}
//boolean condition is false ,expect contains errorMessage
try {
Preconditions.checkElementIndex(100, 50, "index is greater than size");
} catch (IndexOutOfBoundsException ex) {
assertThat(ex.getMessage(), containsString("index is greater than size Index: 100, Size: 50"));
}
}
}

8
dolphinscheduler-dao/pom.xml

@ -108,14 +108,6 @@
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>

2
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java

@ -17,9 +17,9 @@
package org.apache.dolphinscheduler.dao;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.apache.commons.lang3.StringUtils;
import org.apache.dolphinscheduler.common.enums.AlertStatus;
import org.apache.dolphinscheduler.common.enums.AlertType;
import org.apache.dolphinscheduler.common.enums.ShowType;

12
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/ProcessDao.java

@ -18,20 +18,19 @@ package org.apache.dolphinscheduler.dao;
import com.alibaba.fastjson.JSONObject;
import com.cronutils.model.Cron;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.*;
import org.apache.dolphinscheduler.common.model.DateInterval;
import org.apache.dolphinscheduler.common.model.TaskNode;
import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.queue.ITaskQueue;
import org.apache.dolphinscheduler.common.queue.TaskQueueFactory;
import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters;
import org.apache.dolphinscheduler.common.utils.ArrayUtils;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.IpUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.*;
import org.apache.dolphinscheduler.dao.mapper.*;
import org.apache.dolphinscheduler.dao.utils.cron.CronUtils;
@ -458,9 +457,12 @@ public class ProcessDao {
if(tenantId >= 0){
tenant = tenantMapper.queryById(tenantId);
}
if(tenant == null){
if(null == tenant){
User user = userMapper.selectById(userId);
tenant = tenantMapper.queryById(user.getTenantId());
if (null != user) {
tenant = tenantMapper.queryById(user.getTenantId());
}
}
return tenant;
}

2
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java

@ -20,11 +20,11 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.TaskRecordStatus;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.TaskRecord;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

43
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AccessToken.java

@ -20,11 +20,9 @@ import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.util.Date;
@Data
@TableName("t_ds_access_token")
public class AccessToken {
@ -105,4 +103,45 @@ public class AccessToken {
public void setUpdateTime(Date updateTime) {
this.updateTime = updateTime;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
AccessToken that = (AccessToken) o;
if (id != that.id) return false;
if (userId != that.userId) return false;
if (token != null ? !token.equals(that.token) : that.token != null) return false;
if (userName != null ? !userName.equals(that.userName) : that.userName != null) return false;
if (expireTime != null ? !expireTime.equals(that.expireTime) : that.expireTime != null) return false;
if (createTime != null ? !createTime.equals(that.createTime) : that.createTime != null) return false;
return updateTime != null ? updateTime.equals(that.updateTime) : that.updateTime == null;
}
@Override
public int hashCode() {
int result = id;
result = 31 * result + userId;
result = 31 * result + (token != null ? token.hashCode() : 0);
result = 31 * result + (userName != null ? userName.hashCode() : 0);
result = 31 * result + (expireTime != null ? expireTime.hashCode() : 0);
result = 31 * result + (createTime != null ? createTime.hashCode() : 0);
result = 31 * result + (updateTime != null ? updateTime.hashCode() : 0);
return result;
}
@Override
public String toString() {
return "AccessToken{" +
"id=" + id +
", userId=" + userId +
", token='" + token + '\'' +
", userName='" + userName + '\'' +
", expireTime=" + expireTime +
", createTime=" + createTime +
", updateTime=" + updateTime +
'}';
}
}

1
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Alert.java

@ -20,7 +20,6 @@ import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import org.apache.dolphinscheduler.common.enums.AlertStatus;
import org.apache.dolphinscheduler.common.enums.AlertType;
import org.apache.dolphinscheduler.common.enums.ShowType;

14
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/AlertGroup.java

@ -20,11 +20,9 @@ import org.apache.dolphinscheduler.common.enums.AlertType;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.util.Date;
@Data
@TableName("t_ds_alertgroup")
public class AlertGroup {
@ -145,4 +143,16 @@ public class AlertGroup {
result = 31 * result + (updateTime != null ? updateTime.hashCode() : 0);
return result;
}
@Override
public String toString() {
return "AlertGroup{" +
"id=" + id +
", groupName='" + groupName + '\'' +
", groupType=" + groupType +
", description='" + description + '\'' +
", createTime=" + createTime +
", updateTime=" + updateTime +
'}';
}
}

1
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java

@ -20,7 +20,6 @@ import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import org.apache.dolphinscheduler.common.enums.*;
import java.util.Date;

4
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java

@ -16,18 +16,14 @@
*/
package org.apache.dolphinscheduler.dao.entity;
import org.apache.dolphinscheduler.common.enums.DbType;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.util.Date;
@Data
@TableName("t_ds_datasource")
public class DataSource {
/**

2
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java

@ -25,7 +25,6 @@ import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.baomidou.mybatisplus.core.toolkit.StringUtils;
import lombok.Data;
import java.util.Date;
import java.util.List;
@ -36,7 +35,6 @@ import java.util.stream.Collectors;
/**
* process definition
*/
@Data
@TableName("t_ds_process_definition")
public class ProcessDefinition {
/**

96
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java

@ -21,7 +21,6 @@ import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.baomidou.mybatisplus.core.toolkit.StringUtils;
import lombok.Data;
import org.apache.dolphinscheduler.common.enums.*;
import java.util.Date;
@ -29,7 +28,6 @@ import java.util.Date;
/**
* process instance
*/
@Data
@TableName("t_ds_process_instance")
public class ProcessInstance {
@ -619,4 +617,98 @@ public class ProcessInstance {
", receiversCc='" + receiversCc + '\'' +
'}';
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ProcessInstance that = (ProcessInstance) o;
if (id != that.id) return false;
if (processDefinitionId != that.processDefinitionId) return false;
if (runTimes != that.runTimes) return false;
if (maxTryTimes != that.maxTryTimes) return false;
if (executorId != that.executorId) return false;
if (workerGroupId != that.workerGroupId) return false;
if (timeout != that.timeout) return false;
if (tenantId != that.tenantId) return false;
if (state != that.state) return false;
if (recovery != that.recovery) return false;
if (startTime != null ? !startTime.equals(that.startTime) : that.startTime != null) return false;
if (endTime != null ? !endTime.equals(that.endTime) : that.endTime != null) return false;
if (name != null ? !name.equals(that.name) : that.name != null) return false;
if (host != null ? !host.equals(that.host) : that.host != null) return false;
if (processDefinition != null ? !processDefinition.equals(that.processDefinition) : that.processDefinition != null)
return false;
if (commandType != that.commandType) return false;
if (commandParam != null ? !commandParam.equals(that.commandParam) : that.commandParam != null) return false;
if (taskDependType != that.taskDependType) return false;
if (failureStrategy != that.failureStrategy) return false;
if (warningType != that.warningType) return false;
if (warningGroupId != null ? !warningGroupId.equals(that.warningGroupId) : that.warningGroupId != null)
return false;
if (scheduleTime != null ? !scheduleTime.equals(that.scheduleTime) : that.scheduleTime != null) return false;
if (commandStartTime != null ? !commandStartTime.equals(that.commandStartTime) : that.commandStartTime != null)
return false;
if (globalParams != null ? !globalParams.equals(that.globalParams) : that.globalParams != null) return false;
if (processInstanceJson != null ? !processInstanceJson.equals(that.processInstanceJson) : that.processInstanceJson != null)
return false;
if (tenantCode != null ? !tenantCode.equals(that.tenantCode) : that.tenantCode != null) return false;
if (queue != null ? !queue.equals(that.queue) : that.queue != null) return false;
if (isSubProcess != that.isSubProcess) return false;
if (locations != null ? !locations.equals(that.locations) : that.locations != null) return false;
if (connects != null ? !connects.equals(that.connects) : that.connects != null) return false;
if (historyCmd != null ? !historyCmd.equals(that.historyCmd) : that.historyCmd != null) return false;
if (dependenceScheduleTimes != null ? !dependenceScheduleTimes.equals(that.dependenceScheduleTimes) : that.dependenceScheduleTimes != null)
return false;
if (duration != null ? !duration.equals(that.duration) : that.duration != null) return false;
if (processInstancePriority != that.processInstancePriority) return false;
if (workerGroupName != null ? !workerGroupName.equals(that.workerGroupName) : that.workerGroupName != null)
return false;
if (receivers != null ? !receivers.equals(that.receivers) : that.receivers != null) return false;
return receiversCc != null ? receiversCc.equals(that.receiversCc) : that.receiversCc == null;
}
@Override
public int hashCode() {
int result = id;
result = 31 * result + processDefinitionId;
result = 31 * result + (state != null ? state.hashCode() : 0);
result = 31 * result + (recovery != null ? recovery.hashCode() : 0);
result = 31 * result + (startTime != null ? startTime.hashCode() : 0);
result = 31 * result + (endTime != null ? endTime.hashCode() : 0);
result = 31 * result + runTimes;
result = 31 * result + (name != null ? name.hashCode() : 0);
result = 31 * result + (host != null ? host.hashCode() : 0);
result = 31 * result + (processDefinition != null ? processDefinition.hashCode() : 0);
result = 31 * result + (commandType != null ? commandType.hashCode() : 0);
result = 31 * result + (commandParam != null ? commandParam.hashCode() : 0);
result = 31 * result + (taskDependType != null ? taskDependType.hashCode() : 0);
result = 31 * result + maxTryTimes;
result = 31 * result + (failureStrategy != null ? failureStrategy.hashCode() : 0);
result = 31 * result + (warningType != null ? warningType.hashCode() : 0);
result = 31 * result + (warningGroupId != null ? warningGroupId.hashCode() : 0);
result = 31 * result + (scheduleTime != null ? scheduleTime.hashCode() : 0);
result = 31 * result + (commandStartTime != null ? commandStartTime.hashCode() : 0);
result = 31 * result + (globalParams != null ? globalParams.hashCode() : 0);
result = 31 * result + (processInstanceJson != null ? processInstanceJson.hashCode() : 0);
result = 31 * result + executorId;
result = 31 * result + (tenantCode != null ? tenantCode.hashCode() : 0);
result = 31 * result + (queue != null ? queue.hashCode() : 0);
result = 31 * result + (isSubProcess != null ? isSubProcess.hashCode() : 0);
result = 31 * result + (locations != null ? locations.hashCode() : 0);
result = 31 * result + (connects != null ? connects.hashCode() : 0);
result = 31 * result + (historyCmd != null ? historyCmd.hashCode() : 0);
result = 31 * result + (dependenceScheduleTimes != null ? dependenceScheduleTimes.hashCode() : 0);
result = 31 * result + (duration != null ? duration.hashCode() : 0);
result = 31 * result + (processInstancePriority != null ? processInstancePriority.hashCode() : 0);
result = 31 * result + workerGroupId;
result = 31 * result + timeout;
result = 31 * result + tenantId;
result = 31 * result + (workerGroupName != null ? workerGroupName.hashCode() : 0);
result = 31 * result + (receivers != null ? receivers.hashCode() : 0);
result = 31 * result + (receiversCc != null ? receiversCc.hashCode() : 0);
return result;
}
}

24
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstanceMap.java

@ -19,12 +19,10 @@ package org.apache.dolphinscheduler.dao.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
/**
* process instance map
*/
@Data
@TableName("t_ds_relation_process_instance")
public class ProcessInstanceMap {
@ -90,4 +88,26 @@ public class ProcessInstanceMap {
", processInstanceId=" + processInstanceId +
'}';
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ProcessInstanceMap that = (ProcessInstanceMap) o;
if (id != that.id) return false;
if (parentProcessInstanceId != that.parentProcessInstanceId) return false;
if (parentTaskInstanceId != that.parentTaskInstanceId) return false;
return processInstanceId == that.processInstanceId;
}
@Override
public int hashCode() {
int result = id;
result = 31 * result + parentProcessInstanceId;
result = 31 * result + parentTaskInstanceId;
result = 31 * result + processInstanceId;
return result;
}
}

2
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Project.java

@ -20,14 +20,12 @@ import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.util.Date;
/**
* project
*/
@Data
@TableName("t_ds_project")
public class Project {

2
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Schedule.java

@ -20,7 +20,6 @@ import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import org.apache.dolphinscheduler.common.enums.FailureStrategy;
import org.apache.dolphinscheduler.common.enums.Priority;
import org.apache.dolphinscheduler.common.enums.ReleaseState;
@ -32,7 +31,6 @@ import java.util.Date;
* schedule
*
*/
@Data
@TableName("t_ds_schedules")
public class Schedule {

2
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java

@ -26,14 +26,12 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.util.Date;
/**
* task instance
*/
@Data
@TableName("t_ds_task_instance")
public class TaskInstance {

2
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java

@ -22,14 +22,12 @@ import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.util.Date;
/**
* user
*/
@Data
@TableName("t_ds_user")
public class User {

2
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UserAlertGroup.java

@ -20,14 +20,12 @@ import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.util.Date;
/**
* user alert group
*/
@Data
@TableName("t_ds_relation_user_alertgroup")
public class UserAlertGroup {

2
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerGroup.java

@ -19,14 +19,12 @@ package org.apache.dolphinscheduler.dao.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.util.Date;
/**
* worker group for task running
*/
@Data
@TableName("t_ds_worker_group")
public class WorkerGroup {

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save