Browse Source

Merge pull request #2 from apache/dev

upadtecode
pull/3/MERGE
BoYiZhang 4 years ago committed by GitHub
parent
commit
b21aca8e00
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 4
      .github/workflows/ci_e2e.yml
  2. 1
      LICENSE
  3. 12
      NOTICE
  4. 7
      ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/params.py
  5. 13
      docker/docker-swarm/docker-compose.yml
  6. 2
      dolphinscheduler-alert/pom.xml
  7. 2
      dolphinscheduler-api/pom.xml
  8. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/Directory.java
  9. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FileLeaf.java
  10. 16
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java
  11. 9
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java
  12. 18
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java
  13. 54
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java
  14. 8
      dolphinscheduler-common/pom.xml
  15. 5
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
  16. 122
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/WorkerServerModel.java
  17. 3
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java
  18. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
  19. 22
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java
  20. 249
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Preconditions.java
  21. 115
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java
  22. 2
      dolphinscheduler-dao/pom.xml
  23. 53
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java
  24. 16
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java
  25. 85
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSourceTest.java
  26. 2
      dolphinscheduler-dist/pom.xml
  27. 1
      dolphinscheduler-dist/release-docs/LICENSE
  28. 347
      dolphinscheduler-dist/release-docs/NOTICE
  29. 202
      dolphinscheduler-dist/release-docs/licenses/LICENSE-commons-beanutils.txt
  30. 2
      dolphinscheduler-microbench/pom.xml
  31. 2
      dolphinscheduler-plugin-api/pom.xml
  32. 2
      dolphinscheduler-remote/pom.xml
  33. 55
      dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java
  34. 30
      dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java
  35. 2
      dolphinscheduler-server/pom.xml
  36. 16
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/SQLTaskExecutionContext.java
  37. 9
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java
  38. 36
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
  39. 53
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/exceptions/ExecuteException.java
  40. 28
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java
  41. 19
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistry.java
  42. 33
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java
  43. 65
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/UDFUtils.java
  44. 14
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
  45. 115
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java
  46. 20
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
  47. 3
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
  48. 2
      dolphinscheduler-server/src/main/resources/worker.properties
  49. 148
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryTest.java
  50. 2
      dolphinscheduler-service/pom.xml
  51. 117
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java
  52. 38
      dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java
  53. 2
      dolphinscheduler-ui/pom.xml
  54. 3
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/_source/timeoutAlarm.vue
  55. 50
      dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue
  56. 112
      dolphinscheduler-ui/src/js/conf/home/pages/monitor/pages/servers/_source/zookeeperDirectories.vue
  57. 27
      dolphinscheduler-ui/src/js/conf/home/pages/monitor/pages/servers/worker.vue
  58. 4
      dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js
  59. 4
      dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js
  60. 17
      e2e/src/test/java/org/apache/dolphinscheduler/common/BrowserCommon.java
  61. 32
      e2e/src/test/java/org/apache/dolphinscheduler/data/project/RunWorkflowData.java
  62. 35
      e2e/src/test/java/org/apache/dolphinscheduler/data/project/TimingData.java
  63. 10
      e2e/src/test/java/org/apache/dolphinscheduler/data/security/UserManageData.java
  64. 18
      e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateWorkflowLocator.java
  65. 48
      e2e/src/test/java/org/apache/dolphinscheduler/locator/project/RunWorkflowLocator.java
  66. 61
      e2e/src/test/java/org/apache/dolphinscheduler/locator/project/TimingLocator.java
  67. 2
      e2e/src/test/java/org/apache/dolphinscheduler/locator/security/TokenManageLocator.java
  68. 22
      e2e/src/test/java/org/apache/dolphinscheduler/locator/security/UserManageLocator.java
  69. 1
      e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateProjectPage.java
  70. 29
      e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateWorkflowPage.java
  71. 53
      e2e/src/test/java/org/apache/dolphinscheduler/page/project/RunWorkflowPage.java
  72. 134
      e2e/src/test/java/org/apache/dolphinscheduler/page/project/TimingPage.java
  73. 8
      e2e/src/test/java/org/apache/dolphinscheduler/page/security/AlertManagePage.java
  74. 8
      e2e/src/test/java/org/apache/dolphinscheduler/page/security/QueueManagePage.java
  75. 79
      e2e/src/test/java/org/apache/dolphinscheduler/page/security/UserManagePage.java
  76. 11
      e2e/src/test/java/org/apache/dolphinscheduler/testcase/testDeleteData/TestDeleteWorkflow.java
  77. 4
      e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestCreateWorkflow.java
  78. 41
      e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestRunWorkflow.java
  79. 75
      e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestTiming.java
  80. 18
      e2e/src/test/java/org/apache/dolphinscheduler/testcase/testSecurity/TestUserManage.java
  81. 19
      e2e/testng.xml
  82. 7
      pom.xml
  83. 2
      sql/soft_version
  84. 1
      tools/dependencies/known-dependencies.txt

4
.github/workflows/ci_e2e.yml

@ -66,9 +66,9 @@ jobs:
run: cd ./e2e && mvn -B clean test run: cd ./e2e && mvn -B clean test
- name: Collect logs - name: Collect logs
if: failure() if: failure()
uses: actions/upload-artifact@v1 uses: actions/upload-artifact@v2
with: with:
name: dslogs name: dslogs
path: /var/lib/docker/volumes/docker-swarm_dolphinscheduler-logs/_data path: ${{ github.workspace }}/docker/docker-swarm/dolphinscheduler-logs

1
LICENSE

@ -215,3 +215,4 @@ The following components are provided under the Apache License. See project link
The text of each license is the standard Apache 2.0 license. The text of each license is the standard Apache 2.0 license.
ScriptRunner from https://github.com/mybatis/mybatis-3 Apache 2.0 ScriptRunner from https://github.com/mybatis/mybatis-3 Apache 2.0
mvnw files from https://github.com/takari/maven-wrapper Apache 2.0 mvnw files from https://github.com/takari/maven-wrapper Apache 2.0
PropertyPlaceholderHelper from https://github.com/spring-projects/spring-framework Apache 2.0

12
NOTICE

@ -73,3 +73,15 @@ Refactored SqlBuilder class (SQL, AbstractSQL)
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
Spring Framework ${version}
Copyright (c) 2002-${copyright} Pivotal, Inc.
This product is licensed to you under the Apache License, Version 2.0
(the "License"). You may not use this product except in compliance with
the License.
This product may include a number of subcomponents with separate
copyright notices and license terms. Your use of the source code for
these subcomponents is subject to the terms and conditions of the
subcomponent's license, as noted in the license.txt file.

7
ambari_plugin/common-services/DOLPHIN/1.3.0/package/scripts/params.py

@ -145,6 +145,11 @@ if len(zookeeperHosts) > 0 and "clientPort" in config['configurations']['zoo.cfg
zookeeperPort = ":" + clientPort + "," zookeeperPort = ":" + clientPort + ","
dolphin_zookeeper_map['zookeeper.quorum'] = zookeeperPort.join(zookeeperHosts) + ":" + clientPort dolphin_zookeeper_map['zookeeper.quorum'] = zookeeperPort.join(zookeeperHosts) + ":" + clientPort
dolphin_zookeeper_map.update(config['configurations']['dolphin-zookeeper']) dolphin_zookeeper_map.update(config['configurations']['dolphin-zookeeper'])
if 'spring.servlet.multipart.max-file-size' in dolphin_app_api_map:
file_size = dolphin_app_api_map['spring.servlet.multipart.max-file-size']
dolphin_app_api_map['spring.servlet.multipart.max-file-size'] = file_size + "MB"
if 'spring.servlet.multipart.max-request-size' in dolphin_app_api_map:
request_size = dolphin_app_api_map['spring.servlet.multipart.max-request-size']
dolphin_app_api_map['spring.servlet.multipart.max-request-size'] = request_size + "MB"

13
docker/docker-swarm/docker-compose.yml

@ -72,7 +72,7 @@ services:
- dolphinscheduler-postgresql - dolphinscheduler-postgresql
- dolphinscheduler-zookeeper - dolphinscheduler-zookeeper
volumes: volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs - ./dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks: networks:
- dolphinscheduler - dolphinscheduler
@ -95,7 +95,7 @@ services:
depends_on: depends_on:
- dolphinscheduler-api - dolphinscheduler-api
volumes: volumes:
- dolphinscheduler-logs:/var/log/nginx - ./dolphinscheduler-logs:/var/log/nginx
networks: networks:
- dolphinscheduler - dolphinscheduler
@ -133,7 +133,7 @@ services:
depends_on: depends_on:
- dolphinscheduler-postgresql - dolphinscheduler-postgresql
volumes: volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs - ./dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks: networks:
- dolphinscheduler - dolphinscheduler
@ -168,7 +168,7 @@ services:
- dolphinscheduler-postgresql - dolphinscheduler-postgresql
- dolphinscheduler-zookeeper - dolphinscheduler-zookeeper
volumes: volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs - ./dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks: networks:
- dolphinscheduler - dolphinscheduler
@ -210,9 +210,7 @@ services:
- type: volume - type: volume
source: dolphinscheduler-worker-data source: dolphinscheduler-worker-data
target: /tmp/dolphinscheduler target: /tmp/dolphinscheduler
- type: volume - ./dolphinscheduler-logs:/opt/dolphinscheduler/logs
source: dolphinscheduler-logs
target: /opt/dolphinscheduler/logs
networks: networks:
- dolphinscheduler - dolphinscheduler
@ -225,7 +223,6 @@ volumes:
dolphinscheduler-postgresql-initdb: dolphinscheduler-postgresql-initdb:
dolphinscheduler-zookeeper: dolphinscheduler-zookeeper:
dolphinscheduler-worker-data: dolphinscheduler-worker-data:
dolphinscheduler-logs:
configs: configs:
dolphinscheduler-worker-task-env: dolphinscheduler-worker-task-env:

2
dolphinscheduler-alert/pom.xml

@ -21,7 +21,7 @@
<parent> <parent>
<groupId>org.apache.dolphinscheduler</groupId> <groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId> <artifactId>dolphinscheduler</artifactId>
<version>1.3.0-SNAPSHOT</version> <version>1.3.2-SNAPSHOT</version>
</parent> </parent>
<artifactId>dolphinscheduler-alert</artifactId> <artifactId>dolphinscheduler-alert</artifactId>
<name>${project.artifactId}</name> <name>${project.artifactId}</name>

2
dolphinscheduler-api/pom.xml

@ -21,7 +21,7 @@
<parent> <parent>
<groupId>org.apache.dolphinscheduler</groupId> <groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId> <artifactId>dolphinscheduler</artifactId>
<version>1.3.0-SNAPSHOT</version> <version>1.3.2-SNAPSHOT</version>
</parent> </parent>
<artifactId>dolphinscheduler-api</artifactId> <artifactId>dolphinscheduler-api</artifactId>
<name>${project.artifactId}</name> <name>${project.artifactId}</name>

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/Directory.java

@ -1,5 +1,3 @@
package org.apache.dolphinscheduler.api.dto.resources;
/* /*
* Licensed to the Apache Software Foundation (ASF) under one or more * Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with * contributor license agreements. See the NOTICE file distributed with
@ -16,6 +14,8 @@ package org.apache.dolphinscheduler.api.dto.resources;
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.api.dto.resources;
/** /**
* directory * directory
*/ */

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FileLeaf.java

@ -1,5 +1,3 @@
package org.apache.dolphinscheduler.api.dto.resources;
/* /*
* Licensed to the Apache Software Foundation (ASF) under one or more * Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with * contributor license agreements. See the NOTICE file distributed with
@ -16,6 +14,8 @@ package org.apache.dolphinscheduler.api.dto.resources;
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.api.dto.resources;
/** /**
* file leaf * file leaf
*/ */

16
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java

@ -1,11 +1,3 @@
package org.apache.dolphinscheduler.api.dto.resources;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import java.util.ArrayList;
import java.util.List;
/* /*
* Licensed to the Apache Software Foundation (ASF) under one or more * Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with * contributor license agreements. See the NOTICE file distributed with
@ -22,6 +14,14 @@ import java.util.List;
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.api.dto.resources;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import java.util.ArrayList;
import java.util.List;
/** /**
* resource component * resource component
*/ */

9
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java

@ -1,8 +1,3 @@
package org.apache.dolphinscheduler.api.dto.resources.visitor;
import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
/* /*
* Licensed to the Apache Software Foundation (ASF) under one or more * Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with * contributor license agreements. See the NOTICE file distributed with
@ -19,6 +14,10 @@ import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.api.dto.resources.visitor;
import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
/** /**
* Visitor * Visitor
*/ */

18
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java

@ -50,20 +50,10 @@ public class LoginHandlerInterceptor implements HandlerInterceptor {
/** /**
* Intercept the execution of a handler. Called after HandlerMapping determined * Intercept the execution of a handler. Called after HandlerMapping determined
* an appropriate handler object, but before HandlerAdapter invokes the handler. * @param request current HTTP request
* <p>DispatcherServlet processes a handler in an execution chain, consisting * @param response current HTTP response
* of any number of interceptors, with the handler itself at the end. * @param handler chosen handler to execute, for type and/or instance evaluation
* With this method, each interceptor can decide to abort the execution chain, * @return boolean true or false
* typically sending a HTTP error or writing a custom response.
* <p><strong>Note:</strong> special considerations apply for asynchronous
* request processing. For more details see
* {@link org.springframework.web.servlet.AsyncHandlerInterceptor}.
* @param request current HTTP request
* @param response current HTTP response
* @param handler chosen handler to execute, for type and/or instance evaluation
* @return {@code true} if the execution chain should proceed with the
* next interceptor or the handler itself. Else, DispatcherServlet assumes
* that this interceptor has already dealt with the response itself.
*/ */
@Override @Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) { public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) {

54
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java

@ -16,29 +16,33 @@
*/ */
package org.apache.dolphinscheduler.api.service; package org.apache.dolphinscheduler.api.service;
import static org.apache.dolphinscheduler.common.utils.Preconditions.checkNotNull;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.ZookeeperMonitor; import org.apache.dolphinscheduler.api.utils.ZookeeperMonitor;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ZKNodeType; import org.apache.dolphinscheduler.common.enums.ZKNodeType;
import org.apache.dolphinscheduler.dao.MonitorDBDao;
import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.common.model.WorkerServerModel;
import org.apache.dolphinscheduler.dao.MonitorDBDao;
import org.apache.dolphinscheduler.dao.entity.MonitorRecord; import org.apache.dolphinscheduler.dao.entity.MonitorRecord;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.entity.ZookeeperRecord; import org.apache.dolphinscheduler.dao.entity.ZookeeperRecord;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.google.common.collect.Sets;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.apache.dolphinscheduler.common.utils.Preconditions.*;
/** /**
* monitor service * monitor service
*/ */
@Service @Service
public class MonitorService extends BaseService{ public class MonitorService extends BaseService {
@Autowired @Autowired
private ZookeeperMonitor zookeeperMonitor; private ZookeeperMonitor zookeeperMonitor;
@ -108,15 +112,41 @@ public class MonitorService extends BaseService{
public Map<String,Object> queryWorker(User loginUser) { public Map<String,Object> queryWorker(User loginUser) {
Map<String, Object> result = new HashMap<>(5); Map<String, Object> result = new HashMap<>(5);
List<Server> masterServers = getServerListFromZK(false); List<WorkerServerModel> workerServers = getServerListFromZK(false)
.stream()
result.put(Constants.DATA_LIST, masterServers); .map((Server server) -> {
WorkerServerModel model = new WorkerServerModel();
model.setId(server.getId());
model.setHost(server.getHost());
model.setPort(server.getPort());
model.setZkDirectories(Sets.newHashSet(server.getZkDirectory()));
model.setResInfo(server.getResInfo());
model.setCreateTime(server.getCreateTime());
model.setLastHeartbeatTime(server.getLastHeartbeatTime());
return model;
})
.collect(Collectors.toList());
Map<String, WorkerServerModel> workerHostPortServerMapping = workerServers
.stream()
.collect(Collectors.toMap(
(WorkerServerModel worker) -> {
String[] s = worker.getZkDirectories().iterator().next().split("/");
return s[s.length - 1];
}
, Function.identity()
, (WorkerServerModel oldOne, WorkerServerModel newOne) -> {
oldOne.getZkDirectories().addAll(newOne.getZkDirectories());
return oldOne;
}));
result.put(Constants.DATA_LIST, workerHostPortServerMapping.values());
putMsg(result,Status.SUCCESS); putMsg(result,Status.SUCCESS);
return result; return result;
} }
public List<Server> getServerListFromZK(boolean isMaster){ public List<Server> getServerListFromZK(boolean isMaster) {
checkNotNull(zookeeperMonitor); checkNotNull(zookeeperMonitor);
ZKNodeType zkNodeType = isMaster ? ZKNodeType.MASTER : ZKNodeType.WORKER; ZKNodeType zkNodeType = isMaster ? ZKNodeType.MASTER : ZKNodeType.WORKER;

8
dolphinscheduler-common/pom.xml

@ -21,7 +21,7 @@
<parent> <parent>
<groupId>org.apache.dolphinscheduler</groupId> <groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId> <artifactId>dolphinscheduler</artifactId>
<version>1.3.0-SNAPSHOT</version> <version>1.3.2-SNAPSHOT</version>
</parent> </parent>
<artifactId>dolphinscheduler-common</artifactId> <artifactId>dolphinscheduler-common</artifactId>
<name>dolphinscheduler-common</name> <name>dolphinscheduler-common</name>
@ -84,12 +84,14 @@
<groupId>com.fasterxml.jackson.core</groupId> <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId> <artifactId>jackson-databind</artifactId>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.commons</groupId> <groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId> <artifactId>commons-collections4</artifactId>
</dependency> </dependency>
<dependency>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
</dependency>
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId> <artifactId>hadoop-common</artifactId>

5
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@ -978,5 +978,10 @@ public final class Constants {
public static final int NORAML_NODE_STATUS = 0; public static final int NORAML_NODE_STATUS = 0;
public static final int ABNORMAL_NODE_STATUS = 1; public static final int ABNORMAL_NODE_STATUS = 1;
/**
* net system properties
*/
public static final String DOLPHIN_SCHEDULER_PREFERRED_NETWORK_INTERFACE = "dolphin.scheduler.network.interface.preferred";
} }

122
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/WorkerServerModel.java

@ -0,0 +1,122 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.model;
import java.util.Date;
import java.util.Set;
import com.fasterxml.jackson.annotation.JsonFormat;
/**
* server
*/
public class WorkerServerModel {
/**
* id
*/
private int id;
/**
* host
*/
private String host;
/**
* port
*/
private int port;
/**
* worker directories in zookeeper
*/
private Set<String> zkDirectories;
/**
* resource info about CPU and memory
*/
private String resInfo;
/**
* create time
*/
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8")
private Date createTime;
/**
* last heart beat time
*/
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8")
private Date lastHeartbeatTime;
public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
public Date getCreateTime() {
return createTime;
}
public void setCreateTime(Date createTime) {
this.createTime = createTime;
}
public Set<String> getZkDirectories() {
return zkDirectories;
}
public void setZkDirectories(Set<String> zkDirectories) {
this.zkDirectories = zkDirectories;
}
public Date getLastHeartbeatTime() {
return lastHeartbeatTime;
}
public void setLastHeartbeatTime(Date lastHeartbeatTime) {
this.lastHeartbeatTime = lastHeartbeatTime;
}
public String getResInfo() {
return resInfo;
}
public void setResInfo(String resInfo) {
this.resInfo = resInfo;
}
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public int getPort() {
return port;
}
public void setPort(int port) {
this.port = port;
}
}

3
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java

@ -16,8 +16,7 @@
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.beanutils.BeanMap;
import org.apache.commons.collections.BeanMap;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import java.util.*; import java.util.*;

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java

@ -408,7 +408,7 @@ public class HadoopUtils implements Closeable {
* @param applicationId application id * @param applicationId application id
* @return the return may be null or there may be other parse exceptions * @return the return may be null or there may be other parse exceptions
*/ */
public ExecutionStatus getApplicationStatus(String applicationId) throws Exception{ public ExecutionStatus getApplicationStatus(String applicationId) throws Exception {
if (StringUtils.isEmpty(applicationId)) { if (StringUtils.isEmpty(applicationId)) {
return null; return null;
} }

22
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java

@ -21,13 +21,11 @@ import org.slf4j.LoggerFactory;
import java.io.IOException; import java.io.IOException;
import java.net.*; import java.net.*;
import java.util.Enumeration; import java.util.*;
import java.util.LinkedList;
import java.util.List;
import java.util.Optional;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import static java.util.Collections.emptyList; import static java.util.Collections.emptyList;
import static org.apache.dolphinscheduler.common.Constants.DOLPHIN_SCHEDULER_PREFERRED_NETWORK_INTERFACE;
/** /**
* NetUtils * NetUtils
@ -171,9 +169,19 @@ public class NetUtils {
logger.warn("ValidNetworkInterfaces exception", e); logger.warn("ValidNetworkInterfaces exception", e);
} }
NetworkInterface result = null;
// Try to specify config NetWork Interface
for (NetworkInterface networkInterface : validNetworkInterfaces) {
if (isSpecifyNetworkInterface(networkInterface)) {
result = networkInterface;
break;
}
}
if (null != result) {
return result;
}
return validNetworkInterfaces.get(0); return validNetworkInterfaces.get(0);
} }
/** /**
@ -206,4 +214,8 @@ public class NetUtils {
|| !networkInterface.isUp(); || !networkInterface.isUp();
} }
private static boolean isSpecifyNetworkInterface(NetworkInterface networkInterface) {
String preferredNetworkInterface = System.getProperty(DOLPHIN_SCHEDULER_PREFERRED_NETWORK_INTERFACE);
return Objects.equals(networkInterface.getDisplayName(), preferredNetworkInterface);
}
} }

249
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Preconditions.java

@ -18,266 +18,55 @@ package org.apache.dolphinscheduler.common.utils;
/** /**
* A collection of static utility methods to validate input. * utility methods for validating input
* *
* <p>This class is modelled after Google Guava's Preconditions class, and partly takes code
* from that class. We add this code to here base in order to reduce external
* dependencies.
*/ */
public final class Preconditions { public final class Preconditions {
// ------------------------------------------------------------------------ private Preconditions() {}
// Null checks
// ------------------------------------------------------------------------
/** /**
* Ensures that the given object reference is not null. * if obj is null will throw NPE
* Upon violation, a {@code NullPointerException} with no message is thrown.
* *
* @param reference reference * @param obj obj
* @param <T> T * @param <T> T
* @return T * @return T
*/ */
public static <T> T checkNotNull(T reference) { public static <T> T checkNotNull(T obj) {
if (reference == null) { if (obj == null) {
throw new NullPointerException(); throw new NullPointerException();
} }
return reference; return obj;
} }
/** /**
* Ensures that the given object reference is not null. * if obj is null will throw NullPointerException with error message
* Upon violation, a {@code NullPointerException} with the given message is thrown. * @param obj obj
* @param reference reference * @param errorMsg error message
* @param errorMessage errorMessage
* @param <T> T * @param <T> T
* @return T * @return T
*/ */
public static <T> T checkNotNull(T reference, String errorMessage) { public static <T> T checkNotNull(T obj, String errorMsg) {
if (reference == null) { if (obj == null) {
throw new NullPointerException(String.valueOf(errorMessage)); throw new NullPointerException(errorMsg);
}
return reference;
}
/**
* Ensures that the given object reference is not null.
* Upon violation, a {@code NullPointerException} with the given message is thrown.
*
* <p>The error message is constructed from a template and an arguments array, after
* a similar fashion as {@link String#format(String, Object...)}, but supporting only
* {@code %s} as a placeholder.
*
* @param reference The object reference
* @param errorMessageTemplate The message template for the {@code NullPointerException}
* that is thrown if the check fails. The template substitutes its
* {@code %s} placeholders with the error message arguments.
* @param errorMessageArgs The arguments for the error message, to be inserted into the
* message template for the {@code %s} placeholders.
*
* @param <T> T
* @return The object reference itself (generically typed).
*/
public static <T> T checkNotNull(T reference,
String errorMessageTemplate,
Object... errorMessageArgs) {
if (reference == null) {
throw new NullPointerException(format(errorMessageTemplate, errorMessageArgs));
} }
return reference; return obj;
} }
// ------------------------------------------------------------------------
// Boolean Condition Checking (Argument)
// ------------------------------------------------------------------------
/** /**
* Checks the given boolean condition, and throws an {@code IllegalArgumentException} if * if condition is false will throw an IllegalArgumentException with the given message
* the condition is not met (evaluates to {@code false}).
* *
* @param condition The condition to check * @param condition condition
* @param errorMsg error message
* *
* @throws IllegalArgumentException Thrown, if the condition is violated. * @throws IllegalArgumentException Thrown, if the condition is violated.
*/ */
public static void checkArgument(boolean condition) { public static void checkArgument(boolean condition, Object errorMsg) {
if (!condition) {
throw new IllegalArgumentException();
}
}
/**
* Checks the given boolean condition, and throws an {@code IllegalArgumentException} if
* the condition is not met (evaluates to {@code false}). The exception will have the
* given error message.
*
* @param condition The condition to check
* @param errorMessage The message for the {@code IllegalArgumentException} that is thrown if the check fails.
*
* @throws IllegalArgumentException Thrown, if the condition is violated.
*/
public static void checkArgument(boolean condition, Object errorMessage) {
if (!condition) {
throw new IllegalArgumentException(String.valueOf(errorMessage));
}
}
/**
* Checks the given boolean condition, and throws an {@code IllegalArgumentException} if
* the condition is not met (evaluates to {@code false}).
*
* @param condition The condition to check
* @param errorMessageTemplate The message template for the {@code IllegalArgumentException}
* that is thrown if the check fails. The template substitutes its
* {@code %s} placeholders with the error message arguments.
* @param errorMessageArgs The arguments for the error message, to be inserted into the
* message template for the {@code %s} placeholders.
*
* @throws IllegalArgumentException Thrown, if the condition is violated.
*/
public static void checkArgument(boolean condition,
String errorMessageTemplate,
Object... errorMessageArgs) {
if (!condition) {
throw new IllegalArgumentException(format(errorMessageTemplate, errorMessageArgs));
}
}
// ------------------------------------------------------------------------
// Boolean Condition Checking (State)
// ------------------------------------------------------------------------
/**
* Checks the given boolean condition, and throws an {@code IllegalStateException} if
* the condition is not met (evaluates to {@code false}).
*
* @param condition The condition to check
*
* @throws IllegalStateException Thrown, if the condition is violated.
*/
public static void checkState(boolean condition) {
if (!condition) {
throw new IllegalStateException();
}
}
/**
* Checks the given boolean condition, and throws an {@code IllegalStateException} if
* the condition is not met (evaluates to {@code false}). The exception will have the
* given error message.
*
* @param condition The condition to check
* @param errorMessage The message for the {@code IllegalStateException} that is thrown if the check fails.
*
* @throws IllegalStateException Thrown, if the condition is violated.
*/
public static void checkState(boolean condition, Object errorMessage) {
if (!condition) { if (!condition) {
throw new IllegalStateException(String.valueOf(errorMessage)); throw new IllegalArgumentException(String.valueOf(errorMsg));
} }
} }
/**
* Checks the given boolean condition, and throws an {@code IllegalStateException} if
* the condition is not met (evaluates to {@code false}).
*
* @param condition The condition to check
* @param errorMessageTemplate The message template for the {@code IllegalStateException}
* that is thrown if the check fails. The template substitutes its
* {@code %s} placeholders with the error message arguments.
* @param errorMessageArgs The arguments for the error message, to be inserted into the
* message template for the {@code %s} placeholders.
*
* @throws IllegalStateException Thrown, if the condition is violated.
*/
public static void checkState(boolean condition,
String errorMessageTemplate,
Object... errorMessageArgs) {
if (!condition) {
throw new IllegalStateException(format(errorMessageTemplate, errorMessageArgs));
}
}
/**
* Ensures that the given index is valid for an array, list or string of the given size.
*
* @param index index to check
* @param size size of the array, list or string
*
* @throws IllegalArgumentException Thrown, if size is negative.
* @throws IndexOutOfBoundsException Thrown, if the index negative or greater than or equal to size
*/
public static void checkElementIndex(int index, int size) {
checkArgument(size >= 0, "Size was negative.");
if (index < 0 || index >= size) {
throw new IndexOutOfBoundsException("Index: " + index + ", Size: " + size);
}
}
/**
* Ensures that the given index is valid for an array, list or string of the given size.
*
* @param index index to check
* @param size size of the array, list or string
* @param errorMessage The message for the {@code IndexOutOfBoundsException} that is thrown if the check fails.
*
* @throws IllegalArgumentException Thrown, if size is negative.
* @throws IndexOutOfBoundsException Thrown, if the index negative or greater than or equal to size
*/
public static void checkElementIndex(int index, int size, String errorMessage) {
checkArgument(size >= 0, "Size was negative.");
if (index < 0 || index >= size) {
throw new IndexOutOfBoundsException(String.valueOf(errorMessage) + " Index: " + index + ", Size: " + size);
}
}
// ------------------------------------------------------------------------
// Utilities
// ------------------------------------------------------------------------
/**
* A simplified formatting method. Similar to {@link String#format(String, Object...)}, but
* with lower overhead (only String parameters, no locale, no format validation).
*
* <p>This method is taken quasi verbatim from the Guava Preconditions class.
*/
private static String format( String template, Object... args) {
final int numArgs = args == null ? 0 : args.length;
template = String.valueOf(template); // null -> "null"
// start substituting the arguments into the '%s' placeholders
StringBuilder builder = new StringBuilder(template.length() + 16 * numArgs);
int templateStart = 0;
int i = 0;
while (i < numArgs) {
int placeholderStart = template.indexOf("%s", templateStart);
if (placeholderStart == -1) {
break;
}
builder.append(template.substring(templateStart, placeholderStart));
builder.append(args[i++]);
templateStart = placeholderStart + 2;
}
builder.append(template.substring(templateStart));
// if we run out of placeholders, append the extra args in square braces
if (i < numArgs) {
builder.append(" [");
builder.append(args[i++]);
while (i < numArgs) {
builder.append(", ");
builder.append(args[i++]);
}
builder.append(']');
}
return builder.toString();
}
// ------------------------------------------------------------------------
/** Private constructor to prevent instantiation. */
private Preconditions() {}
} }

115
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PreconditionsTest.java

@ -33,11 +33,9 @@ public class PreconditionsTest {
*/ */
@Test @Test
public void testCheckNotNull() throws Exception { public void testCheckNotNull() throws Exception {
String testReference = "test reference"; String testReference = "test object";
//test reference is not null
Assert.assertEquals(testReference, Preconditions.checkNotNull(testReference)); Assert.assertEquals(testReference, Preconditions.checkNotNull(testReference));
Assert.assertEquals(testReference,Preconditions.checkNotNull(testReference,"reference is null")); Assert.assertEquals(testReference,Preconditions.checkNotNull(testReference,"object is null"));
Assert.assertEquals(testReference,Preconditions.checkNotNull(testReference,"%s is null",testReference));
//test reference is null //test reference is null
try { try {
@ -51,120 +49,19 @@ public class PreconditionsTest {
} catch (NullPointerException ex) { } catch (NullPointerException ex) {
assertNull(ex.getMessage()); assertNull(ex.getMessage());
} }
//test reference is null ,expect contains errorMessage
try {
Preconditions.checkNotNull(null,"reference is null");
} catch (NullPointerException ex) {
assertThat(ex.getMessage(), containsString("reference is null"));
}
try {
Preconditions.checkNotNull("","reference is null");
} catch (NullPointerException ex) {
assertThat(ex.getMessage(), containsString("reference is null"));
}
//test reference is null ,expect contains errorMessageTemplate and errorMessageArgs
try { try {
Preconditions.checkNotNull(null,"%s is null",testReference); Preconditions.checkNotNull(null,"object is null");
} catch (NullPointerException ex) { } catch (NullPointerException ex) {
assertThat(ex.getMessage(), containsString(testReference + " is null")); assertThat(ex.getMessage(), containsString("object is null"));
} }
try { try {
Preconditions.checkNotNull("","%s is null",testReference); Preconditions.checkNotNull("","object is null");
} catch (NullPointerException ex) { } catch (NullPointerException ex) {
assertThat(ex.getMessage(), containsString(testReference + " is null")); assertThat(ex.getMessage(), containsString("object is null"));
}
}
/**
* Test checkArgument
*/
@Test
public void testCheckArgument() throws Exception {
int argument = 100;
//boolean condition is true
Preconditions.checkArgument(argument > 0 && argument < 200);
//boolean condition is false
try {
Preconditions.checkArgument(argument > 0 && argument < 50);
} catch (IllegalArgumentException ex) {
assertNull(ex.getMessage());
}
//boolean condition is false ,expect contains errorMessage
try {
Preconditions.checkArgument(argument > 300, "argument is error");
} catch (IllegalArgumentException ex) {
assertThat(ex.getMessage(), containsString("argument is error"));
}
//boolean condition is false,expect contains errorMessageTemplate and errorMessageArgs
try {
Preconditions.checkArgument(argument > 0 && argument < 99, "argument %s is error",argument);
} catch (IllegalArgumentException ex) {
assertThat(ex.getMessage(), containsString( "argument " + argument + " is error"));
}
}
/**
* Test checkState
*/
@Test
public void testCheckState() throws Exception {
int state = 1;
//boolean condition is true
Preconditions.checkState(state == 1);
Preconditions.checkState(state > -1);
//boolean condition is false
try {
Preconditions.checkState(state > 2);
} catch (IllegalStateException ex) {
assertNull(ex.getMessage());
}
//boolean condition is false ,expect contains errorMessage
try {
Preconditions.checkState(state < 1, "state is error");
} catch (IllegalStateException ex) {
assertThat(ex.getMessage(), containsString("state is error"));
} }
//boolean condition is false,expect contains errorMessageTemplate and errorMessageArgs
try {
Preconditions.checkState(state < -1 , "state %s is error",state);
} catch (IllegalStateException ex) {
assertThat(ex.getMessage(), containsString( "state " + state + " is error"));
}
} }
/**
* Test checkElementIndex
*/
@Test
public void testCheckElementIndex() throws Exception {
int index = 2;
int size = 30;
//boolean condition is true
Preconditions.checkElementIndex(index, size);
//boolean condition is false
try {
Preconditions.checkElementIndex(-1, 10);
} catch (IndexOutOfBoundsException ex) {
assertThat(ex.getMessage(), containsString("Index: -1, Size: 10"));
}
//boolean condition is false ,expect contains errorMessage
try {
Preconditions.checkElementIndex(100, 50, "index is greater than size");
} catch (IndexOutOfBoundsException ex) {
assertThat(ex.getMessage(), containsString("index is greater than size Index: 100, Size: 50"));
}
}
} }

2
dolphinscheduler-dao/pom.xml

@ -21,7 +21,7 @@
<parent> <parent>
<groupId>org.apache.dolphinscheduler</groupId> <groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId> <artifactId>dolphinscheduler</artifactId>
<version>1.3.0-SNAPSHOT</version> <version>1.3.2-SNAPSHOT</version>
</parent> </parent>
<artifactId>dolphinscheduler-dao</artifactId> <artifactId>dolphinscheduler-dao</artifactId>
<name>${project.artifactId}</name> <name>${project.artifactId}</name>

53
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java

@ -14,10 +14,16 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.dao.datasource; package org.apache.dolphinscheduler.dao.datasource;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
/** /**
* data source of hive * data source of hive
@ -40,4 +46,51 @@ public class HiveDataSource extends BaseDataSource {
public DbType dbTypeSelector() { public DbType dbTypeSelector() {
return DbType.HIVE; return DbType.HIVE;
} }
/**
* build hive jdbc params,append : ?hive_conf_list
*
* hive jdbc url template:
*
* jdbc:hive2://<host1>:<port1>,<host2>:<port2>/dbName;initFile=<file>;sess_var_list?hive_conf_list#hive_var_list
*
* @param otherParams otherParams
* @return filter otherParams
*/
@Override
protected String filterOther(String otherParams) {
if (StringUtils.isBlank(otherParams)) {
return "";
}
StringBuilder hiveConfListSb = new StringBuilder();
hiveConfListSb.append("?");
StringBuilder sessionVarListSb = new StringBuilder();
String[] otherArray = otherParams.split(";", -1);
// get the default hive conf var name
Set<String> hiveConfSet = Stream.of(ConfVars.values()).map(confVars -> confVars.varname)
.collect(Collectors.toSet());
for (String conf : otherArray) {
if (hiveConfSet.contains(conf.split("=")[0])) {
hiveConfListSb.append(conf).append(";");
} else {
sessionVarListSb.append(conf).append(";");
}
}
// remove the last ";"
if (sessionVarListSb.length() > 0) {
sessionVarListSb.deleteCharAt(sessionVarListSb.length() - 1);
}
if (hiveConfListSb.length() > 0) {
hiveConfListSb.deleteCharAt(hiveConfListSb.length() - 1);
}
return sessionVarListSb.toString() + hiveConfListSb.toString();
}
} }

16
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java

@ -16,6 +16,7 @@
*/ */
package org.apache.dolphinscheduler.dao.entity; package org.apache.dolphinscheduler.dao.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableField;
import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonFormat;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
@ -27,10 +28,11 @@ import org.apache.dolphinscheduler.common.utils.*;
import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName; import com.baomidou.mybatisplus.annotation.TableName;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import java.io.Serializable; import java.io.Serializable;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.Map;
/** /**
* task instance * task instance
@ -213,7 +215,7 @@ public class TaskInstance implements Serializable {
@TableField(exist = false) @TableField(exist = false)
private List<String> resources; private Map<String,String> resources;
@ -455,10 +457,14 @@ public class TaskInstance implements Serializable {
|| (this.getState().typeIsFailure() && !taskCanRetry()); || (this.getState().typeIsFailure() && !taskCanRetry());
} }
public List<String> getResources() { public Map<String, String> getResources() {
return resources; return resources;
} }
public void setResources(Map<String, String> resources) {
this.resources = resources;
}
public boolean isSubProcess(){ public boolean isSubProcess(){
return TaskType.SUB_PROCESS.equals(TaskType.valueOf(this.taskType)); return TaskType.SUB_PROCESS.equals(TaskType.valueOf(this.taskType));
} }
@ -471,9 +477,7 @@ public class TaskInstance implements Serializable {
return TaskType.CONDITIONS.equals(TaskType.valueOf(this.taskType)); return TaskType.CONDITIONS.equals(TaskType.valueOf(this.taskType));
} }
public void setResources(List<String> resources) {
this.resources = resources;
}
/** /**
* determine if you can try again * determine if you can try again

85
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSourceTest.java

@ -0,0 +1,85 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.datasource;
import org.junit.Assert;
import org.junit.Test;
/**
* test data source of hive
*/
public class HiveDataSourceTest {
@Test
public void testfilterOther() {
BaseDataSource hiveDataSource = new HiveDataSource();
// not contain hive_site_conf
String other = hiveDataSource.filterOther("charset=UTF-8");
Assert.assertEquals("charset=UTF-8", other);
// not contain
other = hiveDataSource.filterOther("");
Assert.assertEquals("", other);
// only contain hive_site_conf
other = hiveDataSource.filterOther("hive.mapred.mode=strict");
Assert.assertEquals("?hive.mapred.mode=strict", other);
// contain hive_site_conf at the first
other = hiveDataSource.filterOther("hive.mapred.mode=strict;charset=UTF-8");
Assert.assertEquals("charset=UTF-8?hive.mapred.mode=strict", other);
// contain hive_site_conf in the middle
other = hiveDataSource.filterOther("charset=UTF-8;hive.mapred.mode=strict;foo=bar");
Assert.assertEquals("charset=UTF-8;foo=bar?hive.mapred.mode=strict", other);
// contain hive_site_conf at the end
other = hiveDataSource.filterOther("charset=UTF-8;foo=bar;hive.mapred.mode=strict");
Assert.assertEquals("charset=UTF-8;foo=bar?hive.mapred.mode=strict", other);
// contain multi hive_site_conf
other = hiveDataSource.filterOther("charset=UTF-8;foo=bar;hive.mapred.mode=strict;hive.exec.parallel=true");
Assert.assertEquals("charset=UTF-8;foo=bar?hive.mapred.mode=strict;hive.exec.parallel=true", other);
}
@Test
public void testGetHiveJdbcUrlOther() {
BaseDataSource hiveDataSource = new HiveDataSource();
hiveDataSource.setAddress("jdbc:hive2://127.0.0.1:10000");
hiveDataSource.setDatabase("test");
hiveDataSource.setPassword("123456");
hiveDataSource.setUser("test");
Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test", hiveDataSource.getJdbcUrl());
hiveDataSource.setOther("charset=UTF-8;hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2");
Assert.assertEquals(
"jdbc:hive2://127.0.0.1:10000/test;charset=UTF-8?hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2",
hiveDataSource.getJdbcUrl());
hiveDataSource.setOther("hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2");
Assert.assertEquals(
"jdbc:hive2://127.0.0.1:10000/test;?hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2",
hiveDataSource.getJdbcUrl());
}
}

2
dolphinscheduler-dist/pom.xml vendored

@ -20,7 +20,7 @@
<parent> <parent>
<artifactId>dolphinscheduler</artifactId> <artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId> <groupId>org.apache.dolphinscheduler</groupId>
<version>1.3.0-SNAPSHOT</version> <version>1.3.2-SNAPSHOT</version>
</parent> </parent>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>

1
dolphinscheduler-dist/release-docs/LICENSE vendored

@ -228,6 +228,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
byte-buddy 1.9.10: https://mvnrepository.com/artifact/net.bytebuddy/byte-buddy/1.9.10, Apache 2.0 byte-buddy 1.9.10: https://mvnrepository.com/artifact/net.bytebuddy/byte-buddy/1.9.10, Apache 2.0
classmate 1.4.0: https://mvnrepository.com/artifact/com.fasterxml/classmate/1.4.0, Apache 2.0 classmate 1.4.0: https://mvnrepository.com/artifact/com.fasterxml/classmate/1.4.0, Apache 2.0
clickhouse-jdbc 0.1.52: https://mvnrepository.com/artifact/ru.yandex.clickhouse/clickhouse-jdbc/0.1.52, Apache 2.0 clickhouse-jdbc 0.1.52: https://mvnrepository.com/artifact/ru.yandex.clickhouse/clickhouse-jdbc/0.1.52, Apache 2.0
commons-beanutils 1.7.0 https://mvnrepository.com/artifact/commons-beanutils/commons-beanutils/1.7.0, Apache 2.0
commons-cli 1.2: https://mvnrepository.com/artifact/commons-cli/commons-cli/1.2, Apache 2.0 commons-cli 1.2: https://mvnrepository.com/artifact/commons-cli/commons-cli/1.2, Apache 2.0
commons-codec 1.6: https://mvnrepository.com/artifact/commons-codec/commons-codec/1.6, Apache 2.0 commons-codec 1.6: https://mvnrepository.com/artifact/commons-codec/commons-codec/1.6, Apache 2.0
commons-collections 3.2.2: https://mvnrepository.com/artifact/commons-collections/commons-collections/3.2.2, Apache 2.0 commons-collections 3.2.2: https://mvnrepository.com/artifact/commons-collections/commons-collections/3.2.2, Apache 2.0

347
dolphinscheduler-dist/release-docs/NOTICE vendored

@ -132,18 +132,6 @@ granted provided that the copyright notice appears in all copies.
======================================================================== ========================================================================
Apache Log4j NOTICE
========================================================================
Apache log4j
Copyright 2007 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
========================================================================
Joda Time NOTICE Joda Time NOTICE
======================================================================== ========================================================================
@ -438,17 +426,6 @@ under the Apache License 2.0 (see: StringUtils.containsWhitespace())
======================================================================== ========================================================================
Apache Jakarta Commons Lang NOTICE
========================================================================
Apache Jakarta Commons Lang
Copyright 2001-2007 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).
========================================================================
Apache ZooKeeper NOTICE Apache ZooKeeper NOTICE
======================================================================== ========================================================================
@ -672,109 +649,6 @@ The licenses for these third party components are included in LICENSE.txt
======================================================================== ========================================================================
Apache Commons CLI NOTICE
========================================================================
Apache Commons CLI
Copyright 2001-2009 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).
========================================================================
Apache Commons Collections NOTICE
========================================================================
Apache Commons Collections
Copyright 2001-2015 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).
========================================================================
Apache Commons Collections4 NOTICE
========================================================================
Apache Commons Collections
Copyright 2001-2019 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
========================================================================
Apache Commons Compress NOTICE
========================================================================
Apache Commons Compress
Copyright 2002-2012 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).
========================================================================
Apache Commons Configuration NOTICE
========================================================================
Apache Commons Configuration
Copyright 2001-2013 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
========================================================================
Apache Commons Daemon NOTICE
========================================================================
Copyright 1999-2019 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).
========================================================================
Apache Commons Email NOTICE
========================================================================
Apache Commons Email
Copyright 2001-2017 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
========================================================================
Apache HttpComponents Client NOTICE
========================================================================
Copyright 1999-2018 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
========================================================================
Apache Commons IO NOTICE
========================================================================
Copyright 2002-2019 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
========================================================================
Commons Logging NOTICE Commons Logging NOTICE
======================================================================== ========================================================================
@ -872,18 +746,6 @@ file.
======================================================================== ========================================================================
Apache Commons Pool NOTICE
========================================================================
Apache Commons Pool
Copyright 2001-2012 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache Derby NOTICE Apache Derby NOTICE
=========================================================================== ===========================================================================
@ -1176,18 +1038,6 @@ No other notice covers that jar file.
========================================================================= =========================================================================
Apache HttpClient NOTICE
=========================================================================
Apache HttpComponents Client
Copyright 1999-2015 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache Httpcomponents core NOTICE Apache Httpcomponents core NOTICE
========================================================================= =========================================================================
@ -1229,28 +1079,6 @@ from the source code management (SCM) system project uses.
========================================================================= =========================================================================
Apache Avro NOTICE
=========================================================================
Apache Avro
Copyright 2009-2013 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apach Commons IO NOTICE
=========================================================================
Apache Commons IO
Copyright 2002-2012 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Junit4 NOTICE Junit4 NOTICE
========================================================================= =========================================================================
@ -1335,39 +1163,6 @@ Junit4 NOTICE
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
=========================================================================
Apache Thrift NOTICE
=========================================================================
Apache Thrift
Copyright 2006-2010 The Apache Software Foundation.
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache Commons DBCP NOTICE
=========================================================================
Apache Commons DBCP
Copyright 2001-2010 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache Commons Daemon NOTICE
=========================================================================
Apache Commons Daemon
Copyright 1999-2013 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).
========================================================================= =========================================================================
Bonecp NOTICE Bonecp NOTICE
@ -1500,28 +1295,6 @@ Licensed under the Apache License, Version 2.0 (the "License");
========================================================================= =========================================================================
Apache EL NOTICE
=========================================================================
Apache Tomcat
Copyright 1999-2018 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache Commons Net NOTICE
=========================================================================
Apache Commons Net
Copyright 2001-2012 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache Logging Log4j2 NOTICE Apache Logging Log4j2 NOTICE
========================================================================= =========================================================================
@ -1586,50 +1359,6 @@ Portions of this software were originally based on the following:
========================================================================= =========================================================================
Apache Parquet Hadoop Bundle NOTICE
=========================================================================
Apache Parquet Hadoop Bundle
Copyright 2015 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache Log4j 1.x NOTICE
=========================================================================
Apache Log4j 1.x Compatibility API
Copyright 1999-2019 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache log4j NOTICE
=========================================================================
Apache log4j
Copyright 2007 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache Java Data Objects (JDO) NOTICE
=========================================================================
Apache Java Data Objects (JDO)
Copyright 2005-2006 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Jackson XC NOTICE Jackson XC NOTICE
========================================================================= =========================================================================
@ -1656,17 +1385,6 @@ see CREDITS file.
========================================================================= =========================================================================
Apache HttpClient Mime NOTICE
=========================================================================
Apache HttpClient Mime
Copyright 1999-2019 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache HttpCore NOTICE Apache HttpCore NOTICE
========================================================================= =========================================================================
@ -1693,17 +1411,6 @@ The Apache Software Foundation (http://www.apache.org/).
========================================================================= =========================================================================
Hive Storage API NOTICE
=========================================================================
Apache Hive
Copyright 2016 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Hadoop NOTICE Hadoop NOTICE
========================================================================= =========================================================================
@ -2190,60 +1897,6 @@ Other developers who have contributed code are:
========================================================================= =========================================================================
Apache Yetus NOTICE
=========================================================================
Apache Yetus - Audience Annotations
Copyright 2015-2017 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache Directory LDAP API Utilities NOTICE
=========================================================================
Apache Directory LDAP API Utilities
Copyright 2003-2013 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
Apache Directory API ASN.1 API NOTICE
=========================================================================
Apache Directory API ASN.1 API
Copyright 2003-2013 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
ApacheDS Protocol Kerberos Codec NOTICE
=========================================================================
ApacheDS Protocol Kerberos Codec
Copyright 2003-2013 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
=========================================================================
ApacheDS I18n NOTICE
=========================================================================
ApacheDS I18n
Copyright 2003-2013 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
ANT NOTICE ANT NOTICE
========================================================================= =========================================================================

202
dolphinscheduler-dist/release-docs/licenses/LICENSE-commons-beanutils.txt vendored

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

2
dolphinscheduler-microbench/pom.xml

@ -21,7 +21,7 @@
<parent> <parent>
<artifactId>dolphinscheduler</artifactId> <artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId> <groupId>org.apache.dolphinscheduler</groupId>
<version>1.3.0-SNAPSHOT</version> <version>1.3.2-SNAPSHOT</version>
</parent> </parent>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>

2
dolphinscheduler-plugin-api/pom.xml

@ -23,7 +23,7 @@
<parent> <parent>
<groupId>org.apache.dolphinscheduler</groupId> <groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId> <artifactId>dolphinscheduler</artifactId>
<version>1.3.0-SNAPSHOT</version> <version>1.3.2-SNAPSHOT</version>
</parent> </parent>
<artifactId>dolphinscheduler-plugin-api</artifactId> <artifactId>dolphinscheduler-plugin-api</artifactId>
<name>${project.artifactId}</name> <name>${project.artifactId}</name>

2
dolphinscheduler-remote/pom.xml

@ -20,7 +20,7 @@
<parent> <parent>
<artifactId>dolphinscheduler</artifactId> <artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId> <groupId>org.apache.dolphinscheduler</groupId>
<version>1.3.0-SNAPSHOT</version> <version>1.3.2-SNAPSHOT</version>
</parent> </parent>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>

55
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java

@ -26,69 +26,34 @@ public class RemotingException extends Exception {
super(); super();
} }
/** Constructs a new runtime exception with the specified detail message. /**
* The cause is not initialized, and may subsequently be initialized by a * Construct a new runtime exception with the detail message
* call to {@link #initCause}.
* *
* @param message the detail message. The detail message is saved for * @param message detail message
* later retrieval by the {@link #getMessage()} method.
*/ */
public RemotingException(String message) { public RemotingException(String message) {
super(message); super(message);
} }
/** /**
* Constructs a new runtime exception with the specified detail message and * Construct a new runtime exception with the detail message and cause
* cause. <p>Note that the detail message associated with
* {@code cause} is <i>not</i> automatically incorporated in
* this runtime exception's detail message.
* *
* @param message the detail message (which is saved for later retrieval * @param message the detail message
* by the {@link #getMessage()} method). * @param cause the cause
* @param cause the cause (which is saved for later retrieval by the
* {@link #getCause()} method). (A <tt>null</tt> value is
* permitted, and indicates that the cause is nonexistent or
* unknown.)
* @since 1.4 * @since 1.4
*/ */
public RemotingException(String message, Throwable cause) { public RemotingException(String message, Throwable cause) {
super(message, cause); super(message, cause);
} }
/** Constructs a new runtime exception with the specified cause and a /**
* detail message of <tt>(cause==null ? null : cause.toString())</tt> * Construct a new runtime exception with throwable
* (which typically contains the class and detail message of
* <tt>cause</tt>). This constructor is useful for runtime exceptions
* that are little more than wrappers for other throwables.
* *
* @param cause the cause (which is saved for later retrieval by the * @param cause the cause
* {@link #getCause()} method). (A <tt>null</tt> value is
* permitted, and indicates that the cause is nonexistent or
* unknown.)
* @since 1.4
*/ */
public RemotingException(Throwable cause) { public RemotingException(Throwable cause) {
super(cause); super(cause);
} }
/**
* Constructs a new runtime exception with the specified detail
* message, cause, suppression enabled or disabled, and writable
* stack trace enabled or disabled.
*
* @param message the detail message.
* @param cause the cause. (A {@code null} value is permitted,
* and indicates that the cause is nonexistent or unknown.)
* @param enableSuppression whether or not suppression is enabled
* or disabled
* @param writableStackTrace whether or not the stack trace should
* be writable
*
* @since 1.7
*/
protected RemotingException(String message, Throwable cause,
boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
} }

30
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java

@ -170,21 +170,6 @@ public class ResponseFuture {
} }
} }
@Override
public String toString() {
return "ResponseFuture{" +
"opaque=" + opaque +
", timeoutMillis=" + timeoutMillis +
", invokeCallback=" + invokeCallback +
", releaseSemaphore=" + releaseSemaphore +
", latch=" + latch +
", beginTimestamp=" + beginTimestamp +
", responseCommand=" + responseCommand +
", sendOk=" + sendOk +
", cause=" + cause +
'}';
}
/** /**
* scan future table * scan future table
*/ */
@ -209,4 +194,19 @@ public class ResponseFuture {
} }
} }
} }
@Override
public String toString() {
return "ResponseFuture{" +
"opaque=" + opaque +
", timeoutMillis=" + timeoutMillis +
", invokeCallback=" + invokeCallback +
", releaseSemaphore=" + releaseSemaphore +
", latch=" + latch +
", beginTimestamp=" + beginTimestamp +
", responseCommand=" + responseCommand +
", sendOk=" + sendOk +
", cause=" + cause +
'}';
}
} }

2
dolphinscheduler-server/pom.xml

@ -21,7 +21,7 @@
<parent> <parent>
<groupId>org.apache.dolphinscheduler</groupId> <groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId> <artifactId>dolphinscheduler</artifactId>
<version>1.3.0-SNAPSHOT</version> <version>1.3.2-SNAPSHOT</version>
</parent> </parent>
<artifactId>dolphinscheduler-server</artifactId> <artifactId>dolphinscheduler-server</artifactId>
<name>dolphinscheduler-server</name> <name>dolphinscheduler-server</name>

16
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/SQLTaskExecutionContext.java

@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.server.entity;
import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import java.io.Serializable; import java.io.Serializable;
import java.util.List; import java.util.Map;
/** /**
* SQL Task ExecutionContext * SQL Task ExecutionContext
@ -38,9 +38,9 @@ public class SQLTaskExecutionContext implements Serializable {
*/ */
private String connectionParams; private String connectionParams;
/** /**
* udf function list * udf function tenant code map
*/ */
private List<UdfFunc> udfFuncList; private Map<UdfFunc,String> udfFuncTenantCodeMap;
public int getWarningGroupId() { public int getWarningGroupId() {
@ -51,12 +51,12 @@ public class SQLTaskExecutionContext implements Serializable {
this.warningGroupId = warningGroupId; this.warningGroupId = warningGroupId;
} }
public List<UdfFunc> getUdfFuncList() { public Map<UdfFunc, String> getUdfFuncTenantCodeMap() {
return udfFuncList; return udfFuncTenantCodeMap;
} }
public void setUdfFuncList(List<UdfFunc> udfFuncList) { public void setUdfFuncTenantCodeMap(Map<UdfFunc, String> udfFuncTenantCodeMap) {
this.udfFuncList = udfFuncList; this.udfFuncTenantCodeMap = udfFuncTenantCodeMap;
} }
public String getConnectionParams() { public String getConnectionParams() {
@ -72,7 +72,7 @@ public class SQLTaskExecutionContext implements Serializable {
return "SQLTaskExecutionContext{" + return "SQLTaskExecutionContext{" +
"warningGroupId=" + warningGroupId + "warningGroupId=" + warningGroupId +
", connectionParams='" + connectionParams + '\'' + ", connectionParams='" + connectionParams + '\'' +
", udfFuncList=" + udfFuncList + ", udfFuncTenantCodeMap=" + udfFuncTenantCodeMap +
'}'; '}';
} }
} }

9
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java

@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.remote.utils.JsonSerializer;
import java.io.Serializable; import java.io.Serializable;
import java.util.Date; import java.util.Date;
import java.util.List;
import java.util.Map; import java.util.Map;
/** /**
@ -171,9 +170,9 @@ public class TaskExecutionContext implements Serializable{
private String workerGroup; private String workerGroup;
/** /**
* resources full name * resources full name and tenant code
*/ */
private List<String> resources; private Map<String,String> resources;
/** /**
* sql TaskExecutionContext * sql TaskExecutionContext
@ -446,11 +445,11 @@ public class TaskExecutionContext implements Serializable{
this.dependenceTaskExecutionContext = dependenceTaskExecutionContext; this.dependenceTaskExecutionContext = dependenceTaskExecutionContext;
} }
public List<String> getResources() { public Map<String, String> getResources() {
return resources; return resources;
} }
public void setResources(List<String> resources) { public void setResources(Map<String, String> resources) {
this.resources = resources; this.resources = resources;
} }

36
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java

@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.server.master.consumer;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.SqoopJobType; import org.apache.dolphinscheduler.common.enums.SqoopJobType;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.enums.TaskType;
import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNode;
@ -49,14 +50,10 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
import java.util.ArrayList; import java.util.*;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import static org.apache.dolphinscheduler.common.Constants.SLEEP_TIME_MILLIS;
/** /**
* TaskUpdateQueue consumer * TaskUpdateQueue consumer
*/ */
@ -330,7 +327,13 @@ public class TaskPriorityQueueConsumer extends Thread{
} }
List<UdfFunc> udfFuncList = processService.queryUdfFunListByids(udfFunIdsArray); List<UdfFunc> udfFuncList = processService.queryUdfFunListByids(udfFunIdsArray);
sqlTaskExecutionContext.setUdfFuncList(udfFuncList); Map<UdfFunc,String> udfFuncMap = new HashMap<>();
for(UdfFunc udfFunc : udfFuncList) {
String tenantCode = processService.queryTenantCodeByResName(udfFunc.getResourceName(), ResourceType.UDF);
udfFuncMap.put(udfFunc,tenantCode);
}
sqlTaskExecutionContext.setUdfFuncTenantCodeMap(udfFuncMap);
} }
} }
@ -364,20 +367,23 @@ public class TaskPriorityQueueConsumer extends Thread{
} }
/** /**
* get resource full name list * get resource map key is full name and value is tenantCode
*/ */
private List<String> getResourceFullNames(TaskNode taskNode) { private Map<String,String> getResourceFullNames(TaskNode taskNode) {
List<String> resourceFullNameList = new ArrayList<>(); Map<String,String> resourceMap = new HashMap<>();
AbstractParameters baseParam = TaskParametersUtils.getParameters(taskNode.getType(), taskNode.getParams()); AbstractParameters baseParam = TaskParametersUtils.getParameters(taskNode.getType(), taskNode.getParams());
if (baseParam != null) { if (baseParam != null) {
List<ResourceInfo> projectResourceFiles = baseParam.getResourceFilesList(); List<ResourceInfo> projectResourceFiles = baseParam.getResourceFilesList();
if (projectResourceFiles != null) { if (CollectionUtils.isNotEmpty(projectResourceFiles)) {
// filter the resources that the resource id equals 0 // filter the resources that the resource id equals 0
Set<ResourceInfo> oldVersionResources = projectResourceFiles.stream().filter(t -> t.getId() == 0).collect(Collectors.toSet()); Set<ResourceInfo> oldVersionResources = projectResourceFiles.stream().filter(t -> t.getId() == 0).collect(Collectors.toSet());
if (CollectionUtils.isNotEmpty(oldVersionResources)) { if (CollectionUtils.isNotEmpty(oldVersionResources)) {
resourceFullNameList.addAll(oldVersionResources.stream().map(resource -> resource.getRes()).collect(Collectors.toSet()));
oldVersionResources.forEach(
(t)->resourceMap.put(t.getRes(), processService.queryTenantCodeByResName(t.getRes(), ResourceType.FILE))
);
} }
// get the resource id in order to get the resource names in batch // get the resource id in order to get the resource names in batch
@ -388,13 +394,13 @@ public class TaskPriorityQueueConsumer extends Thread{
Integer[] resourceIds = resourceIdsSet.toArray(new Integer[resourceIdsSet.size()]); Integer[] resourceIds = resourceIdsSet.toArray(new Integer[resourceIdsSet.size()]);
List<Resource> resources = processService.listResourceByIds(resourceIds); List<Resource> resources = processService.listResourceByIds(resourceIds);
resourceFullNameList.addAll(resources.stream() resources.forEach(
.map(resourceInfo -> resourceInfo.getFullName()) (t)->resourceMap.put(t.getFullName(),processService.queryTenantCodeByResName(t.getFullName(), ResourceType.FILE))
.collect(Collectors.toList())); );
} }
} }
} }
return resourceFullNameList; return resourceMap;
} }
} }

53
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/exceptions/ExecuteException.java

@ -26,68 +26,21 @@ public class ExecuteException extends Exception{
super(); super();
} }
/**
* Constructs a new exception with the specified detail message. The
* cause is not initialized, and may subsequently be initialized by
* a call to {@link #initCause}.
*
* @param message the detail message. The detail message is saved for
* later retrieval by the {@link #getMessage()} method.
*/
public ExecuteException(String message) { public ExecuteException(String message) {
super(message); super(message);
} }
/**
* Constructs a new exception with the specified detail message and
* cause. <p>Note that the detail message associated with
* {@code cause} is <i>not</i> automatically incorporated in
* this exception's detail message.
*
* @param message the detail message (which is saved for later retrieval
* by the {@link #getMessage()} method).
* @param cause the cause (which is saved for later retrieval by the
* {@link #getCause()} method). (A <tt>null</tt> value is
* permitted, and indicates that the cause is nonexistent or
* unknown.)
* @since 1.4
*/
public ExecuteException(String message, Throwable cause) { public ExecuteException(String message, Throwable cause) {
super(message, cause); super(message, cause);
} }
/**
* Constructs a new exception with the specified cause and a detail
* message of <tt>(cause==null ? null : cause.toString())</tt> (which
* typically contains the class and detail message of <tt>cause</tt>).
* This constructor is useful for exceptions that are little more than
* wrappers for other throwables (for example, {@link
* java.security.PrivilegedActionException}).
*
* @param cause the cause (which is saved for later retrieval by the
* {@link #getCause()} method). (A <tt>null</tt> value is
* permitted, and indicates that the cause is nonexistent or
* unknown.)
* @since 1.4
*/
public ExecuteException(Throwable cause) { public ExecuteException(Throwable cause) {
super(cause); super(cause);
} }
/**
* Constructs a new exception with the specified detail message,
* cause, suppression enabled or disabled, and writable stack
* trace enabled or disabled.
*
* @param message the detail message.
* @param cause the cause. (A {@code null} value is permitted,
* and indicates that the cause is nonexistent or unknown.)
* @param enableSuppression whether or not suppression is enabled
* or disabled
* @param writableStackTrace whether or not the stack trace should
* be writable
* @since 1.7
*/
protected ExecuteException(String message, Throwable cause, protected ExecuteException(String message, Throwable cause,
boolean enableSuppression, boolean enableSuppression,
boolean writableStackTrace) { boolean writableStackTrace) {

28
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java

@ -17,6 +17,8 @@
package org.apache.dolphinscheduler.server.master.dispatch.executor; package org.apache.dolphinscheduler.server.master.dispatch.executor;
import com.github.rholder.retry.RetryException;
import org.apache.dolphinscheduler.common.utils.RetryerUtils;
import org.apache.dolphinscheduler.remote.NettyRemotingClient; import org.apache.dolphinscheduler.remote.NettyRemotingClient;
import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.CommandType;
@ -36,6 +38,7 @@ import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
import java.util.*; import java.util.*;
import java.util.concurrent.ExecutionException;
/** /**
* netty executor manager * netty executor manager
@ -133,26 +136,13 @@ public class NettyExecutorManager extends AbstractExecutorManager<Boolean>{
* @throws ExecuteException if error throws ExecuteException * @throws ExecuteException if error throws ExecuteException
*/ */
private void doExecute(final Host host, final Command command) throws ExecuteException { private void doExecute(final Host host, final Command command) throws ExecuteException {
/** try {
* retry countdefault retry 3 RetryerUtils.retryCall(() -> {
*/
int retryCount = 3;
boolean success = false;
do {
try {
nettyRemotingClient.send(host, command); nettyRemotingClient.send(host, command);
success = true; return Boolean.TRUE;
} catch (Exception ex) { });
logger.error(String.format("send command : %s to %s error", command, host), ex); } catch (ExecutionException | RetryException e) {
retryCount--; throw new ExecuteException(String.format("send command : %s to %s error", command, host), e);
try {
Thread.sleep(100);
} catch (InterruptedException ignore) {}
}
} while (retryCount >= 0 && !success);
if (!success) {
throw new ExecuteException(String.format("send command : %s to %s error", command, host));
} }
} }

19
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistry.java

@ -16,10 +16,16 @@
*/ */
package org.apache.dolphinscheduler.server.master.registry; package org.apache.dolphinscheduler.server.master.registry;
import java.util.Date;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import javax.annotation.PostConstruct;
import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.state.ConnectionState; import org.apache.curator.framework.state.ConnectionState;
import org.apache.curator.framework.state.ConnectionStateListener; import org.apache.curator.framework.state.ConnectionStateListener;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory; import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory;
@ -31,13 +37,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct; import com.google.common.collect.Sets;
import java.util.Date;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import static org.apache.dolphinscheduler.remote.utils.Constants.COMMA;
/** /**
* master registry * master registry
@ -100,7 +100,7 @@ public class MasterRegistry {
HeartBeatTask heartBeatTask = new HeartBeatTask(startTime, HeartBeatTask heartBeatTask = new HeartBeatTask(startTime,
masterConfig.getMasterReservedMemory(), masterConfig.getMasterReservedMemory(),
masterConfig.getMasterMaxCpuloadAvg(), masterConfig.getMasterMaxCpuloadAvg(),
getMasterPath(), Sets.newHashSet(getMasterPath()),
zookeeperRegistryCenter); zookeeperRegistryCenter);
this.heartBeatExecutor.scheduleAtFixedRate(heartBeatTask, masterHeartbeatInterval, masterHeartbeatInterval, TimeUnit.SECONDS); this.heartBeatExecutor.scheduleAtFixedRate(heartBeatTask, masterHeartbeatInterval, masterHeartbeatInterval, TimeUnit.SECONDS);
@ -113,6 +113,7 @@ public class MasterRegistry {
public void unRegistry() { public void unRegistry() {
String address = getLocalAddress(); String address = getLocalAddress();
String localNodePath = getMasterPath(); String localNodePath = getMasterPath();
heartBeatExecutor.shutdownNow();
zookeeperRegistryCenter.getZookeeperCachedOperator().remove(localNodePath); zookeeperRegistryCenter.getZookeeperCachedOperator().remove(localNodePath);
logger.info("master node : {} unRegistry to ZK.", address); logger.info("master node : {} unRegistry to ZK.", address);
} }

33
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java

@ -17,50 +17,50 @@
package org.apache.dolphinscheduler.server.registry; package org.apache.dolphinscheduler.server.registry;
import static org.apache.dolphinscheduler.remote.utils.Constants.COMMA;
import java.util.Date;
import java.util.Set;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.util.Date; public class HeartBeatTask extends Thread {
import static org.apache.dolphinscheduler.remote.utils.Constants.COMMA;
public class HeartBeatTask extends Thread{
private final Logger logger = LoggerFactory.getLogger(HeartBeatTask.class); private final Logger logger = LoggerFactory.getLogger(HeartBeatTask.class);
private String startTime; private String startTime;
private double reservedMemory; private double reservedMemory;
private double maxCpuloadAvg; private double maxCpuloadAvg;
private String heartBeatPath; private Set<String> heartBeatPaths;
private ZookeeperRegistryCenter zookeeperRegistryCenter; private ZookeeperRegistryCenter zookeeperRegistryCenter;
public HeartBeatTask(String startTime, public HeartBeatTask(String startTime,
double reservedMemory, double reservedMemory,
double maxCpuloadAvg, double maxCpuloadAvg,
String heartBeatPath, Set<String> heartBeatPaths,
ZookeeperRegistryCenter zookeeperRegistryCenter){ ZookeeperRegistryCenter zookeeperRegistryCenter) {
this.startTime = startTime; this.startTime = startTime;
this.reservedMemory = reservedMemory; this.reservedMemory = reservedMemory;
this.maxCpuloadAvg = maxCpuloadAvg; this.maxCpuloadAvg = maxCpuloadAvg;
this.heartBeatPath = heartBeatPath; this.heartBeatPaths = heartBeatPaths;
this.zookeeperRegistryCenter = zookeeperRegistryCenter; this.zookeeperRegistryCenter = zookeeperRegistryCenter;
} }
@Override @Override
public void run() { public void run() {
try { try {
double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize();
double loadAverage = OSUtils.loadAverage(); double loadAverage = OSUtils.loadAverage();
int status = Constants.NORAML_NODE_STATUS; int status = Constants.NORAML_NODE_STATUS;
if(availablePhysicalMemorySize < reservedMemory if (availablePhysicalMemorySize < reservedMemory
|| loadAverage > maxCpuloadAvg){ || loadAverage > maxCpuloadAvg) {
logger.warn("load is too high or availablePhysicalMemorySize(G) is too low, it's availablePhysicalMemorySize(G):{},loadAvg:{}", availablePhysicalMemorySize , loadAverage); logger.warn("load is too high or availablePhysicalMemorySize(G) is too low, it's availablePhysicalMemorySize(G):{},loadAvg:{}", availablePhysicalMemorySize, loadAverage);
status = Constants.ABNORMAL_NODE_STATUS; status = Constants.ABNORMAL_NODE_STATUS;
} }
@ -76,8 +76,11 @@ public class HeartBeatTask extends Thread{
builder.append(status).append(COMMA); builder.append(status).append(COMMA);
//save process id //save process id
builder.append(OSUtils.getProcessID()); builder.append(OSUtils.getProcessID());
zookeeperRegistryCenter.getZookeeperCachedOperator().update(heartBeatPath, builder.toString());
} catch (Throwable ex){ for (String heartBeatPath : heartBeatPaths) {
zookeeperRegistryCenter.getZookeeperCachedOperator().update(heartBeatPath, builder.toString());
}
} catch (Throwable ex) {
logger.error("error write heartbeat info", ex); logger.error("error write heartbeat info", ex);
} }
} }

65
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/UDFUtils.java

@ -16,6 +16,7 @@
*/ */
package org.apache.dolphinscheduler.server.utils; package org.apache.dolphinscheduler.server.utils;
import org.apache.commons.collections.MapUtils;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils;
@ -24,10 +25,8 @@ import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.slf4j.Logger; import org.slf4j.Logger;
import java.text.MessageFormat; import java.text.MessageFormat;
import java.util.ArrayList; import java.util.*;
import java.util.HashSet; import java.util.stream.Collectors;
import java.util.List;
import java.util.Set;
import static org.apache.dolphinscheduler.common.utils.CollectionUtils.isNotEmpty; import static org.apache.dolphinscheduler.common.utils.CollectionUtils.isNotEmpty;
@ -43,53 +42,44 @@ public class UDFUtils {
/** /**
* create function list * create function list
* @param udfFuncs udf functions * @param udfFuncTenantCodeMap key is udf function,value is tenant code
* @param tenantCode tenant code * @param logger logger
* @param logger logger
* @return create function list * @return create function list
*/ */
public static List<String> createFuncs(List<UdfFunc> udfFuncs, String tenantCode,Logger logger){ public static List<String> createFuncs(Map<UdfFunc,String> udfFuncTenantCodeMap, Logger logger){
if (CollectionUtils.isEmpty(udfFuncs)){ if (MapUtils.isEmpty(udfFuncTenantCodeMap)){
logger.info("can't find udf function resource"); logger.info("can't find udf function resource");
return null; return null;
} }
// get hive udf jar path
String hiveUdfJarPath = HadoopUtils.getHdfsUdfDir(tenantCode);
logger.info("hive udf jar path : {}" , hiveUdfJarPath);
// is the root directory of udf defined
if (StringUtils.isEmpty(hiveUdfJarPath)) {
logger.error("not define hive udf jar path");
throw new RuntimeException("hive udf jar base path not defined ");
}
Set<String> resources = getFuncResouces(udfFuncs);
List<String> funcList = new ArrayList<>(); List<String> funcList = new ArrayList<>();
// build jar sql // build jar sql
buildJarSql(funcList, resources, hiveUdfJarPath); buildJarSql(funcList, udfFuncTenantCodeMap);
// build temp function sql // build temp function sql
buildTempFuncSql(funcList, udfFuncs); buildTempFuncSql(funcList, udfFuncTenantCodeMap.keySet().stream().collect(Collectors.toList()));
return funcList; return funcList;
} }
/** /**
* build jar sql * build jar sql
* @param sqls sql list * @param sqls sql list
* @param resources resource set * @param udfFuncTenantCodeMap key is udf function,value is tenant code
* @param uploadPath upload path
*/ */
private static void buildJarSql(List<String> sqls, Set<String> resources, String uploadPath) { private static void buildJarSql(List<String> sqls, Map<UdfFunc,String> udfFuncTenantCodeMap) {
String defaultFS = HadoopUtils.getInstance().getConfiguration().get(Constants.FS_DEFAULTFS); String defaultFS = HadoopUtils.getInstance().getConfiguration().get(Constants.FS_DEFAULTFS);
if (!uploadPath.startsWith("hdfs:")) {
uploadPath = defaultFS + uploadPath;
}
for (String resource : resources) { Set<Map.Entry<UdfFunc,String>> entries = udfFuncTenantCodeMap.entrySet();
sqls.add(String.format("add jar %s/%s", uploadPath, resource)); for (Map.Entry<UdfFunc,String> entry:entries){
String uploadPath = HadoopUtils.getHdfsUdfDir(entry.getValue());
if (!uploadPath.startsWith("hdfs:")) {
uploadPath = defaultFS + uploadPath;
}
sqls.add(String.format("add jar %s%s", uploadPath, entry.getKey().getResourceName()));
} }
} }
/** /**
@ -106,20 +96,5 @@ public class UDFUtils {
} }
} }
/**
* get the resource names of all functions
* @param udfFuncs udf function list
* @return
*/
private static Set<String> getFuncResouces(List<UdfFunc> udfFuncs) {
Set<String> resources = new HashSet<>();
for (UdfFunc udfFunc : udfFuncs) {
resources.add(udfFunc.getResourceName());
}
return resources;
}
} }

14
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java

@ -17,6 +17,8 @@
*/ */
package org.apache.dolphinscheduler.server.worker.config; package org.apache.dolphinscheduler.server.worker.config;
import java.util.Set;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.PropertySource;
@ -41,8 +43,8 @@ public class WorkerConfig {
@Value("${worker.reserved.memory:0.3}") @Value("${worker.reserved.memory:0.3}")
private double workerReservedMemory; private double workerReservedMemory;
@Value("${worker.group: default}") @Value("#{'${worker.groups:default}'.split(',')}")
private String workerGroup; private Set<String> workerGroups;
@Value("${worker.listen.port: 1234}") @Value("${worker.listen.port: 1234}")
private int listenPort; private int listenPort;
@ -55,12 +57,12 @@ public class WorkerConfig {
this.listenPort = listenPort; this.listenPort = listenPort;
} }
public String getWorkerGroup() { public Set<String> getWorkerGroups() {
return workerGroup; return workerGroups;
} }
public void setWorkerGroup(String workerGroup) { public void setWorkerGroups(Set<String> workerGroups) {
this.workerGroup = workerGroup; this.workerGroups = workerGroups;
} }
public int getWorkerExecThreads() { public int getWorkerExecThreads() {

115
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java

@ -16,10 +16,20 @@
*/ */
package org.apache.dolphinscheduler.server.worker.registry; package org.apache.dolphinscheduler.server.worker.registry;
import static org.apache.dolphinscheduler.common.Constants.DEFAULT_WORKER_GROUP;
import static org.apache.dolphinscheduler.common.Constants.SLASH;
import java.util.Date;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import javax.annotation.PostConstruct;
import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.state.ConnectionState; import org.apache.curator.framework.state.ConnectionState;
import org.apache.curator.framework.state.ConnectionStateListener; import org.apache.curator.framework.state.ConnectionStateListener;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.StringUtils;
@ -32,15 +42,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct; import com.google.common.collect.Sets;
import java.util.Date;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import static org.apache.dolphinscheduler.common.Constants.COMMA;
import static org.apache.dolphinscheduler.common.Constants.DEFAULT_WORKER_GROUP;
import static org.apache.dolphinscheduler.common.Constants.SLASH;
/** /**
@ -74,11 +76,11 @@ public class WorkerRegistry {
private String startTime; private String startTime;
private String workerGroup; private Set<String> workerGroups;
@PostConstruct @PostConstruct
public void init(){ public void init() {
this.workerGroup = workerConfig.getWorkerGroup(); this.workerGroups = workerConfig.getWorkerGroups();
this.startTime = DateUtils.dateToString(new Date()); this.startTime = DateUtils.dateToString(new Date());
this.heartBeatExecutor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("HeartBeatExecutor")); this.heartBeatExecutor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("HeartBeatExecutor"));
} }
@ -88,31 +90,35 @@ public class WorkerRegistry {
*/ */
public void registry() { public void registry() {
String address = NetUtils.getHost(); String address = NetUtils.getHost();
String localNodePath = getWorkerPath(); Set<String> workerZkPaths = getWorkerZkPaths();
zookeeperRegistryCenter.getZookeeperCachedOperator().persistEphemeral(localNodePath, "");
zookeeperRegistryCenter.getZookeeperCachedOperator().getZkClient().getConnectionStateListenable().addListener(new ConnectionStateListener() {
@Override
public void stateChanged(CuratorFramework client, ConnectionState newState) {
if(newState == ConnectionState.LOST){
logger.error("worker : {} connection lost from zookeeper", address);
} else if(newState == ConnectionState.RECONNECTED){
logger.info("worker : {} reconnected to zookeeper", address);
zookeeperRegistryCenter.getZookeeperCachedOperator().persistEphemeral(localNodePath, "");
} else if(newState == ConnectionState.SUSPENDED){
logger.warn("worker : {} connection SUSPENDED ", address);
}
}
});
int workerHeartbeatInterval = workerConfig.getWorkerHeartbeatInterval(); int workerHeartbeatInterval = workerConfig.getWorkerHeartbeatInterval();
HeartBeatTask heartBeatTask = new HeartBeatTask(startTime, for (String workerZKPath : workerZkPaths) {
workerConfig.getWorkerReservedMemory(), zookeeperRegistryCenter.getZookeeperCachedOperator().persistEphemeral(workerZKPath, "");
workerConfig.getWorkerMaxCpuloadAvg(), zookeeperRegistryCenter.getZookeeperCachedOperator().getZkClient().getConnectionStateListenable().addListener(new ConnectionStateListener() {
getWorkerPath(), @Override
zookeeperRegistryCenter); public void stateChanged(CuratorFramework client, ConnectionState newState) {
this.heartBeatExecutor.scheduleAtFixedRate(heartBeatTask, workerHeartbeatInterval, workerHeartbeatInterval, TimeUnit.SECONDS); if (newState == ConnectionState.LOST) {
logger.info("worker node : {} registry to ZK successfully with heartBeatInterval : {}s", address, workerHeartbeatInterval); logger.error("worker : {} connection lost from zookeeper", address);
} else if (newState == ConnectionState.RECONNECTED) {
logger.info("worker : {} reconnected to zookeeper", address);
zookeeperRegistryCenter.getZookeeperCachedOperator().persistEphemeral(workerZKPath, "");
} else if (newState == ConnectionState.SUSPENDED) {
logger.warn("worker : {} connection SUSPENDED ", address);
}
}
});
logger.info("worker node : {} registry to ZK {} successfully", address, workerZKPath);
}
HeartBeatTask heartBeatTask = new HeartBeatTask(this.startTime,
this.workerConfig.getWorkerReservedMemory(),
this.workerConfig.getWorkerMaxCpuloadAvg(),
workerZkPaths,
this.zookeeperRegistryCenter);
this.heartBeatExecutor.scheduleAtFixedRate(heartBeatTask, workerHeartbeatInterval, workerHeartbeatInterval, TimeUnit.SECONDS);
logger.info("worker node : {} heartbeat interval {} s", address, workerHeartbeatInterval);
} }
/** /**
@ -120,36 +126,41 @@ public class WorkerRegistry {
*/ */
public void unRegistry() { public void unRegistry() {
String address = getLocalAddress(); String address = getLocalAddress();
String localNodePath = getWorkerPath(); Set<String> workerZkPaths = getWorkerZkPaths();
zookeeperRegistryCenter.getZookeeperCachedOperator().remove(localNodePath); for (String workerZkPath : workerZkPaths) {
zookeeperRegistryCenter.getZookeeperCachedOperator().remove(workerZkPath);
logger.info("worker node : {} unRegistry from ZK {}.", address, workerZkPath);
}
this.heartBeatExecutor.shutdownNow(); this.heartBeatExecutor.shutdownNow();
logger.info("worker node : {} unRegistry to ZK.", address);
} }
/** /**
* get worker path * get worker path
* @return
*/ */
private String getWorkerPath() { private Set<String> getWorkerZkPaths() {
Set<String> workerZkPaths = Sets.newHashSet();
String address = getLocalAddress(); String address = getLocalAddress();
StringBuilder builder = new StringBuilder(100); String workerZkPathPrefix = this.zookeeperRegistryCenter.getWorkerPath();
String workerPath = this.zookeeperRegistryCenter.getWorkerPath();
builder.append(workerPath).append(SLASH); for (String workGroup : this.workerGroups) {
if(StringUtils.isEmpty(workerGroup)){ StringBuilder workerZkPathBuilder = new StringBuilder(100);
workerGroup = DEFAULT_WORKER_GROUP; workerZkPathBuilder.append(workerZkPathPrefix).append(SLASH);
if (StringUtils.isEmpty(workGroup)) {
workGroup = DEFAULT_WORKER_GROUP;
}
// trim and lower case is need
workerZkPathBuilder.append(workGroup.trim().toLowerCase()).append(SLASH);
workerZkPathBuilder.append(address);
workerZkPaths.add(workerZkPathBuilder.toString());
} }
//trim and lower case is need return workerZkPaths;
builder.append(workerGroup.trim().toLowerCase()).append(SLASH);
builder.append(address);
return builder.toString();
} }
/** /**
* get local address * get local address
* @return
*/ */
private String getLocalAddress(){ private String getLocalAddress() {
return NetUtils.getHost() + ":" + workerConfig.getListenPort(); return NetUtils.getHost() + ":" + workerConfig.getListenPort();
} }
} }

20
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java

@ -22,7 +22,9 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.Set;
import org.apache.commons.collections.MapUtils;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNode;
@ -106,7 +108,6 @@ public class TaskExecuteThread implements Runnable {
// copy hdfs/minio file to local // copy hdfs/minio file to local
downloadResource(taskExecutionContext.getExecutePath(), downloadResource(taskExecutionContext.getExecutePath(),
taskExecutionContext.getResources(), taskExecutionContext.getResources(),
taskExecutionContext.getTenantCode(),
logger); logger);
taskExecutionContext.setTaskParams(taskNode.getParams()); taskExecutionContext.setTaskParams(taskNode.getParams());
@ -227,22 +228,25 @@ public class TaskExecuteThread implements Runnable {
* @param logger * @param logger
*/ */
private void downloadResource(String execLocalPath, private void downloadResource(String execLocalPath,
List<String> projectRes, Map<String,String> projectRes,
String tenantCode,
Logger logger) throws Exception { Logger logger) throws Exception {
if (CollectionUtils.isEmpty(projectRes)){ if (MapUtils.isEmpty(projectRes)){
return; return;
} }
for (String resource : projectRes) { Set<Map.Entry<String, String>> resEntries = projectRes.entrySet();
File resFile = new File(execLocalPath, resource);
for (Map.Entry<String,String> resource : resEntries) {
String fullName = resource.getKey();
String tenantCode = resource.getValue();
File resFile = new File(execLocalPath, fullName);
if (!resFile.exists()) { if (!resFile.exists()) {
try { try {
// query the tenant code of the resource according to the name of the resource // query the tenant code of the resource according to the name of the resource
String resHdfsPath = HadoopUtils.getHdfsResourceFileName(tenantCode, resource); String resHdfsPath = HadoopUtils.getHdfsResourceFileName(tenantCode, fullName);
logger.info("get resource file from hdfs :{}", resHdfsPath); logger.info("get resource file from hdfs :{}", resHdfsPath);
HadoopUtils.getInstance().copyHdfsToLocal(resHdfsPath, execLocalPath + File.separator + resource, false, true); HadoopUtils.getInstance().copyHdfsToLocal(resHdfsPath, execLocalPath + File.separator + fullName, false, true);
}catch (Exception e){ }catch (Exception e){
logger.error(e.getMessage(),e); logger.error(e.getMessage(),e);
throw new RuntimeException(e.getMessage()); throw new RuntimeException(e.getMessage());

3
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java

@ -131,8 +131,7 @@ public class SqlTask extends AbstractTask {
.map(this::getSqlAndSqlParamsMap) .map(this::getSqlAndSqlParamsMap)
.collect(Collectors.toList()); .collect(Collectors.toList());
List<String> createFuncs = UDFUtils.createFuncs(sqlTaskExecutionContext.getUdfFuncList(), List<String> createFuncs = UDFUtils.createFuncs(sqlTaskExecutionContext.getUdfFuncTenantCodeMap(),
taskExecutionContext.getTenantCode(),
logger); logger);
// execute sql task // execute sql task

2
dolphinscheduler-server/src/main/resources/worker.properties

@ -31,4 +31,4 @@
#worker.listen.port: 1234 #worker.listen.port: 1234
# default worker group # default worker group
worker.group=default #worker.groups=default

148
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryTest.java

@ -17,62 +17,154 @@
package org.apache.dolphinscheduler.server.worker.registry; package org.apache.dolphinscheduler.server.worker.registry;
import static org.apache.dolphinscheduler.common.Constants.DEFAULT_WORKER_GROUP;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Executor;
import org.apache.curator.framework.imps.CuratorFrameworkImpl;
import org.apache.curator.framework.listen.Listenable;
import org.apache.curator.framework.state.ConnectionStateListener;
import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter;
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
import org.apache.dolphinscheduler.server.zk.SpringZKServer;
import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator;
import org.apache.dolphinscheduler.service.zk.ZookeeperConfig;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired; import org.mockito.InjectMocks;
import org.springframework.test.context.ContextConfiguration; import org.mockito.Mock;
import org.springframework.test.context.junit4.SpringRunner; import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List; import com.google.common.collect.Sets;
import java.util.concurrent.TimeUnit;
import static org.apache.dolphinscheduler.common.Constants.DEFAULT_WORKER_GROUP;
import static org.apache.dolphinscheduler.common.Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH;
/** /**
* worker registry test * worker registry test
*/ */
@RunWith(SpringRunner.class) @RunWith(MockitoJUnitRunner.Silent.class)
@ContextConfiguration(classes={SpringZKServer.class, WorkerRegistry.class,ZookeeperRegistryCenter.class, WorkerConfig.class, ZookeeperCachedOperator.class, ZookeeperConfig.class})
public class WorkerRegistryTest { public class WorkerRegistryTest {
@Autowired private static final Logger LOGGER = LoggerFactory.getLogger(WorkerRegistryTest.class);
private static final String TEST_WORKER_GROUP = "test";
@InjectMocks
private WorkerRegistry workerRegistry; private WorkerRegistry workerRegistry;
@Autowired @Mock
private ZookeeperRegistryCenter zookeeperRegistryCenter; private ZookeeperRegistryCenter zookeeperRegistryCenter;
@Autowired @Mock
private ZookeeperCachedOperator zookeeperCachedOperator;
@Mock
private CuratorFrameworkImpl zkClient;
@Mock
private WorkerConfig workerConfig; private WorkerConfig workerConfig;
@Before
public void before() {
Set<String> workerGroups = Sets.newHashSet(DEFAULT_WORKER_GROUP, TEST_WORKER_GROUP);
Mockito.when(workerConfig.getWorkerGroups()).thenReturn(workerGroups);
Mockito.when(zookeeperRegistryCenter.getWorkerPath()).thenReturn("/dolphinscheduler/nodes/worker");
Mockito.when(zookeeperRegistryCenter.getZookeeperCachedOperator()).thenReturn(zookeeperCachedOperator);
Mockito.when(zookeeperRegistryCenter.getZookeeperCachedOperator().getZkClient()).thenReturn(zkClient);
Mockito.when(zookeeperRegistryCenter.getZookeeperCachedOperator().getZkClient().getConnectionStateListenable()).thenReturn(
new Listenable<ConnectionStateListener>() {
@Override
public void addListener(ConnectionStateListener connectionStateListener) {
LOGGER.info("add listener");
}
@Override
public void addListener(ConnectionStateListener connectionStateListener, Executor executor) {
LOGGER.info("add listener executor");
}
@Override
public void removeListener(ConnectionStateListener connectionStateListener) {
LOGGER.info("remove listener");
}
});
Mockito.when(workerConfig.getWorkerHeartbeatInterval()).thenReturn(10);
Mockito.when(workerConfig.getWorkerReservedMemory()).thenReturn(1.1);
Mockito.when(workerConfig.getWorkerMaxCpuloadAvg()).thenReturn(1);
}
@Test @Test
public void testRegistry() throws InterruptedException { public void testRegistry() {
workerRegistry.init();
workerRegistry.registry(); workerRegistry.registry();
String workerPath = zookeeperRegistryCenter.getWorkerPath(); String workerPath = zookeeperRegistryCenter.getWorkerPath();
Assert.assertEquals(DEFAULT_WORKER_GROUP, workerConfig.getWorkerGroup().trim());
String instancePath = workerPath + "/" + workerConfig.getWorkerGroup().trim() + "/" + (NetUtils.getHost() + ":" + workerConfig.getListenPort()); int i = 0;
TimeUnit.SECONDS.sleep(workerConfig.getWorkerHeartbeatInterval() + 2); //wait heartbeat info write into zk node for (String workerGroup : workerConfig.getWorkerGroups()) {
String heartbeat = zookeeperRegistryCenter.getZookeeperCachedOperator().get(instancePath); String workerZkPath = workerPath + "/" + workerGroup.trim() + "/" + (NetUtils.getHost() + ":" + workerConfig.getListenPort());
Assert.assertEquals(HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH, heartbeat.split(",").length); String heartbeat = zookeeperRegistryCenter.getZookeeperCachedOperator().get(workerZkPath);
if (0 == i) {
Assert.assertTrue(workerZkPath.startsWith("/dolphinscheduler/nodes/worker/test/"));
} else {
Assert.assertTrue(workerZkPath.startsWith("/dolphinscheduler/nodes/worker/default/"));
}
i++;
}
workerRegistry.unRegistry();
workerConfig.getWorkerGroups().add(StringUtils.EMPTY);
workerRegistry.init();
workerRegistry.registry();
workerRegistry.unRegistry();
// testEmptyWorkerGroupsRegistry
workerConfig.getWorkerGroups().remove(StringUtils.EMPTY);
workerConfig.getWorkerGroups().remove(TEST_WORKER_GROUP);
workerConfig.getWorkerGroups().remove(DEFAULT_WORKER_GROUP);
workerRegistry.init();
workerRegistry.registry();
List<String> testWorkerGroupPathZkChildren = zookeeperRegistryCenter.getChildrenKeys(workerPath + "/" + TEST_WORKER_GROUP);
List<String> defaultWorkerGroupPathZkChildren = zookeeperRegistryCenter.getChildrenKeys(workerPath + "/" + DEFAULT_WORKER_GROUP);
Assert.assertEquals(0, testWorkerGroupPathZkChildren.size());
Assert.assertEquals(0, defaultWorkerGroupPathZkChildren.size());
} }
@Test @Test
public void testUnRegistry() throws InterruptedException { public void testUnRegistry() {
workerRegistry.init();
workerRegistry.registry(); workerRegistry.registry();
TimeUnit.SECONDS.sleep(workerConfig.getWorkerHeartbeatInterval() + 2); //wait heartbeat info write into zk node
workerRegistry.unRegistry(); workerRegistry.unRegistry();
String workerPath = zookeeperRegistryCenter.getWorkerPath(); String workerPath = zookeeperRegistryCenter.getWorkerPath();
String workerGroupPath = workerPath + "/" + workerConfig.getWorkerGroup().trim();
List<String> childrenKeys = zookeeperRegistryCenter.getZookeeperCachedOperator().getChildrenKeys(workerGroupPath); for (String workerGroup : workerConfig.getWorkerGroups()) {
Assert.assertTrue(childrenKeys.isEmpty()); String workerGroupPath = workerPath + "/" + workerGroup.trim();
List<String> childrenKeys = zookeeperRegistryCenter.getZookeeperCachedOperator().getChildrenKeys(workerGroupPath);
Assert.assertTrue(childrenKeys.isEmpty());
}
// testEmptyWorkerGroupsUnRegistry
workerConfig.getWorkerGroups().remove(TEST_WORKER_GROUP);
workerConfig.getWorkerGroups().remove(DEFAULT_WORKER_GROUP);
workerRegistry.init();
workerRegistry.registry();
workerRegistry.unRegistry();
} }
} }

2
dolphinscheduler-service/pom.xml

@ -20,7 +20,7 @@
<parent> <parent>
<artifactId>dolphinscheduler</artifactId> <artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId> <groupId>org.apache.dolphinscheduler</groupId>
<version>1.3.0-SNAPSHOT</version> <version>1.3.2-SNAPSHOT</version>
</parent> </parent>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>

117
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZKServer.java

@ -16,6 +16,7 @@
*/ */
package org.apache.dolphinscheduler.service.zk; package org.apache.dolphinscheduler.service.zk;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.zookeeper.server.ZooKeeperServer; import org.apache.zookeeper.server.ZooKeeperServer;
import org.apache.zookeeper.server.ZooKeeperServerMain; import org.apache.zookeeper.server.ZooKeeperServerMain;
import org.apache.zookeeper.server.quorum.QuorumPeerConfig; import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
@ -34,44 +35,62 @@ import java.util.concurrent.atomic.AtomicBoolean;
public class ZKServer { public class ZKServer {
private static final Logger logger = LoggerFactory.getLogger(ZKServer.class); private static final Logger logger = LoggerFactory.getLogger(ZKServer.class);
private static volatile PublicZooKeeperServerMain zkServer = null;
public static final int DEFAULT_ZK_TEST_PORT = 2181; public static final int DEFAULT_ZK_TEST_PORT = 2181;
private static String dataDir = null; private final AtomicBoolean isStarted = new AtomicBoolean(false);
private PublicZooKeeperServerMain zooKeeperServerMain = null;
private int port;
private static final AtomicBoolean isStarted = new AtomicBoolean(false); private String dataDir = null;
private String prefix;
public static void main(String[] args) { public static void main(String[] args) {
if(!isStarted()){ ZKServer zkServer;
ZKServer.start(); if (args.length == 0) {
zkServer = new ZKServer();
/** } else if (args.length == 1){
* register hooks, which are called before the process exits zkServer = new ZKServer(Integer.valueOf(args[0]), "");
*/ } else {
Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { zkServer = new ZKServer(Integer.valueOf(args[0]), args[1]);
@Override }
public void run() { zkServer.registerHook();
stop(); zkServer.start();
} }
}));
}else{ public ZKServer() {
logger.info("zk server aleady started"); this(DEFAULT_ZK_TEST_PORT, "");
}
public ZKServer(int port, String prefix) {
this.port = port;
if (prefix != null && prefix.contains("/")) {
throw new IllegalArgumentException("The prefix of path may not have '/'");
} }
this.prefix = (prefix == null ? null : prefix.trim());
}
private void registerHook() {
/**
* register hooks, which are called before the process exits
*/
Runtime.getRuntime().addShutdownHook(new Thread(this::stop));
} }
/** /**
* start service * start service
*/ */
public static void start() { public void start() {
try { try {
startLocalZkServer(DEFAULT_ZK_TEST_PORT); startLocalZkServer(port);
} catch (Exception e) { } catch (Exception e) {
logger.error("Failed to start ZK: " + e); logger.error("Failed to start ZK ", e);
} }
} }
public static boolean isStarted(){ public boolean isStarted(){
return isStarted.get(); return isStarted.get();
} }
@ -94,8 +113,12 @@ public class ZKServer {
* *
* @param port The port to listen on * @param port The port to listen on
*/ */
public static void startLocalZkServer(final int port) { public void startLocalZkServer(final int port) {
String zkDataDir = System.getProperty("user.dir") +"/zookeeper_data"; String zkDataDir = System.getProperty("user.dir") + (StringUtils.isEmpty(prefix) ? StringUtils.EMPTY : ("/" + prefix)) + "/zookeeper_data";
File file = new File(zkDataDir);
if (file.exists()) {
logger.warn("The path of zk server exists");
}
logger.info("zk server starting, data dir path:{}" , zkDataDir); logger.info("zk server starting, data dir path:{}" , zkDataDir);
startLocalZkServer(port, zkDataDir, ZooKeeperServer.DEFAULT_TICK_TIME,"60"); startLocalZkServer(port, zkDataDir, ZooKeeperServer.DEFAULT_TICK_TIME,"60");
} }
@ -108,31 +131,29 @@ public class ZKServer {
* @param tickTime zk tick time * @param tickTime zk tick time
* @param maxClientCnxns zk max client connections * @param maxClientCnxns zk max client connections
*/ */
private static synchronized void startLocalZkServer(final int port, final String dataDirPath,final int tickTime,String maxClientCnxns) { private void startLocalZkServer(final int port, final String dataDirPath,final int tickTime,String maxClientCnxns) {
if (zkServer != null) { if (isStarted.compareAndSet(false, true)) {
throw new RuntimeException("Zookeeper server is already started!"); zooKeeperServerMain = new PublicZooKeeperServerMain();
} logger.info("Zookeeper data path : {} ", dataDirPath);
zkServer = new PublicZooKeeperServerMain(); dataDir = dataDirPath;
logger.info("Zookeeper data path : {} ", dataDirPath); final String[] args = new String[]{Integer.toString(port), dataDirPath, Integer.toString(tickTime), maxClientCnxns};
dataDir = dataDirPath;
final String[] args = new String[]{Integer.toString(port), dataDirPath, Integer.toString(tickTime), maxClientCnxns};
try { try {
logger.info("Zookeeper server started "); logger.info("Zookeeper server started ");
isStarted.compareAndSet(false, true); isStarted.compareAndSet(false, true);
zkServer.initializeAndRun(args); zooKeeperServerMain.initializeAndRun(args);
} catch (QuorumPeerConfig.ConfigException e) { } catch (QuorumPeerConfig.ConfigException | IOException e) {
logger.warn("Caught exception while starting ZK", e); logger.warn("Caught exception while starting ZK", e);
} catch (IOException e) { throw new RuntimeException(e);
logger.warn("Caught exception while starting ZK", e); }
} }
} }
/** /**
* Stops a local Zk instance, deleting its data directory * Stops a local Zk instance, deleting its data directory
*/ */
public static void stop() { public void stop() {
try { try {
stopLocalZkServer(true); stopLocalZkServer(true);
logger.info("zk server stopped"); logger.info("zk server stopped");
@ -147,15 +168,17 @@ public class ZKServer {
* *
* @param deleteDataDir Whether or not to delete the data directory * @param deleteDataDir Whether or not to delete the data directory
*/ */
private static synchronized void stopLocalZkServer(final boolean deleteDataDir) { private void stopLocalZkServer(final boolean deleteDataDir) {
if (zkServer != null) { if (isStarted.compareAndSet(true, false)) {
try { try {
zkServer.shutdown(); if (zooKeeperServerMain == null) {
zkServer = null; return;
}
zooKeeperServerMain.shutdown();
zooKeeperServerMain = null;
if (deleteDataDir) { if (deleteDataDir) {
org.apache.commons.io.FileUtils.deleteDirectory(new File(dataDir)); org.apache.commons.io.FileUtils.deleteDirectory(new File(dataDir));
} }
isStarted.compareAndSet(true, false);
} catch (Exception e) { } catch (Exception e) {
logger.warn("Caught exception while stopping ZK server", e); logger.warn("Caught exception while stopping ZK server", e);
throw new RuntimeException(e); throw new RuntimeException(e);

38
dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/zk/ZKServerTest.java

@ -18,18 +18,44 @@ package org.apache.dolphinscheduler.service.zk;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
// ZKServer is a process, can't unit test import java.util.concurrent.TimeUnit;
public class ZKServerTest { import java.util.concurrent.atomic.AtomicReference;
public class ZKServerTest {
private static final Logger log = LoggerFactory.getLogger(ZKServerTest.class);
@Test @Test
public void isStarted() { public void testRunWithDefaultPort() {
Assert.assertEquals(false, ZKServer.isStarted()); AtomicReference<ZKServer> zkServer = new AtomicReference<>();
new Thread(() -> {
zkServer.set(new ZKServer());
zkServer.get().start();
}).start();
try {
TimeUnit.SECONDS.sleep(5);
Assert.assertEquals(true, zkServer.get().isStarted());
} catch (InterruptedException e) {
log.error("Thread interrupted", e);
}
zkServer.get().stop();
} }
@Test @Test
public void stop() { public void testRunWithCustomPort() {
ZKServer.stop(); AtomicReference<ZKServer> zkServer = new AtomicReference<>();
new Thread(() -> {
zkServer.set(new ZKServer(2183, null));
zkServer.get().start();
}).start();
try {
TimeUnit.SECONDS.sleep(5);
Assert.assertEquals(true, zkServer.get().isStarted());
} catch (InterruptedException e) {
log.error("Thread interrupted", e);
}
zkServer.get().stop();
} }
} }

2
dolphinscheduler-ui/pom.xml

@ -20,7 +20,7 @@
<parent> <parent>
<artifactId>dolphinscheduler</artifactId> <artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId> <groupId>org.apache.dolphinscheduler</groupId>
<version>1.3.0-SNAPSHOT</version> <version>1.3.2-SNAPSHOT</version>
</parent> </parent>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>

3
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/_source/timeoutAlarm.vue

@ -91,7 +91,8 @@
return false return false
} }
// Verify timeout duration Non 0 positive integer // Verify timeout duration Non 0 positive integer
if (this.enable && !parseInt(this.interval) && !_.isInteger(this.interval)) { const reg = /^[1-9]\d*$/
if (this.enable && !reg.test(this.interval)) {
this.$message.warning(`${this.$t('Timeout must be a positive integer')}`) this.$message.warning(`${this.$t('Timeout must be a positive integer')}`)
return false return false
} }

50
dolphinscheduler-ui/src/js/conf/home/pages/datasource/pages/list/_source/createDataSource.vue

@ -24,16 +24,10 @@
<m-list-box-f> <m-list-box-f>
<template slot="name"><strong>*</strong>{{$t('Datasource')}}</template> <template slot="name"><strong>*</strong>{{$t('Datasource')}}</template>
<template slot="content"> <template slot="content">
<x-radio-group v-model="type" size="small"> <x-select style="width: 100%;" v-model="type">
<x-radio :label="'MYSQL'">MYSQL</x-radio> <x-option v-for="item in datasourceTypeList" :key="item.value" :value="item.value" :label="item.label">
<x-radio :label="'POSTGRESQL'">POSTGRESQL</x-radio> </x-option>
<x-radio :label="'HIVE'">HIVE/IMPALA</x-radio> </x-select>
<x-radio :label="'SPARK'">SPARK</x-radio>
<x-radio :label="'CLICKHOUSE'">CLICKHOUSE</x-radio>
<x-radio :label="'ORACLE'">ORACLE</x-radio>
<x-radio :label="'SQLSERVER'">SQLSERVER</x-radio>
<x-radio :label="'DB2'" class="radio-label-last" >DB2</x-radio>
</x-radio-group>
</template> </template>
</m-list-box-f> </m-list-box-f>
<m-list-box-f> <m-list-box-f>
@ -200,7 +194,41 @@
showdDatabase: false, showdDatabase: false,
showConnectType: false, showConnectType: false,
isShowPrincipal:true, isShowPrincipal:true,
prePortMapper:{} prePortMapper:{},
datasourceTypeList: [
{
value: 'MYSQL',
label: 'MYSQL'
},
{
value: 'POSTGRESQL',
label: 'POSTGRESQL'
},
{
value: 'HIVE',
label: 'HIVE/IMPALA'
},
{
value: 'SPARK',
label: 'SPARK'
},
{
value: 'CLICKHOUSE',
label: 'CLICKHOUSE'
},
{
value: 'ORACLE',
label: 'ORACLE'
},
{
value: 'SQLSERVER',
label: 'SQLSERVER'
},
{
value: 'DB2',
label: 'DB2'
}
]
} }
}, },
props: { props: {

112
dolphinscheduler-ui/src/js/conf/home/pages/monitor/pages/servers/_source/zookeeperDirectories.vue

@ -0,0 +1,112 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
<template>
<div class="container">
<div class="title-box">
<span class="name">{{$t('zkDirectory')}}</span>
</div>
<div class="table-box" v-if="zkDirectories.length > 0">
<table class="fixed">
<caption><!-- placeHolder --></caption>
<tr>
<th scope="col" style="min-width: 40px">
<span>#</span>
</th>
<th scope="col" style="min-width: 40px">
<span>{{$t('zkDirectory')}}</span>
</th>
</tr>
<tr v-for="(item, $index) in zkDirectories" :key="item.id">
<td>
<span>{{$index + 1}}</span>
</td>
<td>
<span>{{item.zkDirectory}}</span>
</td>
</tr>
</table>
</div>
<div v-if="zkDirectories.length === 0">
<m-no-data><!----></m-no-data>
</div>
<div v-if="zkDirectories.length > 0">
<div class="bottom-box">
</div>
</div>
</div>
</template>
<script>
import mNoData from '@/module/components/noData/noData'
export default {
name: 'zookeeperDirectoriesPopup',
data () {
return {
tableHeaders: [
{
label: $t('zkDirectory'),
prop: 'zkDirectory'
}
]
}
},
props: {
zkDirectories: Array
},
components: { mNoData }
}
</script>
<style lang="scss" rel="stylesheet/scss">
.container {
width: 500px;
.title-box {
height: 61px;
border-bottom: 1px solid #DCDEDC;
position: relative;
.name {
position: absolute;
left: 24px;
top: 18px;
font-size: 16px;
}
}
.bottom-box {
position: absolute;
bottom: 0;
left: 0;
width: 100%;
text-align: right;
height: 60px;
line-height: 60px;
border-top: 1px solid #DCDEDC;
background: #fff;
.ans-page {
display: inline-block;
}
}
.table-box {
overflow-y: scroll;
height: calc(100vh - 61px);
padding-bottom: 60px;
}
}
</style>

27
dolphinscheduler-ui/src/js/conf/home/pages/monitor/pages/servers/worker.vue

@ -22,8 +22,8 @@
<div class="row-title"> <div class="row-title">
<div class="left"> <div class="left">
<span class="sp">IP: {{item.host}}</span> <span class="sp">IP: {{item.host}}</span>
<span class="sp">{{$t('Process Pid')}}: {{item.id}}</span> <span class="sp">{{$t('Process Pid')}}: {{item.port}}</span>
<span class="sp">{{$t('Zk registration directory')}}: {{item.zkDirectory}}</span> <span>{{$t('Zk registration directory')}}: <a href="javascript:" @click="_showZkDirectories(item)" class="links">{{$t('Directory detail')}}</a></span>
</div> </div>
<div class="right"> <div class="right">
<span class="sp">{{$t('Create Time')}}: {{item.createTime | formatDate}}</span> <span class="sp">{{$t('Create Time')}}: {{item.createTime | formatDate}}</span>
@ -74,6 +74,7 @@
import mNoData from '@/module/components/noData/noData' import mNoData from '@/module/components/noData/noData'
import themeData from '@/module/echarts/themeData.json' import themeData from '@/module/echarts/themeData.json'
import mListConstruction from '@/module/components/listConstruction/listConstruction' import mListConstruction from '@/module/components/listConstruction/listConstruction'
import zookeeperDirectoriesPopup from './_source/zookeeperDirectories'
export default { export default {
name: 'servers-worker', name: 'servers-worker',
@ -86,7 +87,25 @@
}, },
props: {}, props: {},
methods: { methods: {
...mapActions('monitor', ['getWorkerData']) ...mapActions('monitor', ['getWorkerData']),
_showZkDirectories (item) {
let zkDirectories = []
item.zkDirectories.forEach(zkDirectory => {
zkDirectories.push({
zkDirectory: zkDirectory
})
})
this.$drawer({
direction: 'right',
render (h) {
return h(zookeeperDirectoriesPopup, {
props: {
zkDirectories: zkDirectories
}
})
}
})
}
}, },
watch: {}, watch: {},
created () { created () {
@ -105,7 +124,7 @@
this.isLoading = true this.isLoading = true
}) })
}, },
components: { mList, mListConstruction, mSpin, mNoData, mGauge } components: { mList, mListConstruction, mSpin, mNoData, mGauge, zookeeperDirectoriesPopup }
} }
</script> </script>
<style lang="scss" rel="stylesheet/scss"> <style lang="scss" rel="stylesheet/scss">

4
dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js

@ -616,5 +616,7 @@ export default {
'Disable': 'Disable', 'Disable': 'Disable',
'The Worker group no longer exists, please select the correct Worker group!': 'The Worker group no longer exists, please select the correct Worker group!', 'The Worker group no longer exists, please select the correct Worker group!': 'The Worker group no longer exists, please select the correct Worker group!',
'Please confirm whether the workflow has been saved before downloading': 'Please confirm whether the workflow has been saved before downloading', 'Please confirm whether the workflow has been saved before downloading': 'Please confirm whether the workflow has been saved before downloading',
'User name length is between 3 and 39': 'User name length is between 3 and 39' 'User name length is between 3 and 39': 'User name length is between 3 and 39',
zkDirectory: 'zkDirectory',
'Directory detail': 'Directory detail'
} }

4
dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js

@ -616,5 +616,7 @@ export default {
'Socket Timeout':'Socket超时', 'Socket Timeout':'Socket超时',
'Connect timeout be a positive integer': '连接超时必须为数字', 'Connect timeout be a positive integer': '连接超时必须为数字',
'Socket Timeout be a positive integer': 'Socket超时必须为数字', 'Socket Timeout be a positive integer': 'Socket超时必须为数字',
'ms':'毫秒' 'ms':'毫秒',
zkDirectory: 'zk注册目录',
'Directory detail': '查看目录详情'
} }

17
e2e/src/test/java/org/apache/dolphinscheduler/common/BrowserCommon.java

@ -167,6 +167,23 @@ public class BrowserCommon {
return inputElement; return inputElement;
} }
/**
* input element after clearing
*
* @param locator By
* @param content Input content
* @return inputElement
*/
public WebElement clearSendInput(By locator, String content) {
WebElement inputElement = locateElement(locator);
//input date
inputElement.clear();
inputElement.sendKeys(Keys.HOME,Keys.chord(Keys.SHIFT,Keys.END));
inputElement.sendKeys(content);
return inputElement;
}
/** /**
* clear element * clear element
* *

32
e2e/src/test/java/org/apache/dolphinscheduler/data/project/RunWorkflowData.java

@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.data.project;
public class RunWorkflowData {
/**
* run workflow data
*/
//input shell task name
public static final String RECIPIENT = "123456789@qq.com";
//input shell task description
public static final String Cc = "qwe12312sds@qq.com";
public static final String RUN_WORKFLOW_TITLE = "工作流定义 - DolphinScheduler";
}

35
e2e/src/test/java/org/apache/dolphinscheduler/data/project/TimingData.java

@ -0,0 +1,35 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.data.project;
public class TimingData {
/**
* timing data
*/
//input shell task name
public static final String RECIPIENT = "test123456@qq.com";
//input shell task description
public static final String Cc = "test.123qwe@qq.com";
public static final String EDIT_RECIPIENT = "test.edit123456@qq.com";
public static final String EDIT_Cc = "test.edit123qwe@qq.com";
public static final String WORKFLOW_TITLE = "工作流定义 - DolphinScheduler";
public static final String TIMING_TITLE = "定时任务列表 - DolphinScheduler";
}

10
e2e/src/test/java/org/apache/dolphinscheduler/data/security/UserManageData.java

@ -18,6 +18,9 @@ package org.apache.dolphinscheduler.data.security;
public class UserManageData { public class UserManageData {
/**
* create user
*/
public static final String USERNAME = "selenium_user_1"; public static final String USERNAME = "selenium_user_1";
public static final String PASSWORD = "123456qwe"; public static final String PASSWORD = "123456qwe";
@ -28,4 +31,11 @@ public class UserManageData {
public static final String USER_MANAGE = "用户管理 - DolphinScheduler"; public static final String USER_MANAGE = "用户管理 - DolphinScheduler";
/**
* edit user
*/
public static final String EDIT_USERNAME = "selenium_user_edit";
public static final String EDIT_PASSWORD = "123456qwe";
public static final String EDIT_EMAIL = "123456_edit@qq.com";
public static final String EDIT_PHONE = "15800001111";
} }

18
e2e/src/test/java/org/apache/dolphinscheduler/locator/project/CreateWorkflowLocator.java

@ -135,8 +135,7 @@ public class CreateWorkflowLocator {
//select tenant //select tenant
public static final By CLICK_TENANT = By.xpath("//div[4]/div/div/div/input"); public static final By CLICK_TENANT = By.xpath("//div[4]/div/div/div/input");
public static final By SELECT_TENANT = By.xpath("//div[2]/div/div/div/ul/li/span"); public static final By SELECT_TENANT = By.xpath("//div[4]/div/div[2]/div/div/div/ul/li[2]/span");
//click workflow timeout alarm //click workflow timeout alarm
public static final By CLICK_WORKFLOW_TIMEOUT_ALARM = By.xpath("//span[2]/span/span"); public static final By CLICK_WORKFLOW_TIMEOUT_ALARM = By.xpath("//span[2]/span/span");
@ -170,11 +169,24 @@ public class CreateWorkflowLocator {
//scroll to element bottom //scroll to element bottom
public static final By SCROLL_BOTTOM = By.xpath("//span/a/em"); public static final By SCROLL_BOTTOM = By.xpath("//span/a/em");
/**
* online workflow
*/
// click online workflow button
public static final By CLICK_ONLINE_WORKFLOW_BUTTON = By.xpath("//div[1]/div/table/tr[2]/td[10]/button[4]");
/**
* offline workflow
*/
// click offline workflow button
public static final By CLICK_OFFLINE_WORKFLOW_BUTTON = By.xpath("//div[1]/div/table/tr[2]/td[10]/button[4]");
/** /**
* delete workflow * delete workflow
*/ */
//click delete workflow button //click delete workflow button
public static final By DELETE_WORKFLOW_BOTTOM = By.xpath("//span/button/i"); public static final By DELETE_WORKFLOW_BOTTOM = By.xpath("//div[3]/div[1]/div/table/tr[2]/td[10]/span/button");
//click confirm delete workflow button //click confirm delete workflow button
public static final By CONFIRM_DELETE_WORKFLOW_BOTTOM = By.xpath("//div[2]/div/button[2]/span"); public static final By CONFIRM_DELETE_WORKFLOW_BOTTOM = By.xpath("//div[2]/div/button[2]/span");

48
e2e/src/test/java/org/apache/dolphinscheduler/locator/project/RunWorkflowLocator.java

@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.locator.project;
import org.openqa.selenium.By;
public class RunWorkflowLocator {
/**
* run workflow
*/
// click run workflow button
public static final By CLICK_RUN_WORKFLOW_BUTTON = By.xpath("//div[1]/div/table/tr[2]/td[10]/button[2]");
//set running parameters
public static final By SELECT_FAILURE_STRATEGY_END = By.xpath("//div[5]/div/div[2]/div/div[3]/div[2]/div/label[2]/span[1]/input");
public static final By SELECT_FAILURE_STRATEGY_CONTINUE = By.xpath("//div[3]/div[2]/div/label[1]/span[1]/input");
public static final By CLICK_NOTICE_STRATEGY = By.xpath("//div[4]/div[2]/div/div[1]/div/input");
public static final By SELECT_NOTICE_STRATEGY = By.xpath("//div/ul/li[4]/span");
public static final By CLICK_PROCESS_PRIORITY = By.xpath("//div/div/div/div/div/span[2]");
public static final By SELECT_PROCESS_PRIORITY_HIGHEST = By.xpath("//li[1]/li/span");
public static final By CLICK_WORKER_GROUP = By.xpath("//div[6]/div[2]/div/div[1]/div/input");
public static final By SELECT_WORKER_GROUP = By.xpath("//div[5]/div/div[2]/div/div[6]/div[2]/div/div[2]/div/div/div/ul/li/span");
public static final By CLICK_NOTICE_GROUP = By.xpath("//div[7]/div[2]/div/div[1]/div/input");
public static final By SELECT_NOTICE_GROUP = By.xpath("//div[5]/div/div[2]/div/div[7]/div[2]/div/div[2]/div/div/div/ul/li/span");
public static final By INPUT_RECIPIENT = By.xpath("//div[8]/div[2]/div/div/span/span/input");
public static final By INPUT_Cc = By.xpath("//div[9]/div[2]/div/div/span/span/input");
public static final By CLICK_RUNNING_BUTTON = By.xpath("//div[11]/button[2]");
}

61
e2e/src/test/java/org/apache/dolphinscheduler/locator/project/TimingLocator.java

@ -0,0 +1,61 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.locator.project;
import org.openqa.selenium.By;
public class TimingLocator {
// create timing button
public static final By CLICK_TIMING_BUTTON = By.xpath("//button[3]");
public static final By CLICK_EXECUTION_TIMING_BUTTON = By.xpath("//div[3]/button/span");
public static final By SELECT_FAILURE_STRATEGY_END = By.xpath("//label[2]/span/input");
public static final By SELECT_FAILURE_STRATEGY_CONTINUE = By.xpath("//div[2]/div/label/span[2]");
public static final By CLICK_NOTICE_STRATEGY = By.xpath("//div[6]/div[2]/div/div/div/input");
public static final By SELECT_NOTICE_STRATEGY = By.xpath("//div[2]/div/div/div/ul/li[4]/span");
public static final By CLICK_PROCESS_PRIORITY = By.xpath("//div[7]/div[2]/div/div/div/div/div/span[2]");
public static final By SELECT_PROCESS_PRIORITY = By.xpath("//li/li/span");
public static final By CLICK_WORKER_GROUP = By.xpath("//div[8]/div[2]/div/div/div/input");
public static final By SELECT_WORKER_GROUP = By.xpath("//div[8]/div[2]/div/div[2]/div/div/div/ul/li/span");
public static final By CLICK_NOTICE_GROUP = By.xpath("//div[9]/div[2]/div/div/div/input");
public static final By SELECT_NOTICE_GROUP = By.xpath("//div[9]/div[2]/div/div[2]/div/div/div/ul/li/span");
public static final By INPUT_RECIPIENT = By.xpath("//div[10]/div[2]/div/div/span/span/input");
public static final By INPUT_Cc = By.xpath("//div[11]/div[2]/div/div/span/span/input");
public static final By CLICK_CREATE_BUTTON = By.xpath("//div[12]/button[2]/span");
//edit timing
public static final By CLICK_TIMING_MANAGEMENT_BUTTON = By.xpath("//div[2]/div/div[3]/div[1]/div/table/tr[2]/td[10]/button[6]");
public static final By CLICK_EDIT_TIMING_BUTTON = By.xpath("//div[2]/div[3]/div/div[2]/div[1]/table/tr[2]/td[10]/button[1]");
//online timing
public static final By CLICK_ONLINE_TIMING_BUTTON = By.xpath("//td[10]/button[2]");
//offline timing
public static final By CLICK_OFFLINE_TIMING_BUTTON = By.xpath("//div[2]/div[3]/div/div[2]/div[1]/table/tr[2]/td[10]/button[2]");
//delete timing
public static final By CLICK_DELETE_TIMING_BUTTON = By.xpath("//div[2]/div[3]/div/div[2]/div[1]/table/tr[2]/td[10]/span/button");
public static final By CLICK_CONFIRM_DELETE_TIMING_BUTTON = By.xpath("//div[2]/div/button[2]/span");
}

2
e2e/src/test/java/org/apache/dolphinscheduler/locator/security/TokenManageLocator.java

@ -20,7 +20,7 @@ import org.openqa.selenium.By;
public class TokenManageLocator { public class TokenManageLocator {
//create token //create token
public static final By CLICK_TOKEN_MANAGE = By.xpath("//div[7]/div/a/div/a/span"); public static final By CLICK_TOKEN_MANAGE = By.xpath("/html/body/div[1]/div[2]/div/div[1]/div[7]/div/a/div/a/span");
public static final By CLICK_CREATE_TOKEN = By.xpath("//div[2]/div/div[2]/div[2]/div/div[1]/button/span"); public static final By CLICK_CREATE_TOKEN = By.xpath("//div[2]/div/div[2]/div[2]/div/div[1]/button/span");

22
e2e/src/test/java/org/apache/dolphinscheduler/locator/security/UserManageLocator.java

@ -20,6 +20,9 @@ import org.openqa.selenium.By;
public class UserManageLocator { public class UserManageLocator {
/**
* create user
*/
public static final By CLICK_USER_MANAGE = By.xpath("//div[3]/div/a/div/a/span"); public static final By CLICK_USER_MANAGE = By.xpath("//div[3]/div/a/div/a/span");
public static final By CLICK_CREATE_USER_BUTTON = By.xpath("//span[contains(.,'创建用户')]"); public static final By CLICK_CREATE_USER_BUTTON = By.xpath("//span[contains(.,'创建用户')]");
@ -36,13 +39,26 @@ public class UserManageLocator {
public static final By SELECT_QUEUE = By.xpath("//div[4]/div[2]/div/div[2]/div/div/div/ul/li/span"); public static final By SELECT_QUEUE = By.xpath("//div[4]/div[2]/div/div[2]/div/div/div/ul/li/span");
public static final By TENANT_INPUT_EMAIL = By.xpath("//div[5]/div[2]/div/input"); public static final By INPUT_EMAIL = By.xpath("//div[5]/div[2]/div/input");
public static final By TENANT_INPUT_PHONE = By.xpath("//div[6]/div[2]/div/input"); public static final By INPUT_PHONE = By.xpath("//div[6]/div[2]/div/input");
public static final By SELECT_STOP_STATE = By.xpath("//div[7]/div[2]/div/label[2]/span/input");
public static final By SELECT_ENABLE_STATE = By.xpath("//div[7]/div[2]/div/label[1]/span/input");
public static final By SUBMIT = By.xpath("//div[3]/button[2]/span"); public static final By SUBMIT = By.xpath("//div[3]/button[2]/span");
public static final By DELETE_USER_BUTTON = By.xpath("//span[2]/button"); /**
* edit user
*/
public static final By EDIT_GENERAL_USER_BUTTON = By.xpath("//div[3]/div[1]/div/table/tr[2]/td[11]/button");
public static final By EDIT_ADMIN_USER_BUTTON = By.xpath("//div[3]/div[1]/div/table/tr[3]/td[11]/button");
/**
* delete user
*/
public static final By DELETE_USER_BUTTON = By.xpath("//div[3]/div[1]/div/table/tr[2]/td[11]/span[2]/button");
public static final By CONFIRM_DELETE_USER_BUTTON = By.xpath("//div[2]/div/button[2]/span"); public static final By CONFIRM_DELETE_USER_BUTTON = By.xpath("//div[2]/div/button[2]/span");
} }

1
e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateProjectPage.java

@ -31,7 +31,6 @@ public class CreateProjectPage extends PageCommon {
* jump to ProjectManagePage * jump to ProjectManagePage
*/ */
public boolean jumpProjectManagePage() throws InterruptedException { public boolean jumpProjectManagePage() throws InterruptedException {
Thread.sleep(TestConstant.THREE_THOUSAND);
clickTopElement(CreateProjectLocator.PROJECT_MANAGE); clickTopElement(CreateProjectLocator.PROJECT_MANAGE);
Thread.sleep(TestConstant.ONE_THOUSAND); Thread.sleep(TestConstant.ONE_THOUSAND);
return ifTitleContains(CreateProjectData.PROJECT_TITLE); return ifTitleContains(CreateProjectData.PROJECT_TITLE);

29
e2e/src/test/java/org/apache/dolphinscheduler/page/project/CreateWorkflowPage.java

@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.common.PageCommon;
import org.apache.dolphinscheduler.constant.TestConstant; import org.apache.dolphinscheduler.constant.TestConstant;
import org.apache.dolphinscheduler.data.project.CreateWorkflowData; import org.apache.dolphinscheduler.data.project.CreateWorkflowData;
import org.apache.dolphinscheduler.locator.project.CreateWorkflowLocator; import org.apache.dolphinscheduler.locator.project.CreateWorkflowLocator;
import org.apache.dolphinscheduler.locator.project.RunWorkflowLocator;
import org.openqa.selenium.WebDriver; import org.openqa.selenium.WebDriver;
public class CreateWorkflowPage extends PageCommon { public class CreateWorkflowPage extends PageCommon {
@ -183,13 +184,37 @@ public class CreateWorkflowPage extends PageCommon {
Thread.sleep(TestConstant.ONE_THOUSAND); Thread.sleep(TestConstant.ONE_THOUSAND);
//click add button //click add button
clickButton(CreateWorkflowLocator.CLICK_ADD_BUTTON);
System.out.println("submit workflow"); System.out.println("submit workflow");
clickButton(CreateWorkflowLocator.CLICK_ADD_BUTTON);
return ifTitleContains(CreateWorkflowData.CREATE_WORKFLOW_TITLE); return ifTitleContains(CreateWorkflowData.CREATE_WORKFLOW_TITLE);
} }
public boolean onlineWorkflow() throws InterruptedException {
clickElement(CreateWorkflowLocator.CLICK_WORKFLOW_DEFINE);
// click online button
System.out.println("Click online workflow button");
clickButton(CreateWorkflowLocator.CLICK_ONLINE_WORKFLOW_BUTTON);
return ifTitleContains(CreateWorkflowData.WORKFLOW_TITLE);
}
public boolean offlineWorkflow() throws InterruptedException {
clickElement(CreateWorkflowLocator.CLICK_WORKFLOW_DEFINE);
// click offline button
System.out.println("offline workflow");
Thread.sleep(500);
clickButton(CreateWorkflowLocator.CLICK_OFFLINE_WORKFLOW_BUTTON);
return ifTitleContains(CreateWorkflowData.WORKFLOW_TITLE);
}
public boolean deleteWorkflow() throws InterruptedException { public boolean deleteWorkflow() throws InterruptedException {
//click delete project //click delete workflow
Thread.sleep(500);
clickButton(CreateWorkflowLocator.DELETE_WORKFLOW_BOTTOM); clickButton(CreateWorkflowLocator.DELETE_WORKFLOW_BOTTOM);
//click confirm delete project //click confirm delete project

53
e2e/src/test/java/org/apache/dolphinscheduler/page/project/RunWorkflowPage.java

@ -0,0 +1,53 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.page.project;
import org.apache.dolphinscheduler.common.PageCommon;
import org.apache.dolphinscheduler.data.project.RunWorkflowData;
import org.apache.dolphinscheduler.locator.project.RunWorkflowLocator;
import org.openqa.selenium.WebDriver;
public class RunWorkflowPage extends PageCommon {
public RunWorkflowPage(WebDriver driver) {
super(driver);
}
public boolean runWorkflow() throws InterruptedException {
// click run workflow button
System.out.println("Click run workflow button");
Thread.sleep(1000);
clickButton(RunWorkflowLocator.CLICK_RUN_WORKFLOW_BUTTON);
Thread.sleep(1000);
clickElement(RunWorkflowLocator.SELECT_FAILURE_STRATEGY_END);
clickElement(RunWorkflowLocator.SELECT_FAILURE_STRATEGY_CONTINUE);
clickElement(RunWorkflowLocator.CLICK_NOTICE_STRATEGY);
clickElement(RunWorkflowLocator.SELECT_NOTICE_STRATEGY);
Thread.sleep(500);
clickElement(RunWorkflowLocator.CLICK_PROCESS_PRIORITY);
clickElement(RunWorkflowLocator.SELECT_PROCESS_PRIORITY_HIGHEST);
clickElement(RunWorkflowLocator.CLICK_WORKER_GROUP);
clickElement(RunWorkflowLocator.SELECT_WORKER_GROUP);
clickElement(RunWorkflowLocator.CLICK_NOTICE_GROUP);
clickElement(RunWorkflowLocator.SELECT_NOTICE_GROUP);
sendInput(RunWorkflowLocator.INPUT_RECIPIENT, RunWorkflowData.RECIPIENT);
sendInput(RunWorkflowLocator.INPUT_Cc,RunWorkflowData.Cc);
clickButton(RunWorkflowLocator.CLICK_RUNNING_BUTTON);
return ifTitleContains(RunWorkflowData.RUN_WORKFLOW_TITLE);
}
}

134
e2e/src/test/java/org/apache/dolphinscheduler/page/project/TimingPage.java

@ -0,0 +1,134 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.page.project;
import org.apache.dolphinscheduler.common.PageCommon;
import org.apache.dolphinscheduler.data.project.TimingData;
import org.apache.dolphinscheduler.locator.project.RunWorkflowLocator;
import org.apache.dolphinscheduler.locator.project.TimingLocator;
import org.openqa.selenium.WebDriver;
public class TimingPage extends PageCommon {
public TimingPage(WebDriver driver) {
super(driver);
}
/**
* create timing
*/
public boolean createTiming() throws InterruptedException {
// click timing button
System.out.println("Click timing button");
Thread.sleep(1000);
clickButton(TimingLocator.CLICK_TIMING_BUTTON);
System.out.println("Click execution timing button");
clickButton(TimingLocator.CLICK_EXECUTION_TIMING_BUTTON);
Thread.sleep(1000);
clickElement(TimingLocator.SELECT_FAILURE_STRATEGY_END);
clickElement(TimingLocator.SELECT_FAILURE_STRATEGY_CONTINUE);
clickElement(TimingLocator.CLICK_NOTICE_STRATEGY);
clickElement(TimingLocator.SELECT_NOTICE_STRATEGY);
Thread.sleep(500);
clickElement(TimingLocator.CLICK_PROCESS_PRIORITY);
clickElement(TimingLocator.SELECT_PROCESS_PRIORITY);
clickElement(TimingLocator.CLICK_WORKER_GROUP);
clickElement(TimingLocator.SELECT_WORKER_GROUP);
clickElement(TimingLocator.CLICK_NOTICE_GROUP);
clickElement(TimingLocator.SELECT_NOTICE_GROUP);
sendInput(TimingLocator.INPUT_RECIPIENT, TimingData.RECIPIENT);
sendInput(TimingLocator.INPUT_Cc,TimingData.Cc);
clickButton(TimingLocator.CLICK_CREATE_BUTTON);
return ifTitleContains(TimingData.WORKFLOW_TITLE);
}
/**
* edit timing
*/
public boolean editTiming() throws InterruptedException {
// click timing button
System.out.println("Click timing management button");
clickButton(TimingLocator.CLICK_TIMING_MANAGEMENT_BUTTON);
Thread.sleep(500);
System.out.println("Click edit timing button");
clickButton(TimingLocator.CLICK_EDIT_TIMING_BUTTON);
Thread.sleep(500);
System.out.println("Click execution timing button");
clickButton(TimingLocator.CLICK_EXECUTION_TIMING_BUTTON);
Thread.sleep(1000);
clickElement(TimingLocator.SELECT_FAILURE_STRATEGY_END);
clickElement(TimingLocator.SELECT_FAILURE_STRATEGY_CONTINUE);
clickElement(TimingLocator.CLICK_NOTICE_STRATEGY);
clickElement(TimingLocator.SELECT_NOTICE_STRATEGY);
Thread.sleep(500);
clickElement(TimingLocator.CLICK_PROCESS_PRIORITY);
clickElement(TimingLocator.SELECT_PROCESS_PRIORITY);
clickElement(TimingLocator.CLICK_WORKER_GROUP);
clickElement(TimingLocator.SELECT_WORKER_GROUP);
clickElement(TimingLocator.CLICK_NOTICE_GROUP);
clickElement(TimingLocator.SELECT_NOTICE_GROUP);
sendInput(TimingLocator.INPUT_RECIPIENT, TimingData.EDIT_RECIPIENT);
sendInput(TimingLocator.INPUT_Cc,TimingData.EDIT_Cc);
clickButton(TimingLocator.CLICK_CREATE_BUTTON);
return ifTitleContains(TimingData.TIMING_TITLE );
}
/**
* online timing
*/
public boolean onlineTiming() throws InterruptedException {
// click online timing button
System.out.println("Click online timing button");
Thread.sleep(500);
clickButton(TimingLocator.CLICK_ONLINE_TIMING_BUTTON);
return ifTitleContains(TimingData.TIMING_TITLE );
}
/**
* offline timing
*/
public boolean offlineTiming() throws InterruptedException {
// click offline timing button
System.out.println("Click offline timing button");
Thread.sleep(500);
clickButton(TimingLocator.CLICK_OFFLINE_TIMING_BUTTON);
return ifTitleContains(TimingData.TIMING_TITLE );
}
/**
* delete timing
*/
public boolean deleteTiming() throws InterruptedException {
// click offline timing button
System.out.println("Click delete timing button");
Thread.sleep(500);
clickButton(TimingLocator.CLICK_DELETE_TIMING_BUTTON);
clickButton(TimingLocator.CLICK_CONFIRM_DELETE_TIMING_BUTTON);
return ifTitleContains(TimingData.WORKFLOW_TITLE );
}
}

8
e2e/src/test/java/org/apache/dolphinscheduler/page/security/AlertManagePage.java

@ -37,14 +37,18 @@ public class AlertManagePage extends PageCommon {
*/ */
public boolean createAlert() throws InterruptedException { public boolean createAlert() throws InterruptedException {
// click alert manage // click alert manage
Thread.sleep(500);
System.out.println("start click alert manage button");
clickElement(AlertManageLocator.CLICK_ALERT_MANAGE); clickElement(AlertManageLocator.CLICK_ALERT_MANAGE);
Thread.sleep(1000); Thread.sleep(500);
// click create alert button // click create alert button
System.out.println("start click create alert button");
clickElement(AlertManageLocator.CLICK_CREATE_ALERT); clickElement(AlertManageLocator.CLICK_CREATE_ALERT);
Thread.sleep(1000); Thread.sleep(500);
// input alert data // input alert data
System.out.println("start input alert ");
sendInput(AlertManageLocator.INPUT_ALERT_NAME, AlertManageData.ALERT_NAME); sendInput(AlertManageLocator.INPUT_ALERT_NAME, AlertManageData.ALERT_NAME);
clickElement(AlertManageLocator.CLICK_ALERT_TYPE); clickElement(AlertManageLocator.CLICK_ALERT_TYPE);

8
e2e/src/test/java/org/apache/dolphinscheduler/page/security/QueueManagePage.java

@ -37,14 +37,18 @@ public class QueueManagePage extends PageCommon {
*/ */
public boolean createQueue() throws InterruptedException { public boolean createQueue() throws InterruptedException {
// click queue manage // click queue manage
Thread.sleep(500);
System.out.println("start click queue manage button");
clickElement(QueueManageLocator.CLICK_QUEUE_MANAGE); clickElement(QueueManageLocator.CLICK_QUEUE_MANAGE);
Thread.sleep(1000); Thread.sleep(500);
// click create queue button // click create queue button
System.out.println("start click create queue button");
clickElement(QueueManageLocator.CLICK_CREATE_QUEUE); clickElement(QueueManageLocator.CLICK_CREATE_QUEUE);
Thread.sleep(1000); Thread.sleep(500);
// input queue data // input queue data
System.out.println("start input queue");
sendInput(QueueManageLocator.INPUT_QUEUE_NAME, QueueManageData.QUEUE_NAME); sendInput(QueueManageLocator.INPUT_QUEUE_NAME, QueueManageData.QUEUE_NAME);
sendInput(QueueManageLocator.INPUT_QUEUE_VALUE, QueueManageData.QUEUE_VALUE); sendInput(QueueManageLocator.INPUT_QUEUE_VALUE, QueueManageData.QUEUE_VALUE);

79
e2e/src/test/java/org/apache/dolphinscheduler/page/security/UserManagePage.java

@ -27,7 +27,7 @@ public class UserManagePage extends PageCommon {
} }
/** /**
* createTenant * create user
* *
* @return Whether to enter the specified page after creat tenant * @return Whether to enter the specified page after creat tenant
*/ */
@ -45,8 +45,10 @@ public class UserManagePage extends PageCommon {
clickButton(UserManageLocator.SELECT_TENANT); clickButton(UserManageLocator.SELECT_TENANT);
clickButton(UserManageLocator.CLICK_QUEUE); clickButton(UserManageLocator.CLICK_QUEUE);
clickButton(UserManageLocator.SELECT_QUEUE); clickButton(UserManageLocator.SELECT_QUEUE);
sendInput(UserManageLocator.TENANT_INPUT_EMAIL, UserManageData.EMAIL); sendInput(UserManageLocator.INPUT_EMAIL, UserManageData.EMAIL);
sendInput(UserManageLocator.TENANT_INPUT_PHONE, UserManageData.PHONE); sendInput(UserManageLocator.INPUT_PHONE, UserManageData.PHONE);
clickElement(UserManageLocator.SELECT_STOP_STATE);
clickElement(UserManageLocator.SELECT_ENABLE_STATE);
// click button // click button
clickButton(UserManageLocator.SUBMIT); clickButton(UserManageLocator.SUBMIT);
@ -55,15 +57,84 @@ public class UserManagePage extends PageCommon {
return ifTitleContains(UserManageData.USER_MANAGE); return ifTitleContains(UserManageData.USER_MANAGE);
} }
public boolean deleteUser() throws InterruptedException { /**
* edit general user
*/
public boolean editGeneralUser() throws InterruptedException {
//edit general user
// click user manage
System.out.println("start edit general user");
Thread.sleep(500);
clickElement(UserManageLocator.CLICK_USER_MANAGE);
// click edit user button
clickButton(UserManageLocator.EDIT_GENERAL_USER_BUTTON );
// input user data
clearSendInput(UserManageLocator.INPUT_USERNAME, UserManageData.EDIT_USERNAME);
clearSendInput(UserManageLocator.INPUT_PASSWORD, UserManageData.EDIT_PASSWORD);
clickButton(UserManageLocator.CLICK_TENANT);
clickButton(UserManageLocator.SELECT_TENANT);
clickButton(UserManageLocator.CLICK_QUEUE);
clickButton(UserManageLocator.SELECT_QUEUE);
clearSendInput(UserManageLocator.INPUT_EMAIL, UserManageData.EDIT_EMAIL);
clearSendInput(UserManageLocator.INPUT_PHONE, UserManageData.EDIT_PHONE);
clickElement(UserManageLocator.SELECT_STOP_STATE);
clickElement(UserManageLocator.SELECT_ENABLE_STATE);
// click button
clickButton(UserManageLocator.SUBMIT);
System.out.println("end edit general user");
// Whether to enter the specified page after submit
return ifTitleContains(UserManageData.USER_MANAGE);
}
/**
* edit admin user
*/
public boolean editAdminlUser() throws InterruptedException {
//edit admin user
// click user manage
System.out.println("start edit admin user");
Thread.sleep(500);
clickElement(UserManageLocator.CLICK_USER_MANAGE);
// click edit user button
clickButton(UserManageLocator.EDIT_ADMIN_USER_BUTTON );
// select tenant
clickButton(UserManageLocator.CLICK_TENANT);
clickButton(UserManageLocator.SELECT_TENANT);
// click button
clickButton(UserManageLocator.SUBMIT);
System.out.println("end edit admin user");
Thread.sleep(500);
// Whether to enter the specified page after submit
return ifTitleContains(UserManageData.USER_MANAGE);
}
/**
* delete user
*/
public boolean deleteUser() throws InterruptedException {
System.out.println("jump to user manage");
// click user manage // click user manage
clickElement(UserManageLocator.CLICK_USER_MANAGE); clickElement(UserManageLocator.CLICK_USER_MANAGE);
// click delete user button // click delete user button
System.out.println("click delete user");
clickButton(UserManageLocator.DELETE_USER_BUTTON ); clickButton(UserManageLocator.DELETE_USER_BUTTON );
// click confirm delete button // click confirm delete button
System.out.println("click confirm delete user");
clickButton(UserManageLocator.CONFIRM_DELETE_USER_BUTTON); clickButton(UserManageLocator.CONFIRM_DELETE_USER_BUTTON);
// Whether to enter the specified page after submit // Whether to enter the specified page after submit

11
e2e/src/test/java/org/apache/dolphinscheduler/testcase/testDeleteData/TestDeleteWorkflow.java

@ -29,12 +29,15 @@ public class TestDeleteWorkflow extends BaseTest {
public void testDeleteWorkflow() throws InterruptedException { public void testDeleteWorkflow() throws InterruptedException {
createProjectPage = new CreateProjectPage(driver); createProjectPage = new CreateProjectPage(driver);
//jump to project manage page //jump to project manage page
System.out.println("jump to the project manage page to delete workflow"); // System.out.println("jump to the project manage page to delete workflow");
createProjectPage.jumpProjectManagePage(); // createProjectPage.jumpProjectManagePage();
createWorkflowPage = new CreateWorkflowPage(driver); createWorkflowPage = new CreateWorkflowPage(driver);
createWorkflowPage.jumpWorkflowPage(); // createWorkflowPage.jumpWorkflowPage();
//assert tenant manage page System.out.println("start offline workflow");
assert createWorkflowPage.offlineWorkflow();
System.out.println("end offline workflow");
System.out.println("start delete workflow"); System.out.println("start delete workflow");
assert createWorkflowPage.deleteWorkflow(); assert createWorkflowPage.deleteWorkflow();
System.out.println("end delete workflow"); System.out.println("end delete workflow");

4
e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestCreateWorkflow.java

@ -40,6 +40,10 @@ public class TestCreateWorkflow extends BaseTest {
assert createWorkflowPage.createWorkflow(); assert createWorkflowPage.createWorkflow();
assert createWorkflowPage.saveWorkflow(); assert createWorkflowPage.saveWorkflow();
System.out.println("end create workflow"); System.out.println("end create workflow");
System.out.println("start online workflow");
assert createWorkflowPage.onlineWorkflow();
System.out.println("end online workflow");
System.out.println("==================================="); System.out.println("===================================");
} }
} }

41
e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestRunWorkflow.java

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.testcase.testProject;
import org.apache.dolphinscheduler.base.BaseTest;
import org.apache.dolphinscheduler.page.project.CreateProjectPage;
import org.apache.dolphinscheduler.page.project.CreateWorkflowPage;
import org.apache.dolphinscheduler.page.project.RunWorkflowPage;
import org.testng.annotations.Test;
public class TestRunWorkflow extends BaseTest {
private CreateWorkflowPage createWorkflowPage;
private CreateProjectPage createProjectPage;
private RunWorkflowPage runWorkflowPage;
@Test(groups={"functionTests","runWorkflow"},dependsOnGroups = { "login","workflow" },description = "TestRunWorkflow")
public void testRunWorkflow() throws InterruptedException {
runWorkflowPage = new RunWorkflowPage(driver);
createProjectPage = new CreateProjectPage(driver);
System.out.println("start run workflow");
assert runWorkflowPage.runWorkflow();
System.out.println("end run workflow");
System.out.println("===================================");
}
}

75
e2e/src/test/java/org/apache/dolphinscheduler/testcase/testProject/TestTiming.java

@ -0,0 +1,75 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.testcase.testProject;
import org.apache.dolphinscheduler.base.BaseTest;
import org.apache.dolphinscheduler.page.project.TimingPage;
import org.testng.annotations.Test;
public class TestTiming extends BaseTest {
private TimingPage timingPage;
@Test(groups={"functionTests","createTiming"},dependsOnGroups = { "login","workflow"},description = "TestCreateTiming")
public void testCreateTiming() throws InterruptedException {
timingPage = new TimingPage(driver);
System.out.println("start create timing");
assert timingPage.createTiming();
System.out.println("end create timing");
System.out.println("===================================");
}
@Test(groups={"functionTests","timing"},dependsOnGroups = { "login","workflow"},description = "TestEditTiming")
public void testEditTiming() throws InterruptedException {
timingPage = new TimingPage(driver);
System.out.println("start edit timing");
assert timingPage.editTiming();
System.out.println("end edit timing");
System.out.println("===================================");
}
@Test(groups={"functionTests","timing"},dependsOnGroups = { "login","workflow" },description = "TestOnlineTiming")
public void testOnlineTiming() throws InterruptedException {
timingPage = new TimingPage(driver);
System.out.println("start online timing");
assert timingPage.onlineTiming();
System.out.println("end online timing");
System.out.println("===================================");
}
@Test(groups={"functionTests","timing"},dependsOnGroups = { "login","workflow"},description = "TestOfflineTiming")
public void testOfflineTiming() throws InterruptedException {
timingPage = new TimingPage(driver);
System.out.println("start offline timing");
assert timingPage.offlineTiming();
System.out.println("end offline timing");
System.out.println("===================================");
}
@Test(groups={"functionTests","timing"},dependsOnGroups = { "login","workflow"},description = "TestDeleteTiming")
public void testDeleteTiming() throws InterruptedException {
timingPage = new TimingPage(driver);
System.out.println("start delete timing");
assert timingPage.deleteTiming();
System.out.println("end delete timing");
System.out.println("===================================");
}
}

18
e2e/src/test/java/org/apache/dolphinscheduler/testcase/testSecurity/TestUserManage.java

@ -23,14 +23,26 @@ import org.testng.annotations.Test;
public class TestUserManage extends BaseTest { public class TestUserManage extends BaseTest {
private UserManagePage userManagePage; private UserManagePage userManagePage;
@Test(groups={"functionTests","user"},dependsOnGroups = { "login" },description = "TestUserManage") @Test(groups={"functionTests","user"},dependsOnGroups = { "login" },description = "TestCreateUser")
public void testUserManage() throws InterruptedException { public void testCreateUser() throws InterruptedException {
userManagePage = new UserManagePage(driver); userManagePage = new UserManagePage(driver);
//assert user manage page //create user
System.out.println("start create user"); System.out.println("start create user");
assert userManagePage.createUser(); assert userManagePage.createUser();
System.out.println("end create user"); System.out.println("end create user");
System.out.println("==================================="); System.out.println("===================================");
}
@Test(groups={"functionTests","user"},dependsOnGroups = { "login" },description = "TestEditUser")
public void testEditUser() throws InterruptedException {
userManagePage = new UserManagePage(driver);
//edit user
System.out.println("start edit user");
assert userManagePage.editGeneralUser();
assert userManagePage.editAdminlUser();
System.out.println("end edit user");
System.out.println("===================================");
} }
} }

19
e2e/testng.xml

@ -26,13 +26,30 @@
<classes> <classes>
<class name="org.apache.dolphinscheduler.testcase.TestLogin"></class> <class name="org.apache.dolphinscheduler.testcase.TestLogin"></class>
<class name="org.apache.dolphinscheduler.testcase.testSecurity.TestTenantManage"></class> <class name="org.apache.dolphinscheduler.testcase.testSecurity.TestTenantManage"></class>
<class name="org.apache.dolphinscheduler.testcase.testSecurity.TestUserManage"></class>
<class name="org.apache.dolphinscheduler.testcase.testSecurity.TestAlertManage"></class> <class name="org.apache.dolphinscheduler.testcase.testSecurity.TestAlertManage"></class>
<class name="org.apache.dolphinscheduler.testcase.testSecurity.TestQueueManage"></class> <class name="org.apache.dolphinscheduler.testcase.testSecurity.TestQueueManage"></class>
<class name="org.apache.dolphinscheduler.testcase.testSecurity.TestTokenManage"></class> <class name="org.apache.dolphinscheduler.testcase.testSecurity.TestTokenManage"></class>
<class name="org.apache.dolphinscheduler.testcase.testSecurity.TestUserManage">
<methods>
<include name="testCreateUser" />
<include name="testEditUser" />
</methods>
</class>
<class name="org.apache.dolphinscheduler.testcase.testProject.TestCreateProject"></class> <class name="org.apache.dolphinscheduler.testcase.testProject.TestCreateProject"></class>
<class name="org.apache.dolphinscheduler.testcase.testProject.TestCreateWorkflow"></class> <class name="org.apache.dolphinscheduler.testcase.testProject.TestCreateWorkflow"></class>
<class name="org.apache.dolphinscheduler.testcase.testProject.TestRunWorkflow"></class>
<class name="org.apache.dolphinscheduler.testcase.testProject.TestTiming">
<methods>
<include name="testCreateTiming" />
<include name="testEditTiming" />
<include name="testOnlineTiming" />
<include name="testOfflineTiming" />
<include name="testDeleteTiming" />
</methods>
</class>
<class name="org.apache.dolphinscheduler.testcase.testDeleteData.TestDeleteWorkflow"></class> <class name="org.apache.dolphinscheduler.testcase.testDeleteData.TestDeleteWorkflow"></class>
<class name="org.apache.dolphinscheduler.testcase.testDeleteData.TestDeleteProject"></class> <class name="org.apache.dolphinscheduler.testcase.testDeleteData.TestDeleteProject"></class>

7
pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>org.apache.dolphinscheduler</groupId> <groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId> <artifactId>dolphinscheduler</artifactId>
<version>1.3.0-SNAPSHOT</version> <version>1.3.2-SNAPSHOT</version>
<packaging>pom</packaging> <packaging>pom</packaging>
<name>${project.artifactId}</name> <name>${project.artifactId}</name>
<url>http://dolphinscheduler.apache.org</url> <url>http://dolphinscheduler.apache.org</url>
@ -817,7 +817,7 @@
<include>**/server/utils/ProcessUtilsTest.java</include> <include>**/server/utils/ProcessUtilsTest.java</include>
<include>**/server/utils/SparkArgsUtilsTest.java</include> <include>**/server/utils/SparkArgsUtilsTest.java</include>
<!--<include>**/server/worker/processor/TaskCallbackServiceTest.java</include>--> <!--<include>**/server/worker/processor/TaskCallbackServiceTest.java</include>-->
<!--<include>**/server/worker/registry/WorkerRegistryTest.java</include>--> <include>**/server/worker/registry/WorkerRegistryTest.java</include>
<include>**/server/worker/shell/ShellCommandExecutorTest.java</include> <include>**/server/worker/shell/ShellCommandExecutorTest.java</include>
<include>**/server/worker/sql/SqlExecutorTest.java</include> <include>**/server/worker/sql/SqlExecutorTest.java</include>
<include>**/server/worker/task/spark/SparkTaskTest.java</include> <include>**/server/worker/task/spark/SparkTaskTest.java</include>
@ -852,6 +852,7 @@
<include>**/dao/utils/DagHelperTest.java</include> <include>**/dao/utils/DagHelperTest.java</include>
<include>**/dao/AlertDaoTest.java</include> <include>**/dao/AlertDaoTest.java</include>
<include>**/dao/datasource/OracleDataSourceTest.java</include> <include>**/dao/datasource/OracleDataSourceTest.java</include>
<include>**/dao/datasource/HiveDataSourceTest.java</include>
<include>**/dao/upgrade/ProcessDefinitionDaoTest.java</include> <include>**/dao/upgrade/ProcessDefinitionDaoTest.java</include>
<include>**/dao/upgrade/WokrerGrouopDaoTest.java</include> <include>**/dao/upgrade/WokrerGrouopDaoTest.java</include>
<include>**/dao/upgrade/UpgradeDaoTest.java</include> <include>**/dao/upgrade/UpgradeDaoTest.java</include>
@ -915,7 +916,7 @@
<exclude>**/dist/**</exclude> <exclude>**/dist/**</exclude>
<exclude>**/licenses/**</exclude> <exclude>**/licenses/**</exclude>
<exclude>.github/**</exclude> <exclude>.github/**</exclude>
<exclude>sql/soft_version</exclude> <exclude>**/sql/soft_version</exclude>
<exclude>**/common/utils/ScriptRunner.java</exclude> <exclude>**/common/utils/ScriptRunner.java</exclude>
<exclude>**/*.json</exclude> <exclude>**/*.json</exclude>
<!-- document files --> <!-- document files -->

2
sql/soft_version

@ -1 +1 @@
1.3.0 1.3.1

1
tools/dependencies/known-dependencies.txt

@ -24,6 +24,7 @@ commons-compress-1.4.1.jar
commons-compiler-3.0.12.jar commons-compiler-3.0.12.jar
commons-configuration-1.10.jar commons-configuration-1.10.jar
commons-daemon-1.0.13.jar commons-daemon-1.0.13.jar
commons-beanutils-1.7.0.jar
commons-dbcp-1.4.jar commons-dbcp-1.4.jar
commons-email-1.5.jar commons-email-1.5.jar
commons-httpclient-3.0.1.jar commons-httpclient-3.0.1.jar

Loading…
Cancel
Save