Browse Source

Merge remote-tracking branch 'upstream/dev' into dev

# Conflicts:
#	dolphinscheduler-service/src/main/resources/logback-zookeeper.xml
pull/3/MERGE
dailidong 4 years ago
parent
commit
0f42af8d30
  1. 3
      docker/build/conf/nginx/dolphinscheduler.conf
  2. 1
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java
  3. 40
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java
  4. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java
  5. 81
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageController.java
  6. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java
  7. 17
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
  8. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java
  9. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java
  10. 6
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java
  11. 49
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java
  12. 10
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
  13. 13
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java
  14. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SessionService.java
  15. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java
  16. 17
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java
  17. 97
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkFlowLineageService.java
  18. 1
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java
  19. 3
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceControllerTest.java
  20. 69
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageControllerTest.java
  21. 2
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java
  22. 24
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java
  23. 4
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java
  24. 38
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java
  25. 88
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkFlowLineageServiceTest.java
  26. 10
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
  27. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java
  28. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskStateType.java
  29. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskType.java
  30. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java
  31. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java
  32. 46
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java
  33. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
  34. 103
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java
  35. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java
  36. 10
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java
  37. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java
  38. 6
      dolphinscheduler-common/src/main/resources/common.properties
  39. 39
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java
  40. 14
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
  41. 8
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java
  42. 2
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java
  43. 7
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java
  44. 2
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java
  45. 14
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java
  46. 94
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowLineage.java
  47. 38
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowRelation.java
  48. 13
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java
  49. 32
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapper.java
  50. 10
      dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml
  51. 103
      dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapper.xml
  52. 49
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java
  53. 35
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSourceTest.java
  54. 1
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java
  55. 1
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java
  56. 1
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java
  57. 1
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java
  58. 87
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java
  59. 10
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java
  60. 62
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapperTest.java
  61. 1
      dolphinscheduler-dist/pom.xml
  62. 1
      dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml
  63. 1
      dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml
  64. 4
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
  65. 1
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java
  66. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java
  67. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java
  68. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java
  69. 65
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java
  70. 29
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
  71. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java
  72. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
  73. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskManager.java
  74. 3
      dolphinscheduler-server/src/main/resources/config/install_config.conf
  75. 2
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java
  76. 2
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java
  77. 21
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java
  78. 21
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java
  79. 2
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java
  80. 1
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java
  81. 4
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java
  82. 1
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryTest.java
  83. 71
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java
  84. 12
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManagerTest.java
  85. 11
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTest.java
  86. 1
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryTest.java
  87. 1
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogPromise.java
  88. 19
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
  89. 1
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperCachedOperator.java
  90. 4
      dolphinscheduler-service/src/main/resources/logback-zookeeper.xml
  91. 1
      dolphinscheduler-ui/build/config.js
  92. 6
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js
  93. 3
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss
  94. 12
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue
  95. 6
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue
  96. 467
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/waterdrop.vue
  97. 17
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue
  98. BIN
      dolphinscheduler-ui/src/js/conf/home/pages/dag/img/toolbar_WATERDROP.png
  99. 4
      dolphinscheduler-ui/src/js/conf/home/pages/projects/index.vue
  100. 2
      dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/_source/instanceConditions/common.js
  101. Some files were not shown because too many files have changed in this diff Show More

3
docker/build/conf/nginx/dolphinscheduler.conf

@ -24,6 +24,9 @@ server {
root /opt/dolphinscheduler/ui;
index index.html index.html;
}
location /dolphinscheduler/ui{
alias /opt/dolphinscheduler/ui;
}
location /dolphinscheduler {
proxy_pass http://FRONTEND_API_SERVER_HOST:FRONTEND_API_SERVER_PORT;
proxy_set_header Host $host;

1
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java

@ -22,7 +22,6 @@ import org.springframework.boot.web.servlet.ServletComponentScan;
import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.FilterType;
import springfox.documentation.swagger2.annotations.EnableSwagger2;
@SpringBootApplication
@ServletComponentScan

40
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java

@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.User;
import io.swagger.annotations.*;
import org.slf4j.Logger;
@ -204,6 +205,39 @@ public class ProcessInstanceController extends BaseController {
return returnDataList(result);
}
/**
* query top n process instance order by running duration
*
* @param loginUser login user
* @param projectName project name
* @param size number of process instance
* @param startTime start time
* @param endTime end time
* @return list of process instance
*/
@ApiOperation(value = "queryTopNLongestRunningProcessInstance", notes = "QUERY_TOPN_LONGEST_RUNNING_PROCESS_INSTANCE_NOTES")
@ApiImplicitParams({
@ApiImplicitParam(name = "size", value = "PROCESS_INSTANCE_SIZE", dataType = "Int", example = "10"),
@ApiImplicitParam(name = "startTime", value = "PROCESS_INSTANCE_START_TIME", dataType = "String"),
@ApiImplicitParam(name = "endTime", value = "PROCESS_INSTANCE_END_TIME", dataType = "String"),
})
@GetMapping(value = "/top-n")
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_PROCESS_INSTANCE_BY_ID_ERROR)
public Result<ProcessInstance> queryTopNLongestRunningProcessInstance(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName,
@RequestParam("size") Integer size,
@RequestParam(value = "startTime",required = true) String startTime,
@RequestParam(value = "endTime",required = true) String endTime
) {
projectName=ParameterUtils.handleEscapes(projectName);
logger.info("query top {} SUCCESS process instance order by running time whprojectNameich started between {} and {} ,login user:{},project name:{}", size, startTime, endTime,
loginUser.getUserName(), projectName);
Map<String,Object> result=processInstanceService.queryTopNLongestRunningProcessInstance(loginUser, projectName, size, startTime, endTime);
return returnDataList(result);
}
/**
* delete process instance by id, at the same time,
* delete task instance and their mapping relation data
@ -220,9 +254,9 @@ public class ProcessInstanceController extends BaseController {
@GetMapping(value = "/delete")
@ResponseStatus(HttpStatus.OK)
@ApiException(DELETE_PROCESS_INSTANCE_BY_ID_ERROR)
public Result deleteProcessInstanceById(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName,
@RequestParam("processInstanceId") Integer processInstanceId
public Result<ProcessInstance> deleteProcessInstanceById(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName,
@RequestParam("processInstanceId") Integer processInstanceId
) {
logger.info("delete process instance by id, login user:{}, project name:{}, process instance id:{}",
loginUser.getUserName(), projectName, processInstanceId);

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java

@ -52,7 +52,7 @@ public class SchedulerController extends BaseController {
public static final String DEFAULT_WARNING_TYPE = "NONE";
public static final String DEFAULT_NOTIFY_GROUP_ID = "1";
public static final String DEFAULT_FAILURE_POLICY = "CONTINUE";
public static final String DEFAULT_PROCESS_INSTANCE_PRIORITY = "MEDIUM";
@Autowired
private SchedulerService schedulerService;
@ -99,7 +99,7 @@ public class SchedulerController extends BaseController {
@RequestParam(value = "receivers", required = false) String receivers,
@RequestParam(value = "receiversCc", required = false) String receiversCc,
@RequestParam(value = "workerGroup", required = false, defaultValue = "default") String workerGroup,
@RequestParam(value = "processInstancePriority", required = false) Priority processInstancePriority) throws IOException {
@RequestParam(value = "processInstancePriority", required = false, defaultValue = DEFAULT_PROCESS_INSTANCE_PRIORITY) Priority processInstancePriority) throws IOException {
logger.info("login user {}, project name: {}, process name: {}, create schedule: {}, warning type: {}, warning group id: {}," +
"failure policy: {},receivers : {},receiversCc : {},processInstancePriority : {}, workGroupId:{}",
loginUser.getUserName(), projectName, processDefinitionId, schedule, warningType, warningGroupId,

81
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageController.java

@ -0,0 +1,81 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.controller;
import org.apache.dolphinscheduler.api.service.WorkFlowLineageService;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import io.swagger.annotations.ApiParam;
import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.*;
import springfox.documentation.annotations.ApiIgnore;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_WORKFLOW_LINEAGE_ERROR;
@RestController
@RequestMapping("lineages/{projectId}")
public class WorkFlowLineageController extends BaseController {
private static final Logger logger = LoggerFactory.getLogger(WorkFlowLineageController.class);
@Autowired
private WorkFlowLineageService workFlowLineageService;
@GetMapping(value="/list-name")
@ResponseStatus(HttpStatus.OK)
public Result<List<WorkFlowLineage>> queryWorkFlowLineageByName(@ApiIgnore @RequestParam(value = "searchVal", required = false) String searchVal, @ApiParam(name = "projectId", value = "PROJECT_ID", required = true) @PathVariable int projectId) {
try {
searchVal = ParameterUtils.handleEscapes(searchVal);
Map<String, Object> result = workFlowLineageService.queryWorkFlowLineageByName(searchVal,projectId);
return returnDataList(result);
} catch (Exception e){
logger.error(QUERY_WORKFLOW_LINEAGE_ERROR.getMsg(),e);
return error(QUERY_WORKFLOW_LINEAGE_ERROR.getCode(), QUERY_WORKFLOW_LINEAGE_ERROR.getMsg());
}
}
@GetMapping(value="/list-ids")
@ResponseStatus(HttpStatus.OK)
public Result<Map<String, Object>> queryWorkFlowLineageByIds(@ApiIgnore @RequestParam(value = "ids", required = false) String ids,@ApiParam(name = "projectId", value = "PROJECT_ID", required = true) @PathVariable int projectId) {
try {
ids = ParameterUtils.handleEscapes(ids);
Set<Integer> idsSet = new HashSet<>();
if(ids != null) {
String[] idsStr = ids.split(",");
for (String id : idsStr)
{
idsSet.add(Integer.parseInt(id));
}
}
Map<String, Object> result = workFlowLineageService.queryWorkFlowLineageByIds(idsSet, projectId);
return returnDataList(result);
} catch (Exception e){
logger.error(QUERY_WORKFLOW_LINEAGE_ERROR.getMsg(),e);
return error(QUERY_WORKFLOW_LINEAGE_ERROR.getCode(), QUERY_WORKFLOW_LINEAGE_ERROR.getMsg());
}
}
}

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java

@ -62,7 +62,7 @@ public class TaskCountDto {
case SUBMITTED_SUCCESS:
submittedSuccess += taskInstanceStateCount.getCount();
break;
case RUNNING_EXEUTION:
case RUNNING_EXECUTION:
runningExeution += taskInstanceStateCount.getCount();
break;
case READY_PAUSE:
@ -99,7 +99,7 @@ public class TaskCountDto {
}
this.taskCountDtos = new ArrayList<>();
this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.SUBMITTED_SUCCESS, submittedSuccess));
this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.RUNNING_EXEUTION, runningExeution));
this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.RUNNING_EXECUTION, runningExeution));
this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.READY_PAUSE, readyPause));
this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.PAUSE, pause));
this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.READY_STOP, readyStop));

17
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java

@ -168,15 +168,15 @@ public enum Status {
PREVIEW_SCHEDULE_ERROR(10139,"preview schedule error", "预览调度配置错误"),
PARSE_TO_CRON_EXPRESSION_ERROR(10140,"parse cron to cron expression error", "解析调度表达式错误"),
SCHEDULE_START_TIME_END_TIME_SAME(10141,"The start time must not be the same as the end", "开始时间不能和结束时间一样"),
DELETE_TENANT_BY_ID_FAIL(10142,"delete tenant by id fail, for there are {0} process instances in executing using it", "删除租户失败,有[{0}]个运行中的工作流实例正在使用"),
DELETE_TENANT_BY_ID_FAIL_DEFINES(10143,"delete tenant by id fail, for there are {0} process definitions using it", "删除租户失败,有[{0}]个工作流定义正在使用"),
DELETE_TENANT_BY_ID_FAIL_USERS(10144,"delete tenant by id fail, for there are {0} users using it", "删除租户失败,有[{0}]个用户正在使用"),
DELETE_WORKER_GROUP_BY_ID_FAIL(10145,"delete worker group by id fail, for there are {0} process instances in executing using it", "删除Worker分组失败,有[{0}]个运行中的工作流实例正在使用"),
QUERY_WORKER_GROUP_FAIL(10146,"query worker group fail ", "查询worker分组失败"),
DELETE_WORKER_GROUP_FAIL(10147,"delete worker group fail ", "删除worker分组失败"),
DELETE_TENANT_BY_ID_FAIL(100142,"delete tenant by id fail, for there are {0} process instances in executing using it", "删除租户失败,有[{0}]个运行中的工作流实例正在使用"),
DELETE_TENANT_BY_ID_FAIL_DEFINES(100143,"delete tenant by id fail, for there are {0} process definitions using it", "删除租户失败,有[{0}]个工作流定义正在使用"),
DELETE_TENANT_BY_ID_FAIL_USERS(100144,"delete tenant by id fail, for there are {0} users using it", "删除租户失败,有[{0}]个用户正在使用"),
DELETE_WORKER_GROUP_BY_ID_FAIL(100145,"delete worker group by id fail, for there are {0} process instances in executing using it", "删除Worker分组失败,有[{0}]个运行中的工作流实例正在使用"),
QUERY_WORKER_GROUP_FAIL(100146,"query worker group fail ", "查询worker分组失败"),
DELETE_WORKER_GROUP_FAIL(100147,"delete worker group fail ", "删除worker分组失败"),
QUERY_WORKFLOW_LINEAGE_ERROR(10143,"query workflow lineage error", "查询血缘失败"),
COPY_PROCESS_DEFINITION_ERROR(10148,"copy process definition error", "复制工作流错误"),
USER_DISABLED(10149,"The current user is disabled", "当前用户已停用"),
UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found", "UDF函数不存在"),
UDF_FUNCTION_EXISTS(20002, "UDF function already exists", "UDF函数已存在"),
RESOURCE_NOT_EXIST(20004, "resource not exist", "资源不存在"),
@ -249,7 +249,8 @@ public enum Status {
COMMAND_STATE_COUNT_ERROR(80001,"task instance state count error", "查询各状态任务实例数错误"),
NEGTIVE_SIZE_NUMBER_ERROR(80002,"query size number error","查询size错误"),
START_TIME_BIGGER_THAN_END_TIME_ERROR(80003,"start time bigger than end time error","开始时间在结束时间之后错误"),
QUEUE_COUNT_ERROR(90001,"queue count error", "查询队列数据错误"),
KERBEROS_STARTUP_STATE(100001,"get kerberos startup state error", "获取kerberos启动状态错误"),

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java

@ -177,7 +177,7 @@ public class AlertGroupService extends BaseService{
* @param id alert group id
* @return delete result code
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public Map<String, Object> delAlertgroupById(User loginUser, int id) {
Map<String, Object> result = new HashMap<>(5);
result.put(Constants.STATUS, false);

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java

@ -527,7 +527,7 @@ public class DataSourceService extends BaseService{
parameterMap.put(Constants.DATABASE, database);
parameterMap.put(Constants.JDBC_URL, jdbcUrl);
parameterMap.put(Constants.USER, userName);
parameterMap.put(Constants.PASSWORD, password);
parameterMap.put(Constants.PASSWORD, CommonUtils.encodePassword(password));
if (CommonUtils.getKerberosStartupState() &&
(type == DbType.HIVE || type == DbType.SPARK)){
parameterMap.put(Constants.PRINCIPAL,principal);
@ -600,7 +600,7 @@ public class DataSourceService extends BaseService{
* @param datasourceId data source id
* @return delete result code
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public Result delete(User loginUser, int datasourceId) {
Result result = new Result();
try {

6
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java

@ -438,7 +438,7 @@ public class ProcessDefinitionService extends BaseDAGService {
* @param processDefinitionId process definition id
* @return delete result code
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public Map<String, Object> deleteProcessDefinitionById(User loginUser, String projectName, Integer processDefinitionId) {
Map<String, Object> result = new HashMap<>(5);
@ -504,7 +504,7 @@ public class ProcessDefinitionService extends BaseDAGService {
* @param releaseState release state
* @return release result code
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public Map<String, Object> releaseProcessDefinition(User loginUser, String projectName, int id, int releaseState) {
HashMap<String, Object> result = new HashMap<>();
Project project = projectMapper.queryByName(projectName);
@ -751,7 +751,7 @@ public class ProcessDefinitionService extends BaseDAGService {
* @param currentProjectName current project name
* @return import process
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public Map<String, Object> importProcessDefinition(User loginUser, MultipartFile file, String currentProjectName) {
Map<String, Object> result = new HashMap<>(5);
String processMetaJson = FileUtils.file2String(file);

49
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java

@ -98,6 +98,53 @@ public class ProcessInstanceService extends BaseDAGService {
@Autowired
UsersService usersService;
/**
* return top n SUCCESS process instance order by running time which started between startTime and endTime
* @param loginUser
* @param projectName
* @param size
* @param startTime
* @param endTime
* @return
*/
public Map<String, Object> queryTopNLongestRunningProcessInstance(User loginUser, String projectName, int size, String startTime, String endTime) {
Map<String, Object> result = new HashMap<>();
Project project = projectMapper.queryByName(projectName);
Map<String, Object> checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName);
Status resultEnum = (Status) checkResult.get(Constants.STATUS);
if (resultEnum != Status.SUCCESS) {
return checkResult;
}
if (0 > size) {
putMsg(result, Status.NEGTIVE_SIZE_NUMBER_ERROR, size);
return result;
}
if (Objects.isNull(startTime)) {
putMsg(result, Status.DATA_IS_NULL, Constants.START_TIME);
return result;
}
Date start = DateUtils.stringToDate(startTime);
if (Objects.isNull(endTime)) {
putMsg(result, Status.DATA_IS_NULL, Constants.END_TIME);
return result;
}
Date end = DateUtils.stringToDate(endTime);
if(start == null || end == null) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "startDate,endDate");
return result;
}
if (start.getTime() > end.getTime()) {
putMsg(result, Status.START_TIME_BIGGER_THAN_END_TIME_ERROR, startTime, endTime);
return result;
}
List<ProcessInstance> processInstances = processInstanceMapper.queryTopNProcessInstance(size, start, end, ExecutionStatus.SUCCESS);
result.put(DATA_LIST, processInstances);
putMsg(result, Status.SUCCESS);
return result;
}
/**
* query process instance by id
*
@ -466,7 +513,7 @@ public class ProcessInstanceService extends BaseDAGService {
* @param processInstanceId process instance id
* @return delete result code
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public Map<String, Object> deleteProcessInstanceById(User loginUser, String projectName, Integer processInstanceId) {
Map<String, Object> result = new HashMap<>(5);

10
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java

@ -86,7 +86,7 @@ public class ResourcesService extends BaseService {
* @param currentDir current directory
* @return create directory result
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public Result createDirectory(User loginUser,
String name,
String description,
@ -160,7 +160,7 @@ public class ResourcesService extends BaseService {
* @param currentDir current directory
* @return create result code
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public Result createResource(User loginUser,
String name,
String desc,
@ -288,7 +288,7 @@ public class ResourcesService extends BaseService {
* @param type resource type
* @return update result code
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public Result updateResource(User loginUser,
int resourceId,
String name,
@ -827,7 +827,7 @@ public class ResourcesService extends BaseService {
* @param content content
* @return create result code
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public Result onlineCreateResource(User loginUser, ResourceType type, String fileName, String fileSuffix, String desc, String content,int pid,String currentDirectory) {
Result result = new Result();
// if resource upload startup
@ -889,7 +889,7 @@ public class ResourcesService extends BaseService {
* @param content content
* @return update result cod
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public Result updateResourceContent(int resourceId, String content) {
Result result = new Result();

13
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java

@ -47,7 +47,6 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.io.IOException;
import java.text.ParseException;
import java.util.*;
@ -95,9 +94,8 @@ public class SchedulerService extends BaseService {
* @param receiversCc receivers cc
* @param workerGroup worker group
* @return create result code
* @throws IOException ioexception
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public Map<String, Object> insertSchedule(User loginUser, String projectName,
Integer processDefineId,
String schedule,
@ -107,7 +105,7 @@ public class SchedulerService extends BaseService {
String receivers,
String receiversCc,
Priority processInstancePriority,
String workerGroup) throws IOException {
String workerGroup) {
Map<String, Object> result = new HashMap<String, Object>(5);
@ -192,9 +190,8 @@ public class SchedulerService extends BaseService {
* @param receivers receivers
* @param scheduleStatus schedule status
* @return update result code
* @throws IOException ioexception
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public Map<String, Object> updateSchedule(User loginUser,
String projectName,
Integer id,
@ -206,7 +203,7 @@ public class SchedulerService extends BaseService {
String receiversCc,
ReleaseState scheduleStatus,
Priority processInstancePriority,
String workerGroup) throws IOException {
String workerGroup) {
Map<String, Object> result = new HashMap<String, Object>(5);
Project project = projectMapper.queryByName(projectName);
@ -296,7 +293,7 @@ public class SchedulerService extends BaseService {
* @param scheduleStatus schedule status
* @return publish result code
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public Map<String, Object> setScheduleState(User loginUser,
String projectName,
Integer id,

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SessionService.java

@ -81,7 +81,7 @@ public class SessionService extends BaseService{
* @param ip ip
* @return session string
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public String createSession(User user, String ip) {
Session session = null;

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java

@ -300,7 +300,7 @@ public class UdfFuncService extends BaseService{
* @param id udf function id
* @return delete result code
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public Result delete(int id) {
Result result = new Result();

17
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java

@ -138,14 +138,14 @@ public class UsersService extends BaseService {
}
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public User createUser(String userName,
String userPassword,
String email,
int tenantId,
String phone,
String queue,
int state) throws Exception {
int state) {
User user = new User();
Date now = new Date();
@ -430,7 +430,7 @@ public class UsersService extends BaseService {
* @param projectIds project id array
* @return grant result code
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public Map<String, Object> grantProject(User loginUser, int userId, String projectIds) {
Map<String, Object> result = new HashMap<>(5);
result.put(Constants.STATUS, false);
@ -480,7 +480,7 @@ public class UsersService extends BaseService {
* @param resourceIds resource id array
* @return grant result code
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public Map<String, Object> grantResources(User loginUser, int userId, String resourceIds) {
Map<String, Object> result = new HashMap<>(5);
//only admin can operate
@ -577,7 +577,7 @@ public class UsersService extends BaseService {
* @param udfIds udf id array
* @return grant result code
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public Map<String, Object> grantUDFFunction(User loginUser, int userId, String udfIds) {
Map<String, Object> result = new HashMap<>(5);
@ -624,7 +624,7 @@ public class UsersService extends BaseService {
* @param datasourceIds data source id array
* @return grant result code
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public Map<String, Object> grantDataSource(User loginUser, int userId, String datasourceIds) {
Map<String, Object> result = new HashMap<>(5);
result.put(Constants.STATUS, false);
@ -917,10 +917,9 @@ public class UsersService extends BaseService {
* @param repeatPassword repeat password
* @param email email
* @return register result code
* @throws Exception exception
*/
@Transactional(rollbackFor = Exception.class)
public Map<String, Object> registerUser(String userName, String userPassword, String repeatPassword, String email) throws Exception {
@Transactional(rollbackFor = RuntimeException.class)
public Map<String, Object> registerUser(String userName, String userPassword, String repeatPassword, String email) {
Map<String, Object> result = new HashMap<>(5);
//check user params

97
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkFlowLineageService.java

@ -0,0 +1,97 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper;
import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage;
import org.apache.dolphinscheduler.dao.entity.WorkFlowRelation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.*;
@Service
public class WorkFlowLineageService extends BaseService {
@Autowired
private WorkFlowLineageMapper workFlowLineageMapper;
public Map<String, Object> queryWorkFlowLineageByName(String workFlowName, int projectId) {
Map<String, Object> result = new HashMap<>(5);
List<WorkFlowLineage> workFlowLineageList = workFlowLineageMapper.queryByName(workFlowName, projectId);
result.put(Constants.DATA_LIST, workFlowLineageList);
putMsg(result, Status.SUCCESS);
return result;
}
private List<WorkFlowRelation> getWorkFlowRelationRecursion(Set<Integer> ids, List<WorkFlowRelation> workFlowRelations,Set<Integer> sourceIds) {
for(int id : ids) {
sourceIds.addAll(ids);
List<WorkFlowRelation> workFlowRelationsTmp = workFlowLineageMapper.querySourceTarget(id);
if(workFlowRelationsTmp != null && !workFlowRelationsTmp.isEmpty()) {
Set<Integer> idsTmp = new HashSet<>();
for(WorkFlowRelation workFlowRelation:workFlowRelationsTmp) {
if(!sourceIds.contains(workFlowRelation.getTargetWorkFlowId())){
idsTmp.add(workFlowRelation.getTargetWorkFlowId());
}
}
workFlowRelations.addAll(workFlowRelationsTmp);
getWorkFlowRelationRecursion(idsTmp, workFlowRelations,sourceIds);
}
}
return workFlowRelations;
}
public Map<String, Object> queryWorkFlowLineageByIds(Set<Integer> ids,int projectId) {
Map<String, Object> result = new HashMap<>(5);
List<WorkFlowLineage> workFlowLineageList = workFlowLineageMapper.queryByIds(ids, projectId);
Map<String, Object> workFlowLists = new HashMap<>(5);
Set<Integer> idsV = new HashSet<>();
if(ids == null || ids.isEmpty()){
for(WorkFlowLineage workFlowLineage:workFlowLineageList) {
idsV.add(workFlowLineage.getWorkFlowId());
}
} else {
idsV = ids;
}
List<WorkFlowRelation> workFlowRelations = new ArrayList<>();
Set<Integer> sourceIds = new HashSet<>();
getWorkFlowRelationRecursion(idsV, workFlowRelations, sourceIds);
Set<Integer> idSet = new HashSet<>();
//If the incoming parameter is not empty, you need to add downstream workflow detail attributes
if(ids != null && !ids.isEmpty()) {
for(WorkFlowRelation workFlowRelation : workFlowRelations) {
idSet.add(workFlowRelation.getTargetWorkFlowId());
}
for(int id : ids){
idSet.remove(id);
}
if(!idSet.isEmpty()) {
workFlowLineageList.addAll(workFlowLineageMapper.queryByIds(idSet, projectId));
}
}
workFlowLists.put("workFlowList",workFlowLineageList);
workFlowLists.put("workFlowRelationList",workFlowRelations);
result.put(Constants.DATA_LIST, workFlowLists);
putMsg(result, Status.SUCCESS);
return result;
}
}

1
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java

@ -149,6 +149,7 @@ public class ZooKeeperState {
return sendThread.ret;
} catch (InterruptedException e) {
logger.error("send " + cmd + " to server " + host + ":" + port + " failed!", e);
Thread.currentThread().interrupt();
}
return "";
}

3
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceControllerTest.java

@ -74,7 +74,8 @@ public class ProcessInstanceControllerTest extends AbstractControllerTest {
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assert.assertEquals(Status.PROJECT_NOT_FOUNT,result.getCode().intValue());
assert result != null;
Assert.assertEquals(Status.PROJECT_NOT_FOUNT.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}

69
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageControllerTest.java

@ -0,0 +1,69 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.controller;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.MediaType;
import org.springframework.test.web.servlet.MvcResult;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
public class WorkFlowLineageControllerTest extends AbstractControllerTest {
private static Logger logger = LoggerFactory.getLogger(WorkFlowLineageControllerTest.class);
@Test
public void testQueryWorkFlowLineageByName() throws Exception {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("searchVal","test");
MvcResult mvcResult = mockMvc.perform(get("/lineages/1/list-name")
.header("sessionId", sessionId)
.params(paramsMap))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
@Test
public void testQueryWorkFlowLineageByIds() throws Exception {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("ids","1");
MvcResult mvcResult = mockMvc.perform(get("/lineages/1/list-ids")
.header("sessionId", sessionId)
.params(paramsMap))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
}

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java

@ -180,7 +180,7 @@ public class DataAnalysisServiceTest {
List<ExecuteStatusCount> taskInstanceStateCounts = new ArrayList<>(1);
ExecuteStatusCount executeStatusCount = new ExecuteStatusCount();
executeStatusCount.setExecutionStatus(ExecutionStatus.RUNNING_EXEUTION);
executeStatusCount.setExecutionStatus(ExecutionStatus.RUNNING_EXECUTION);
taskInstanceStateCounts.add(executeStatusCount);
return taskInstanceStateCounts;

24
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java

@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbConnectType;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
@ -103,7 +104,28 @@ public class DataSourceServiceTest {
public void buildParameter(){
String param = dataSourceService.buildParameter("","", DbType.ORACLE, "192.168.9.1","1521","im"
,"","test","test", DbConnectType.ORACLE_SERVICE_NAME,"");
String expected = "{\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}";
String expected = "{\"connectType\":\"ORACLE_SERVICE_NAME\",\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}";
Assert.assertEquals(expected, param);
}
@Test
public void buildParameterWithDecodePassword(){
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true");
String param = dataSourceService.buildParameter("name","desc", DbType.MYSQL, "192.168.9.1","1521","im"
,"","test","123456", null,"");
String expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\"}";
Assert.assertEquals(expected, param);
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false");
param = dataSourceService.buildParameter("name","desc", DbType.MYSQL, "192.168.9.1","1521","im"
,"","test","123456", null,"");
expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"123456\"}";
Assert.assertEquals(expected, param);
}
}

4
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java

@ -474,7 +474,7 @@ public class ProcessDefinitionServiceTest {
ProcessInstance processInstance = new ProcessInstance();
processInstance.setId(1);
processInstance.setName("test_instance");
processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
processInstance.setHost("192.168.xx.xx");
processInstance.setStartTime(new Date());
processInstance.setEndTime(new Date());
@ -486,7 +486,7 @@ public class ProcessDefinitionServiceTest {
taskInstance.setTaskType("SHELL");
taskInstance.setId(1);
taskInstance.setName("test_task_instance");
taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
taskInstance.setHost("192.168.xx.xx");
//task instance not exist

38
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java

@ -147,6 +147,40 @@ public class ProcessInstanceServiceTest {
}
@Test
public void testQueryTopNLongestRunningProcessInstance() {
String projectName = "project_test1";
User loginUser = getAdminUser();
Map<String, Object> result = new HashMap<>(5);
putMsg(result, Status.PROJECT_NOT_FOUNT, projectName);
int size=10;
String startTime="2020-01-01 00:00:00";
String endTime="2020-08-02 00:00:00";
Date start = DateUtils.getScheduleDate(startTime);
Date end = DateUtils.getScheduleDate(endTime);
//project auth fail
when(projectMapper.queryByName(projectName)).thenReturn(null);
when(projectService.checkProjectAndAuth(loginUser, null, projectName)).thenReturn(result);
Map<String, Object> proejctAuthFailRes = processInstanceService.queryTopNLongestRunningProcessInstance(loginUser,projectName,size,startTime,endTime);
Assert.assertEquals(Status.PROJECT_NOT_FOUNT, proejctAuthFailRes.get(Constants.STATUS));
//project auth success
putMsg(result, Status.SUCCESS, projectName);
Project project = getProject(projectName);
ProcessInstance processInstance = getProcessInstance();
List<ProcessInstance> processInstanceList = new ArrayList<>();
processInstanceList.add(processInstance);
when(projectMapper.queryByName(projectName)).thenReturn(project);
when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result);
when(usersService.queryUser(loginUser.getId())).thenReturn(loginUser);
when(usersService.getUserIdByName(loginUser.getUserName())).thenReturn(loginUser.getId());
when(usersService.queryUser(processInstance.getExecutorId())).thenReturn(loginUser);
Map<String, Object> successRes = processInstanceService.queryTopNLongestRunningProcessInstance(loginUser,projectName,size,startTime,endTime);
Assert.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS));
}
@Test
public void testQueryProcessInstanceById() {
String projectName = "project_test1";
@ -307,7 +341,7 @@ public class ProcessInstanceServiceTest {
//process instance not finish
when(processService.findProcessInstanceDetailById(1)).thenReturn(processInstance);
processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
Map<String, Object> processInstanceNotFinishRes = processInstanceService.updateProcessInstance(loginUser, projectName, 1,
shellJson, "2020-02-21 00:00:00", true, Flag.YES, "", "");
Assert.assertEquals(Status.PROCESS_INSTANCE_STATE_OPERATION_ERROR, processInstanceNotFinishRes.get(Constants.STATUS));
@ -414,7 +448,7 @@ public class ProcessInstanceServiceTest {
ProcessInstance processInstance = getProcessInstance();
processInstance.setProcessInstanceJson(shellJson);
TaskInstance taskInstance = getTaskInstance();
taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
taskInstance.setStartTime(new Date());
when(processInstanceMapper.queryDetailById(1)).thenReturn(processInstance);
when(taskInstanceMapper.queryByInstanceIdAndName(Mockito.anyInt(), Mockito.any())).thenReturn(taskInstance);

88
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkFlowLineageServiceTest.java

@ -0,0 +1,88 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.EncryptionUtils;
import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage;
import org.apache.dolphinscheduler.dao.entity.WorkFlowRelation;
import org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnitRunner;
import java.util.*;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
public class WorkFlowLineageServiceTest {
@InjectMocks
private WorkFlowLineageService workFlowLineageService;
@Mock
private WorkFlowLineageMapper workFlowLineageMapper;
@Test
public void testQueryWorkFlowLineageByName() {
String searchVal = "test";
when(workFlowLineageMapper.queryByName(searchVal, 1)).thenReturn(getWorkFlowLineages());
Map<String, Object> result = workFlowLineageService.queryWorkFlowLineageByName(searchVal,1);
List<WorkFlowLineage> workFlowLineageList = (List<WorkFlowLineage>)result.get(Constants.DATA_LIST);
Assert.assertTrue(workFlowLineageList.size()>0);
}
@Test
public void testQueryWorkFlowLineageByIds() {
Set<Integer> ids = new HashSet<>();
ids.add(1);
ids.add(2);
when(workFlowLineageMapper.queryByIds(ids, 1)).thenReturn(getWorkFlowLineages());
when(workFlowLineageMapper.querySourceTarget(1)).thenReturn(getWorkFlowRelation());
Map<String, Object> result = workFlowLineageService.queryWorkFlowLineageByIds(ids,1);
Map<String, Object> workFlowLists = (Map<String, Object>)result.get(Constants.DATA_LIST);
List<WorkFlowLineage> workFlowLineages = (List<WorkFlowLineage>)workFlowLists.get("workFlowList");
List<WorkFlowRelation> workFlowRelations = (List<WorkFlowRelation>)workFlowLists.get("workFlowRelationList");
Assert.assertTrue(workFlowLineages.size()>0);
Assert.assertTrue(workFlowRelations.size()>0);
}
private List<WorkFlowLineage> getWorkFlowLineages() {
List<WorkFlowLineage> workFlowLineages = new ArrayList<>();
WorkFlowLineage workFlowLineage = new WorkFlowLineage();
workFlowLineage.setWorkFlowId(1);
workFlowLineage.setWorkFlowName("testdag");
workFlowLineages.add(workFlowLineage);
return workFlowLineages;
}
private List<WorkFlowRelation> getWorkFlowRelation(){
List<WorkFlowRelation> workFlowRelations = new ArrayList<>();
WorkFlowRelation workFlowRelation = new WorkFlowRelation();
workFlowRelation.setSourceWorkFlowId(1);
workFlowRelation.setTargetWorkFlowId(2);
workFlowRelations.add(workFlowRelation);
return workFlowRelations;
}
}

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@ -831,7 +831,7 @@ public final class Constants {
public static final int[] NOT_TERMINATED_STATES = new int[]{
ExecutionStatus.SUBMITTED_SUCCESS.ordinal(),
ExecutionStatus.RUNNING_EXEUTION.ordinal(),
ExecutionStatus.RUNNING_EXECUTION.ordinal(),
ExecutionStatus.READY_PAUSE.ordinal(),
ExecutionStatus.READY_STOP.ordinal(),
ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal(),
@ -978,6 +978,8 @@ public final class Constants {
public static final int NORAML_NODE_STATUS = 0;
public static final int ABNORMAL_NODE_STATUS = 1;
public static final String START_TIME = "start time";
public static final String END_TIME = "end time";
/**
* system line separator
*/
@ -989,4 +991,10 @@ public final class Constants {
public static final String DOLPHIN_SCHEDULER_PREFERRED_NETWORK_INTERFACE = "dolphin.scheduler.network.interface.preferred";
/**
* datasource encryption salt
*/
public static final String DATASOURCE_ENCRYPTION_SALT_DEFAULT = "!@#$%^&*";
public static final String DATASOURCE_ENCRYPTION_ENABLE = "datasource.encryption.enable";
public static final String DATASOURCE_ENCRYPTION_SALT = "datasource.encryption.salt";
}

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java

@ -43,7 +43,7 @@ public enum ExecutionStatus {
* 11 waiting depend node complete
*/
SUBMITTED_SUCCESS(0, "submit success"),
RUNNING_EXEUTION(1, "running"),
RUNNING_EXECUTION(1, "running"),
READY_PAUSE(2, "ready pause"),
PAUSE(3, "pause"),
READY_STOP(4, "ready stop"),
@ -126,7 +126,7 @@ public enum ExecutionStatus {
* @return status
*/
public boolean typeIsRunning(){
return this == RUNNING_EXEUTION || this == WAITTING_DEPEND;
return this == RUNNING_EXECUTION || this == WAITTING_DEPEND;
}
/**

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskStateType.java

@ -50,7 +50,7 @@ public enum TaskStateType {
};
case RUNNING:
return new int[]{ExecutionStatus.SUBMITTED_SUCCESS.ordinal(),
ExecutionStatus.RUNNING_EXEUTION.ordinal(),
ExecutionStatus.RUNNING_EXECUTION.ordinal(),
ExecutionStatus.READY_PAUSE.ordinal(),
ExecutionStatus.READY_STOP.ordinal()};
case WAITTING:

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskType.java

@ -36,6 +36,7 @@ public enum TaskType {
* 10 DATAX
* 11 CONDITIONS
* 12 SQOOP
* 13 WATERDROP
*/
SHELL(0, "shell"),
SQL(1, "sql"),
@ -49,7 +50,8 @@ public enum TaskType {
HTTP(9, "http"),
DATAX(10, "datax"),
CONDITIONS(11, "conditions"),
SQOOP(12, "sqoop");
SQOOP(12, "sqoop"),
WATERDROP(13, "waterdrop");
TaskType(int code, String descp){
this.code = code;

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java

@ -78,7 +78,7 @@ public abstract class AbstractShell {
/**
* If or not script finished executing
*/
private volatile AtomicBoolean completed;
private AtomicBoolean completed;
public AbstractShell() {
this(0L);
@ -202,7 +202,7 @@ public abstract class AbstractShell {
} catch (InterruptedException ie) {
logger.warn("Interrupted while reading the error and in stream", ie);
}
completed.set(true);
completed.compareAndSet(false,true);
//the timeout thread handling
//taken care in finally block
if (exitCode != 0 || errMsg.length() > 0) {

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java

@ -231,6 +231,8 @@ public class ThreadUtils {
public static void sleep(final long millis) {
try {
Thread.sleep(millis);
} catch (final InterruptedException ignore) {}
} catch (final InterruptedException ignore) {
Thread.currentThread().interrupt();
}
}
}

46
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java

@ -16,6 +16,7 @@
*/
package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.codec.binary.Base64;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.hadoop.conf.Configuration;
@ -23,8 +24,8 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.URL;
import java.nio.charset.StandardCharsets;
/**
* common utils
@ -32,6 +33,8 @@ import java.net.URL;
public class CommonUtils {
private static final Logger logger = LoggerFactory.getLogger(CommonUtils.class);
private static final Base64 BASE64 = new Base64();
private CommonUtils() {
throw new IllegalStateException("CommonUtils class");
}
@ -90,4 +93,45 @@ public class CommonUtils {
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH));
}
}
/**
* encode password
* @param password
* @return
*/
public static String encodePassword(String password) {
if(StringUtils.isEmpty(password)){return StringUtils.EMPTY; }
//if encryption is not turned on, return directly
boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE,false);
if ( !encryptionEnable){ return password; }
// Using Base64 + salt to process password
String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
String passwordWithSalt = salt + new String(BASE64.encode(password.getBytes(StandardCharsets.UTF_8))) ;
return new String(BASE64.encode(passwordWithSalt.getBytes(StandardCharsets.UTF_8)));
}
/**
* decode password
* @param password
* @return
*/
public static String decodePassword(String password) {
if(StringUtils.isEmpty(password)){return StringUtils.EMPTY ; }
//if encryption is not turned on, return directly
boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE,false);
if ( !encryptionEnable){ return password; }
// Using Base64 + salt to process password
String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
String passwordWithSalt = new String(BASE64.decode(password), StandardCharsets.UTF_8) ;
if(!passwordWithSalt.startsWith(salt)){
logger.warn("There is a password and salt mismatch: {} ",password);
return password;
}
return new String(BASE64.decode(passwordWithSalt.substring(salt.length())), StandardCharsets.UTF_8) ;
}
}

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java

@ -448,7 +448,7 @@ public class HadoopUtils implements Closeable {
case Constants.RUNNING:
default:
return ExecutionStatus.RUNNING_EXEUTION;
return ExecutionStatus.RUNNING_EXECUTION;
}
}

103
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java

@ -18,41 +18,116 @@ package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.http.HttpEntity;
import org.apache.http.client.config.AuthSchemes;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import java.io.IOException;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.X509Certificate;
import java.util.Arrays;
/**
* http utils
*/
public class HttpUtils {
public static final Logger logger = LoggerFactory.getLogger(HttpUtils.class);
private HttpUtils() {
}
public static CloseableHttpClient getInstance(){
return HttpClientInstance.httpClient;
}
private static class HttpClientInstance{
private static final CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(requestConfig).build();
}
private static PoolingHttpClientConnectionManager cm;
private static SSLContext ctx = null;
private static SSLConnectionSocketFactory socketFactory;
private static RequestConfig requestConfig;
private static Registry<ConnectionSocketFactory> socketFactoryRegistry;
private static X509TrustManager xtm = new X509TrustManager() {
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType) {
}
@Override
public void checkServerTrusted(X509Certificate[] chain, String authType) {
}
@Override
public X509Certificate[] getAcceptedIssuers() {
return null;
}
};
static {
try {
ctx = SSLContext.getInstance(SSLConnectionSocketFactory.TLS);
ctx.init(null, new TrustManager[] { xtm }, null);
} catch (NoSuchAlgorithmException e) {
logger.error("SSLContext init with NoSuchAlgorithmException", e);
} catch (KeyManagementException e) {
logger.error("SSLContext init with KeyManagementException", e);
}
socketFactory = new SSLConnectionSocketFactory(ctx, NoopHostnameVerifier.INSTANCE);
/** set timeout、request time、socket timeout */
requestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES)
.setExpectContinueEnabled(Boolean.TRUE)
.setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST))
.setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC))
.setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT).setSocketTimeout(Constants.SOCKET_TIMEOUT)
.setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT).setRedirectsEnabled(true)
.build();
socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE).register("https", socketFactory).build();
cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
cm.setDefaultMaxPerRoute(60);
cm.setMaxTotal(100);
}
/**
* get http request content
* @param url url
* @return http get request response content
*/
public static String get(String url){
CloseableHttpClient httpclient = HttpClients.createDefault();
CloseableHttpClient httpclient = HttpUtils.getInstance();
HttpGet httpget = new HttpGet(url);
/** set timeout、request time、socket timeout */
RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT)
.setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT)
.setSocketTimeout(Constants.SOCKET_TIMEOUT)
.setRedirectsEnabled(true)
.build();
httpget.setConfig(requestConfig);
String responseContent = null;
CloseableHttpResponse response = null;
@ -85,12 +160,6 @@ public class HttpUtils {
httpget.releaseConnection();
httpget.abort();
}
try {
httpclient.close();
} catch (IOException e) {
logger.error(e.getMessage(),e);
}
}
return responseContent;
}

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java

@ -207,7 +207,7 @@ public class ParameterUtils {
public static String handleEscapes(String inputString){
if(StringUtils.isNotEmpty(inputString)){
return inputString.replace("%", "////%");
return inputString.replace("%", "////%").replaceAll("[\n|\r\t]", "_");
}
return inputString;
}

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java

@ -250,4 +250,14 @@ public class PropertyUtils {
}
return matchedProperties;
}
/**
*
* @param key
* @param value
*/
public static void setValue(String key, String value) {
properties.setProperty(key,value);
}
}

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java

@ -53,6 +53,8 @@ public class TaskParametersUtils {
switch (EnumUtils.getEnum(TaskType.class,taskType)) {
case SUB_PROCESS:
return JSONUtils.parseObject(parameter, SubProcessParameters.class);
case WATERDROP:
return JSONUtils.parseObject(parameter, ShellParameters.class);
case SHELL:
return JSONUtils.parseObject(parameter, ShellParameters.class);
case PROCEDURE:

6
dolphinscheduler-common/src/main/resources/common.properties

@ -67,4 +67,8 @@ yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s
development.state=false
# kerberos tgt expire time, unit is hours
kerberos.expire.time=2
kerberos.expire.time=2
# datasource encryption salt
datasource.encryption.enable=false
datasource.encryption.salt=!@#$%^&*

39
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java

@ -16,6 +16,7 @@
*/
package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
@ -89,4 +90,42 @@ public class CommonUtilsTest {
}
Assert.assertTrue(true);
}
@Test
public void encodePassword() {
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true");
Assert.assertEquals("",CommonUtils.encodePassword(""));
Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==",CommonUtils.encodePassword("123456"));
Assert.assertEquals("IUAjJCVeJipJVkZCV2xoVFYwQT0=",CommonUtils.encodePassword("!QAZXSW@"));
Assert.assertEquals("IUAjJCVeJipOV1JtWjJWeUtFQT0=",CommonUtils.encodePassword("5dfger(@"));
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false");
Assert.assertEquals("",CommonUtils.encodePassword(""));
Assert.assertEquals("123456",CommonUtils.encodePassword("123456"));
Assert.assertEquals("!QAZXSW@",CommonUtils.encodePassword("!QAZXSW@"));
Assert.assertEquals("5dfger(@",CommonUtils.encodePassword("5dfger(@"));
}
@Test
public void decodePassword() {
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "true");
Assert.assertEquals("", CommonUtils.decodePassword(""));
Assert.assertEquals("123456", CommonUtils.decodePassword("IUAjJCVeJipNVEl6TkRVMg=="));
Assert.assertEquals("!QAZXSW@", CommonUtils.decodePassword("IUAjJCVeJipJVkZCV2xoVFYwQT0="));
Assert.assertEquals("5dfger(@", CommonUtils.decodePassword("IUAjJCVeJipOV1JtWjJWeUtFQT0="));
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "false");
Assert.assertEquals("", CommonUtils.decodePassword(""));
Assert.assertEquals("123456", CommonUtils.decodePassword("123456"));
Assert.assertEquals("!QAZXSW@", CommonUtils.decodePassword("!QAZXSW@"));
Assert.assertEquals("5dfger(@", CommonUtils.decodePassword("5dfger(@"));
}
}

14
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java

@ -16,6 +16,7 @@
*/
package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -67,4 +68,17 @@ public class FileUtilsTest {
Assert.assertTrue(false);
}
}
@Test
public void testSetValue() {
try {
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true");
Assert.assertTrue(PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE));
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false");
Assert.assertFalse(PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE));
} catch (Exception e) {
Assert.assertTrue(false);
}
}
}

8
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java

@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.common.utils;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.http.impl.client.CloseableHttpClient;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
@ -42,4 +43,11 @@ public class HttpUtilsTest {
result = HttpUtils.get("https://123.333.111.33/ccc");
Assert.assertNull(result);
}
@Test
public void testGetHttpClient() {
CloseableHttpClient httpClient1 = HttpUtils.getInstance();
CloseableHttpClient httpClient2 = HttpUtils.getInstance();
Assert.assertEquals(httpClient1, httpClient2);
}
}

2
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java

@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.junit.Assert;
import org.junit.Test;
import static org.junit.Assert.assertNotNull;
@ -26,6 +27,5 @@ public class PropertyUtilsTest {
@Test
public void getString() {
assertNotNull(PropertyUtils.getString(Constants.FS_DEFAULTFS));
assertNotNull(PropertyUtils.getInt("spring.redis.port"));
}
}

7
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java

@ -20,6 +20,7 @@ import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -182,8 +183,12 @@ public abstract class BaseDataSource {
this.user = user;
}
/**
* password need decode
* @return
*/
public String getPassword() {
return password;
return CommonUtils.decodePassword(password);
}
public void setPassword(String password) {

2
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java

@ -82,6 +82,8 @@ public class MySQLDataSource extends BaseDataSource {
@Override
public String getPassword() {
// password need decode
password = super.getPassword();
if(password.contains(sensitiveParam)){
logger.warn("sensitive param : {} in password field is filtered", sensitiveParam);
password = password.replace(sensitiveParam, "");

14
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java

@ -26,6 +26,8 @@ import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.dao.utils.PropertyUtils;
import org.apache.ibatis.mapping.DatabaseIdProvider;
import org.apache.ibatis.mapping.VendorDatabaseIdProvider;
import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.apache.ibatis.type.JdbcType;
@ -39,6 +41,8 @@ import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.core.io.support.ResourcePatternResolver;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import java.util.Properties;
/**
* data source connection factory
@ -129,6 +133,7 @@ public class SpringConnectionFactory {
ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
sqlSessionFactoryBean.setMapperLocations(resolver.getResources("org/apache/dolphinscheduler/dao/mapper/*Mapper.xml"));
sqlSessionFactoryBean.setTypeEnumsPackage("org.apache.dolphinscheduler.*.enums");
sqlSessionFactoryBean.setDatabaseIdProvider(databaseIdProvider());
return sqlSessionFactoryBean.getObject();
}
@ -142,4 +147,13 @@ public class SpringConnectionFactory {
return new SqlSessionTemplate(sqlSessionFactory());
}
@Bean
public DatabaseIdProvider databaseIdProvider(){
DatabaseIdProvider databaseIdProvider = new VendorDatabaseIdProvider();
Properties properties = new Properties();
properties.setProperty("MySQL", "mysql");
properties.setProperty("PostgreSQL", "pg");
databaseIdProvider.setProperties(properties);
return databaseIdProvider;
}
}

94
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowLineage.java

@ -0,0 +1,94 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.entity;
import java.util.Date;
public class WorkFlowLineage {
private int workFlowId;
private String workFlowName;
private String workFlowPublishStatus;
private Date scheduleStartTime;
private Date scheduleEndTime;
private String crontab;
private int schedulePublishStatus;
private String sourceWorkFlowId;
public String getSourceWorkFlowId() {
return sourceWorkFlowId;
}
public void setSourceWorkFlowId(String sourceWorkFlowId) {
this.sourceWorkFlowId = sourceWorkFlowId;
}
public int getWorkFlowId() {
return workFlowId;
}
public void setWorkFlowId(int workFlowId) {
this.workFlowId = workFlowId;
}
public String getWorkFlowName() {
return workFlowName;
}
public void setWorkFlowName(String workFlowName) {
this.workFlowName = workFlowName;
}
public String getWorkFlowPublishStatus() {
return workFlowPublishStatus;
}
public void setWorkFlowPublishStatus(String workFlowPublishStatus) {
this.workFlowPublishStatus = workFlowPublishStatus;
}
public Date getScheduleStartTime() {
return scheduleStartTime;
}
public void setScheduleStartTime(Date scheduleStartTime) {
this.scheduleStartTime = scheduleStartTime;
}
public Date getScheduleEndTime() {
return scheduleEndTime;
}
public void setScheduleEndTime(Date scheduleEndTime) {
this.scheduleEndTime = scheduleEndTime;
}
public String getCrontab() {
return crontab;
}
public void setCrontab(String crontab) {
this.crontab = crontab;
}
public int getSchedulePublishStatus() {
return schedulePublishStatus;
}
public void setSchedulePublishStatus(int schedulePublishStatus) {
this.schedulePublishStatus = schedulePublishStatus;
}
}

38
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowRelation.java

@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.entity;
public class WorkFlowRelation {
private int sourceWorkFlowId;
private int targetWorkFlowId;
public int getSourceWorkFlowId() {
return sourceWorkFlowId;
}
public void setSourceWorkFlowId(int sourceWorkFlowId) {
this.sourceWorkFlowId = sourceWorkFlowId;
}
public int getTargetWorkFlowId() {
return targetWorkFlowId;
}
public void setTargetWorkFlowId(int targetWorkFlowId) {
this.targetWorkFlowId = targetWorkFlowId;
}
}

13
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java

@ -193,4 +193,17 @@ public interface ProcessInstanceMapper extends BaseMapper<ProcessInstance> {
ProcessInstance queryLastManualProcess(@Param("processDefinitionId") int definitionId,
@Param("startTime") Date startTime,
@Param("endTime") Date endTime);
/**
* query top n process instance order by running duration
* @param size
* @param status process instance status
* @param startTime
* @param endTime
* @return ProcessInstance list
*/
List<ProcessInstance> queryTopNProcessInstance(@Param("size") int size,
@Param("startTime") Date startTime,
@Param("endTime") Date endTime,
@Param("status")ExecutionStatus status);
}

32
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapper.java

@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage;
import org.apache.dolphinscheduler.dao.entity.WorkFlowRelation;
import org.apache.ibatis.annotations.Param;
import java.util.List;
import java.util.Set;
public interface WorkFlowLineageMapper {
public List<WorkFlowLineage> queryByName(@Param("searchVal") String searchVal, @Param("projectId") int projectId);
public List<WorkFlowLineage> queryByIds(@Param("ids") Set<Integer> ids, @Param("projectId") int projectId);
public List<WorkFlowRelation> querySourceTarget(@Param("id") int id);
}

10
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml

@ -37,6 +37,16 @@
order by id asc
</select>
<select id="queryTopNProcessInstance" resultType="org.apache.dolphinscheduler.dao.entity.ProcessInstance">
select *
from t_ds_process_instance
where state = #{status}
and start_time between
#{startTime} and #{endTime}
order by end_time-start_time desc
limit #{size}
</select>
<select id="queryByTenantIdAndStatus" resultType="org.apache.dolphinscheduler.dao.entity.ProcessInstance">
select *
from t_ds_process_instance

103
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapper.xml

@ -0,0 +1,103 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper">
<select id="queryByName" resultType="org.apache.dolphinscheduler.dao.entity.WorkFlowLineage">
select tepd.id as work_flow_id,tepd.name as work_flow_name
from t_ds_process_definition tepd
left join t_ds_schedules tes on tepd.id = tes.process_definition_id
where tepd.project_id = #{projectId}
<if test="searchVal != null and searchVal != ''">
and tepd.name like concat('%', #{searchVal}, '%')
</if>
</select>
<select id="queryByIds" resultType="org.apache.dolphinscheduler.dao.entity.WorkFlowLineage" databaseId="mysql">
select tepd.id as work_flow_id,tepd.name as work_flow_name,
(case when json_extract(tepd.process_definition_json, '$**.dependItemList') is not null then 1 else 0 end) as is_depend_work_flow,
json_extract(tepd.process_definition_json, '$**.definitionId') as source_work_flow_id,
tepd.release_state as work_flow_publish_status,
tes.start_time as schedule_start_time,
tes.end_time as schedule_end_time,
tes.crontab as crontab,
tes.release_state as schedule_publish_status
from t_ds_process_definition tepd
left join t_ds_schedules tes on tepd.id = tes.process_definition_id
where tepd.project_id = #{projectId}
<if test="ids != null and ids.size()>0">
and tepd.id in
<foreach collection="ids" index="index" item="i" open="(" separator="," close=")">
#{i}
</foreach>
</if>
</select>
<select id="queryByIds" resultType="org.apache.dolphinscheduler.dao.entity.WorkFlowLineage" databaseId="pg">
select a.work_flow_id,
a.work_flow_name,
a.is_depend_work_flow,
array_agg(a.source_id) as source_id,
a.work_flow_publish_status,
a.schedule_start_time,
a.schedule_end_time,
a.crontab,
a.schedule_publish_status
from (
select tepd.id as work_flow_id,tepd.name as work_flow_name,
case when tepd.process_definition_json::json#>'{tasks,1,dependence}' is not null then 1 else 0 end as is_depend_work_flow,
(json_array_elements(tepd.process_definition_json::json#>'{tasks}')#>>'{dependence,dependTaskList,0,dependItemList,0,definitionId}') as source_id,
tepd.release_state as work_flow_publish_status,
tes.start_time as schedule_start_time,
tes.end_time as schedule_end_time,
tes.crontab as crontab,
tes.release_state as schedule_publish_status
from t_ds_process_definition tepd
left join t_ds_schedules tes on tepd.id = tes.process_definition_id
where tepd.project_id = #{projectId}
<if test="ids != null and ids.size()>0">
and tepd.id in
<foreach collection="ids" index="index" item="i" open="(" separator="," close=")">
#{i}
</foreach>
</if>
) a
where (a.is_depend_work_flow = 1 and source_id is not null) or (a.is_depend_work_flow = 0)
group by a.work_flow_id,a.work_flow_name,a.is_depend_work_flow,a.work_flow_publish_status,a.schedule_start_time,
a.schedule_end_time,a.crontab,a.schedule_publish_status
</select>
<select id="querySourceTarget" resultType="org.apache.dolphinscheduler.dao.entity.WorkFlowRelation" databaseId="mysql">
select id as target_work_flow_id,#{id} as source_work_flow_id
from t_ds_process_definition t
where json_extract(t.process_definition_json, '$**.dependItemList') is not null
and find_in_set(#{id}, replace(replace(replace(json_extract(t.process_definition_json, '$**.definitionId'), '[', ''),']', ''), ' ', '')) > 0
</select>
<select id="querySourceTarget" resultType="org.apache.dolphinscheduler.dao.entity.WorkFlowRelation" databaseId="pg">
select a.work_flow_id as target_work_flow_id,
a.source_id as source_work_flow_id
from (
select tepd.id as work_flow_id,
(json_array_elements(tepd.process_definition_json::json#>'{tasks}')#>>'{dependence,dependTaskList,0,dependItemList,0,definitionId}') as source_id
from t_ds_process_definition tepd
left join t_ds_schedules tes on tepd.id = tes.process_definition_id
where tepd.project_id = 1) a
where source_id = #{id}::text;
</select>
</mapper>

49
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java

@ -17,6 +17,8 @@
package org.apache.dolphinscheduler.dao.datasource;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.junit.Assert;
import org.junit.Test;
@ -112,4 +114,51 @@ public class BaseDataSourceTest {
}
@Test
public void testGetPassword() {
BaseDataSource dataSource = new BaseDataSource() {
@Override
public String driverClassSelector() {
return null;
}
@Override
public DbType dbTypeSelector() {
return null;
}
};
String password= "";
dataSource.setPassword(password);
Assert.assertEquals("", dataSource.getPassword());
password= "IUAjJCVeJipNVEl6TkRVMg==";
dataSource.setPassword(password);
Assert.assertNotNull(dataSource.getPassword());
Assert.assertNotNull(dataSource.getPassword());
dataSource.setPassword(password);
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true");
Assert.assertEquals("123456", dataSource.getPassword());
dataSource.setPassword(password);
Assert.assertEquals("123456", dataSource.getPassword());
Assert.assertEquals("123456", dataSource.getPassword());
Assert.assertEquals("123456", dataSource.getPassword());
dataSource.setPassword(password);
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false");
Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
dataSource.setPassword(password);
Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
}
}

35
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSourceTest.java

@ -16,6 +16,8 @@
*/
package org.apache.dolphinscheduler.dao.datasource;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.junit.Assert;
import org.junit.Test;
@ -46,6 +48,7 @@ public class MySQLDataSourceTest {
Assert.assertEquals("test_pwd?", dataSource.getPassword());
}
@Test
public void testFilterOther(){
MySQLDataSource dataSource = new MySQLDataSource();
@ -61,4 +64,36 @@ public class MySQLDataSourceTest {
other = dataSource.filterOther("serverTimezone=Asia/Shanghai&autoDeserialize=true&characterEncoding=utf8");
Assert.assertEquals("serverTimezone=Asia/Shanghai&characterEncoding=utf8", other);
}
@Test
public void testGetPasswordWithDecodePassword(){
MySQLDataSource dataSource = new MySQLDataSource();
String password= "";
dataSource.setPassword(password);
Assert.assertEquals("", dataSource.getPassword());
password= "IUAjJCVeJipNVEl6TkRVMg==";
dataSource.setPassword(password);
Assert.assertNotNull(dataSource.getPassword());
Assert.assertNotNull(dataSource.getPassword());
dataSource.setPassword(password);
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true");
Assert.assertEquals("123456", dataSource.getPassword());
dataSource.setPassword(password);
Assert.assertEquals("123456", dataSource.getPassword());
Assert.assertEquals("123456", dataSource.getPassword());
Assert.assertEquals("123456", dataSource.getPassword());
dataSource.setPassword(password);
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false");
Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
dataSource.setPassword(password);
Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
Assert.assertEquals("IUAjJCVeJipNVEl6TkRVMg==", dataSource.getPassword());
}
}

1
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java

@ -63,7 +63,6 @@ public class AccessTokenMapperTest {
Integer userId = 1;
AccessToken accessToken = createAccessToken(userId);
assertNotNull(accessToken.getId());
assertThat(accessToken.getId(), greaterThan(0));
}

1
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertMapperTest.java

@ -54,7 +54,6 @@ public class AlertMapperTest {
@Test
public void testInsert(){
Alert expectedAlert = createAlert();
assertNotNull(expectedAlert.getId());
assertThat(expectedAlert.getId(), greaterThan(0));
}

1
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java

@ -62,7 +62,6 @@ public class CommandMapperTest {
@Test
public void testInsert(){
Command command = createCommand();
assertNotNull(command.getId());
assertThat(command.getId(),greaterThan(0));
}

1
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java

@ -71,7 +71,6 @@ public class DataSourceMapperTest {
@Test
public void testInsert(){
DataSource dataSource = createDataSource();
assertNotNull(dataSource.getId());
assertThat(dataSource.getId(), greaterThan(0));
}

87
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java

@ -52,6 +52,25 @@ public class ProcessInstanceMapperTest {
ProjectMapper projectMapper;
/**
* insert process instance with specified start time and end time,set state to SUCCESS
*
* @param startTime
* @param endTime
* @return
*/
private ProcessInstance insertOne(Date startTime, Date endTime) {
ProcessInstance processInstance = new ProcessInstance();
Date start = startTime;
Date end = endTime;
processInstance.setStartTime(start);
processInstance.setEndTime(end);
processInstance.setState(ExecutionStatus.SUCCESS);
processInstanceMapper.insert(processInstance);
return processInstance;
}
/**
* insert
* @return ProcessInstance
@ -113,7 +132,7 @@ public class ProcessInstanceMapperTest {
processInstanceMapper.updateById(processInstance);
ProcessInstance processInstance1 = processInstanceMapper.queryDetailById(processInstance.getId());
Assert.assertNotEquals(processInstance1, 50);
Assert.assertNotNull(processInstance1);
processInstanceMapper.deleteById(processInstance.getId());
}
@ -124,11 +143,11 @@ public class ProcessInstanceMapperTest {
public void testQueryByHostAndStates() {
ProcessInstance processInstance = insertOne();
processInstance.setHost("192.168.2.155");
processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
processInstanceMapper.updateById(processInstance);
int[] stateArray = new int[]{
ExecutionStatus.RUNNING_EXEUTION.ordinal(),
ExecutionStatus.RUNNING_EXECUTION.ordinal(),
ExecutionStatus.SUCCESS.ordinal()};
List<ProcessInstance> processInstances = processInstanceMapper.queryByHostAndStatus(null, stateArray);
@ -145,7 +164,7 @@ public class ProcessInstanceMapperTest {
int[] stateArray = new int[]{
ExecutionStatus.RUNNING_EXEUTION.ordinal(),
ExecutionStatus.RUNNING_EXECUTION.ordinal(),
ExecutionStatus.SUCCESS.ordinal()};
ProcessDefinition processDefinition = new ProcessDefinition();
@ -155,7 +174,7 @@ public class ProcessInstanceMapperTest {
ProcessInstance processInstance = insertOne();
processInstance.setProcessDefinitionId(processDefinition.getId());
processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
processInstance.setIsSubProcess(Flag.NO);
processInstance.setStartTime(new Date());
@ -188,12 +207,12 @@ public class ProcessInstanceMapperTest {
public void testSetFailoverByHostAndStateArray() {
int[] stateArray = new int[]{
ExecutionStatus.RUNNING_EXEUTION.ordinal(),
ExecutionStatus.RUNNING_EXECUTION.ordinal(),
ExecutionStatus.SUCCESS.ordinal()};
ProcessInstance processInstance = insertOne();
processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
processInstance.setHost("192.168.2.220");
processInstanceMapper.updateById(processInstance);
String host = processInstance.getHost();
@ -214,9 +233,9 @@ public class ProcessInstanceMapperTest {
ProcessInstance processInstance = insertOne();
processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
processInstanceMapper.updateById(processInstance);
processInstanceMapper.updateProcessInstanceByState(ExecutionStatus.RUNNING_EXEUTION, ExecutionStatus.SUCCESS);
processInstanceMapper.updateProcessInstanceByState(ExecutionStatus.RUNNING_EXECUTION, ExecutionStatus.SUCCESS);
ProcessInstance processInstance1 = processInstanceMapper.selectById(processInstance.getId());
@ -294,11 +313,11 @@ public class ProcessInstanceMapperTest {
@Test
public void testQueryLastRunningProcess() {
ProcessInstance processInstance = insertOne();
processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
processInstanceMapper.updateById(processInstance);
int[] stateArray = new int[]{
ExecutionStatus.RUNNING_EXEUTION.ordinal(),
ExecutionStatus.RUNNING_EXECUTION.ordinal(),
ExecutionStatus.SUBMITTED_SUCCESS.ordinal()};
ProcessInstance processInstance1 = processInstanceMapper.queryLastRunningProcess(processInstance.getProcessDefinitionId(), null, null , stateArray);
@ -329,4 +348,50 @@ public class ProcessInstanceMapperTest {
processInstanceMapper.deleteById(processInstance.getId());
}
/**
* test whether it is in descending order by running duration
*
* @param processInstances
* @return
*/
private boolean isSortedByDuration(List<ProcessInstance> processInstances) {
for (int i = 1; i < processInstances.size(); i++) {
long d1 = processInstances.get(i).getEndTime().getTime() - processInstances.get(i).getStartTime().getTime();
long d2 = processInstances.get(i - 1).getEndTime().getTime() - processInstances.get(i - 1).getStartTime().getTime();
if (d1 > d2) {
return false;
}
}
return true;
}
/**
* test query top n process instance order by running duration
*/
@Test
public void testQueryTopNProcessInstance() {
Date startTime1 = new Date(2019, 7, 9, 10, 9, 9);
Date endTime1 = new Date(2019, 7, 9, 10, 9, 14);
Date startTime2 = new Date(2020, 7, 9, 10, 9, 9);
Date endTime2 = new Date(2020, 7, 9, 10, 9, 30);
Date startTime3 = new Date(2020, 6, 9, 10, 9, 9);
Date endTime3 = new Date(2020, 7, 9, 10, 9, 30);
ProcessInstance processInstance1 = insertOne(startTime1, endTime1);
ProcessInstance processInstance2 = insertOne(startTime2, endTime2);
ProcessInstance processInstance3 = insertOne(startTime3, endTime3);
Date start = new Date(2020, 1, 1, 1, 1, 1);
Date end = new Date(2021, 1, 1, 1, 1, 1);
List<ProcessInstance> processInstances = processInstanceMapper.queryTopNProcessInstance(2, start, end,ExecutionStatus.SUCCESS);
Assert.assertEquals(2, processInstances.size());
Assert.assertTrue(isSortedByDuration(processInstances));
for (ProcessInstance processInstance : processInstances) {
Assert.assertTrue(processInstance.getState().typeIsSuccess());
}
processInstanceMapper.deleteById(processInstance1.getId());
processInstanceMapper.deleteById(processInstance2.getId());
processInstanceMapper.deleteById(processInstance3.getId());
}
}

10
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java

@ -64,7 +64,7 @@ public class TaskInstanceMapperTest {
TaskInstance taskInstance = new TaskInstance();
taskInstance.setFlag(Flag.YES);
taskInstance.setName("ut task");
taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
taskInstance.setStartTime(new Date());
taskInstance.setEndTime(new Date());
taskInstance.setTaskJson("{}");
@ -118,7 +118,7 @@ public class TaskInstanceMapperTest {
taskInstanceMapper.updateById(task);
List<Integer> taskInstances = taskInstanceMapper.queryTaskByProcessIdAndState(
task.getProcessInstanceId(),
ExecutionStatus.RUNNING_EXEUTION.ordinal()
ExecutionStatus.RUNNING_EXECUTION.ordinal()
);
taskInstanceMapper.deleteById(task.getId());
Assert.assertNotEquals(taskInstances.size(), 0);
@ -162,7 +162,7 @@ public class TaskInstanceMapperTest {
taskInstanceMapper.updateById(task);
List<TaskInstance> taskInstances = taskInstanceMapper.queryByHostAndStatus(
task.getHost(), new int[]{ExecutionStatus.RUNNING_EXEUTION.ordinal()}
task.getHost(), new int[]{ExecutionStatus.RUNNING_EXECUTION.ordinal()}
);
taskInstanceMapper.deleteById(task.getId());
Assert.assertNotEquals(taskInstances.size(), 0);
@ -179,7 +179,7 @@ public class TaskInstanceMapperTest {
int setResult = taskInstanceMapper.setFailoverByHostAndStateArray(
task.getHost(),
new int[]{ExecutionStatus.RUNNING_EXEUTION.ordinal()},
new int[]{ExecutionStatus.RUNNING_EXECUTION.ordinal()},
ExecutionStatus.NEED_FAULT_TOLERANCE
);
taskInstanceMapper.deleteById(task.getId());
@ -268,7 +268,7 @@ public class TaskInstanceMapperTest {
ProcessInstance processInstance = new ProcessInstance();
processInstance.setProcessDefinitionId(definition.getId());
processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
processInstance.setName("ut process");
processInstance.setStartTime(new Date());
processInstance.setEndTime(new Date());

62
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapperTest.java

@ -0,0 +1,62 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage;
import org.apache.dolphinscheduler.dao.entity.WorkFlowRelation;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.annotation.Rollback;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.transaction.annotation.Transactional;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@RunWith(SpringRunner.class)
@SpringBootTest
@Transactional
@Rollback(true)
public class WorkFlowLineageMapperTest {
@Autowired
private WorkFlowLineageMapper workFlowLineageMapper;
@Test
public void testQueryByName() {
List<WorkFlowLineage> workFlowLineages = workFlowLineageMapper.queryByName("test",1);
Assert.assertNotEquals(workFlowLineages.size(), 0);
}
@Test
public void testQueryByIds() {
Set<Integer> ids = new HashSet<>();
ids.add(1);
List<WorkFlowLineage> workFlowLineages = workFlowLineageMapper.queryByIds(ids,1);
Assert.assertNotEquals(workFlowLineages.size(), 0);
}
@Test
public void testQuerySourceTarget() {
List<WorkFlowRelation> workFlowRelations = workFlowLineageMapper.querySourceTarget(1);
Assert.assertNotEquals(workFlowRelations.size(), 0);
}
}

1
dolphinscheduler-dist/pom.xml vendored

@ -318,6 +318,7 @@
<include>start-all.sh</include>
<include>stop-all.sh</include>
<include>dolphinscheduler-daemon.sh</include>
<include>status-all.sh</include>
</includes>
</source>
</sources>

1
dolphinscheduler-dist/src/main/assembly/dolphinscheduler-binary.xml vendored

@ -189,6 +189,7 @@
<include>start-all.sh</include>
<include>stop-all.sh</include>
<include>dolphinscheduler-daemon.sh</include>
<include>status-all.sh</include>
</includes>
<outputDirectory>./bin</outputDirectory>
</fileSet>

1
dolphinscheduler-dist/src/main/assembly/dolphinscheduler-nginx.xml vendored

@ -197,6 +197,7 @@
<include>start-all.sh</include>
<include>stop-all.sh</include>
<include>dolphinscheduler-daemon.sh</include>
<include>status-all.sh</include>
</includes>
<outputDirectory>./bin</outputDirectory>
</fileSet>

4
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java

@ -123,8 +123,8 @@ public class MasterServer {
// self tolerant
this.zkMasterClient.start();
//
masterSchedulerService.start();
// scheduler start
this.masterSchedulerService.start();
// start QuartzExecutors
// what system should do if exception

1
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java

@ -89,6 +89,7 @@ public class TaskResponseService {
eventQueue.put(taskResponseEvent);
} catch (InterruptedException e) {
logger.error("put task : {} error :{}", taskResponseEvent,e);
Thread.currentThread().interrupt();
}
}

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java

@ -124,7 +124,7 @@ public class ConditionsTaskExecThread extends MasterBaseTaskExecThread {
private void initTaskParameters() {
this.taskInstance.setLogPath(getTaskLogPath(taskInstance));
this.taskInstance.setHost(NetUtils.getHost() + Constants.COLON + masterConfig.getListenPort());
taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
taskInstance.setStartTime(new Date());
this.processService.saveTaskInstance(taskInstance);

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java

@ -173,7 +173,7 @@ public class DependentTaskExecThread extends MasterBaseTaskExecThread {
private void initTaskParameters() {
taskInstance.setLogPath(getTaskLogPath(taskInstance));
taskInstance.setHost(NetUtils.getHost() + Constants.COLON + masterConfig.getListenPort());
taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
taskInstance.setStartTime(new Date());
processService.updateTaskInstance(taskInstance);
}

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java

@ -172,7 +172,7 @@ public class MasterBaseTaskExecThread implements Callable<Boolean> {
return true;
}
// task cannot submit when running
if(taskInstance.getState() == ExecutionStatus.RUNNING_EXEUTION){
if(taskInstance.getState() == ExecutionStatus.RUNNING_EXECUTION){
logger.info(String.format("submit to task, but task [%s] state already be running. ", taskInstance.getName()));
return true;
}

65
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java

@ -35,7 +35,6 @@ import org.apache.dolphinscheduler.dao.utils.DagHelper;
import org.apache.dolphinscheduler.remote.NettyRemotingClient;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.utils.AlertManager;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
import org.slf4j.Logger;
@ -123,12 +122,12 @@ public class MasterExecThread implements Runnable {
/**
* alert manager
*/
private AlertManager alertManager = new AlertManager();
private AlertManager alertManager;
/**
* the object of DAG
*/
private DAG<String,TaskNode,TaskNodeRelation> dag;
private DAG<String, TaskNode, TaskNodeRelation> dag;
/**
* process service
@ -151,15 +150,20 @@ public class MasterExecThread implements Runnable {
* @param processService processService
* @param nettyRemotingClient nettyRemotingClient
*/
public MasterExecThread(ProcessInstance processInstance, ProcessService processService, NettyRemotingClient nettyRemotingClient){
public MasterExecThread(ProcessInstance processInstance
, ProcessService processService
, NettyRemotingClient nettyRemotingClient
, AlertManager alertManager
, MasterConfig masterConfig) {
this.processService = processService;
this.processInstance = processInstance;
this.masterConfig = SpringApplicationContext.getBean(MasterConfig.class);
this.masterConfig = masterConfig;
int masterTaskExecNum = masterConfig.getMasterExecTaskNum();
this.taskExecService = ThreadUtils.newDaemonFixedThreadExecutor("Master-Task-Exec-Thread",
masterTaskExecNum);
this.nettyRemotingClient = nettyRemotingClient;
this.alertManager = alertManager;
}
@ -248,6 +252,9 @@ public class MasterExecThread implements Runnable {
}
while(Stopper.isRunning()){
logger.info("process {} start to complement {} data",
processInstance.getId(), DateUtils.dateToString(scheduleDate));
// prepare dag and other info
prepareProcess();
@ -262,13 +269,13 @@ public class MasterExecThread implements Runnable {
// execute process ,waiting for end
runProcess();
endProcess();
// process instance failure ,no more complements
if(!processInstance.getState().typeIsSuccess()){
logger.info("process {} state {}, complement not completely!",
processInstance.getId(), processInstance.getState());
break;
}
// current process instance success ,next execute
if(null == iterator){
// loop by day
@ -287,9 +294,7 @@ public class MasterExecThread implements Runnable {
}
scheduleDate = iterator.next();
}
logger.info("process {} start to complement {} data",
processInstance.getId(), DateUtils.dateToString(scheduleDate));
// flow end
// execute next process instance complement data
processInstance.setScheduleTime(scheduleDate);
if(cmdParam.containsKey(Constants.CMDPARAM_RECOVERY_START_NODE_STRING)){
@ -297,23 +302,16 @@ public class MasterExecThread implements Runnable {
processInstance.setCommandParam(JSONUtils.toJsonString(cmdParam));
}
List<TaskInstance> taskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId());
for(TaskInstance taskInstance : taskInstanceList){
taskInstance.setFlag(Flag.NO);
processService.updateTaskInstance(taskInstance);
}
processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
processInstance.setGlobalParams(ParameterUtils.curingGlobalParams(
processInstance.getProcessDefinition().getGlobalParamMap(),
processInstance.getProcessDefinition().getGlobalParamList(),
CommandType.COMPLEMENT_DATA, processInstance.getScheduleTime()));
processInstance.setId(0);
processInstance.setStartTime(new Date());
processInstance.setEndTime(null);
processService.saveProcessInstance(processInstance);
}
// flow end
endProcess();
}
@ -727,7 +725,7 @@ public class MasterExecThread implements Runnable {
// if the running task is not completed, the state remains unchanged
return state;
}else{
return ExecutionStatus.RUNNING_EXEUTION;
return ExecutionStatus.RUNNING_EXECUTION;
}
}
@ -804,7 +802,7 @@ public class MasterExecThread implements Runnable {
ProcessInstance instance = processService.findProcessInstanceById(processInstance.getId());
ExecutionStatus state = instance.getState();
if(activeTaskNode.size() > 0 || retryTaskExists()){
if(activeTaskNode.size() > 0 || hasRetryTaskInStandBy()){
// active task and retry task exists
return runningState(state);
}
@ -837,11 +835,11 @@ public class MasterExecThread implements Runnable {
}
// success
if(state == ExecutionStatus.RUNNING_EXEUTION){
if(state == ExecutionStatus.RUNNING_EXECUTION){
List<TaskInstance> killTasks = getCompleteTaskByState(ExecutionStatus.KILL);
if(readyToSubmitTaskList.size() > 0){
//tasks currently pending submission, no retries, indicating that depend is waiting to complete
return ExecutionStatus.RUNNING_EXEUTION;
return ExecutionStatus.RUNNING_EXECUTION;
}else if(CollectionUtils.isNotEmpty(killTasks)){
// tasks maybe killed manually
return ExecutionStatus.FAILURE;
@ -854,24 +852,6 @@ public class MasterExecThread implements Runnable {
return state;
}
/**
* whether standby task list have retry tasks
* @return
*/
private boolean retryTaskExists() {
boolean result = false;
for(String taskName : readyToSubmitTaskList.keySet()){
TaskInstance task = readyToSubmitTaskList.get(taskName);
if(task.getState().typeIsFailure()){
result = true;
break;
}
}
return result;
}
/**
* whether complement end
* @return Boolean whether is complement end
@ -1046,6 +1026,7 @@ public class MasterExecThread implements Runnable {
Thread.sleep(Constants.SLEEP_TIME_MILLIS);
} catch (InterruptedException e) {
logger.error(e.getMessage(),e);
Thread.currentThread().interrupt();
}
updateProcessInstanceState();
}

29
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java

@ -16,6 +16,11 @@
*/
package org.apache.dolphinscheduler.server.master.runner;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import javax.annotation.PostConstruct;
import org.apache.curator.framework.imps.CuratorFrameworkState;
import org.apache.curator.framework.recipes.locks.InterProcessMutex;
import org.apache.dolphinscheduler.common.Constants;
@ -28,6 +33,7 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.remote.NettyRemotingClient;
import org.apache.dolphinscheduler.remote.config.NettyClientConfig;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.utils.AlertManager;
import org.apache.dolphinscheduler.server.zk.ZKMasterClient;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.slf4j.Logger;
@ -35,10 +41,6 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
* master scheduler thread
*/
@ -68,6 +70,11 @@ public class MasterSchedulerService extends Thread {
@Autowired
private MasterConfig masterConfig;
/**
* alert manager
*/
private AlertManager alertManager = new AlertManager();
/**
* netty remoting client
*/
@ -90,7 +97,7 @@ public class MasterSchedulerService extends Thread {
}
@Override
public void start(){
public synchronized void start(){
super.setName("MasterSchedulerService");
super.start();
}
@ -100,7 +107,9 @@ public class MasterSchedulerService extends Thread {
boolean terminated = false;
try {
terminated = masterExecService.awaitTermination(5, TimeUnit.SECONDS);
} catch (InterruptedException ignore) {}
} catch (InterruptedException ignore) {
Thread.currentThread().interrupt();
}
if(!terminated){
logger.warn("masterExecService shutdown without terminated, increase await time");
}
@ -139,7 +148,13 @@ public class MasterSchedulerService extends Thread {
this.masterConfig.getMasterExecThreads() - activeCount, command);
if (processInstance != null) {
logger.info("start master exec thread , split DAG ...");
masterExecService.execute(new MasterExecThread(processInstance, processService, nettyRemotingClient));
masterExecService.execute(
new MasterExecThread(
processInstance
, processService
, nettyRemotingClient
, alertManager
, masterConfig));
}
}catch (Exception e){
logger.error("scan command error ", e);

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/SubProcessTaskExecThread.java

@ -93,7 +93,7 @@ public class SubProcessTaskExecThread extends MasterBaseTaskExecThread {
return false;
}
taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
taskInstance.setStartTime(new Date());
processService.updateTaskInstance(taskInstance);
return true;

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java

@ -167,7 +167,7 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
private TaskExecuteAckCommand buildAckCommand(TaskExecutionContext taskExecutionContext) {
TaskExecuteAckCommand ackCommand = new TaskExecuteAckCommand();
ackCommand.setTaskInstanceId(taskExecutionContext.getTaskInstanceId());
ackCommand.setStatus(ExecutionStatus.RUNNING_EXEUTION.getCode());
ackCommand.setStatus(ExecutionStatus.RUNNING_EXECUTION.getCode());
ackCommand.setLogPath(getTaskLogPath(taskExecutionContext));
ackCommand.setHost(taskExecutionContext.getHost());
ackCommand.setStartTime(new Date());

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskManager.java

@ -50,6 +50,8 @@ public class TaskManager {
switch (EnumUtils.getEnum(TaskType.class,taskExecutionContext.getTaskType())) {
case SHELL:
return new ShellTask(taskExecutionContext, logger);
case WATERDROP:
return new ShellTask(taskExecutionContext, logger);
case PROCEDURE:
return new ProcedureTask(taskExecutionContext, logger);
case SQL:

3
dolphinscheduler-server/src/main/resources/config/install_config.conf

@ -38,6 +38,9 @@ dbname="dolphinscheduler"
# zk cluster
zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181"
# zk root directory
zkRoot="/dolphinscheduler"
# Note: the target installation path for dolphinscheduler, please not config as the same as the current path (pwd)
installPath="/data1_1T/dolphinscheduler"

2
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java

@ -124,7 +124,7 @@ public class ConditionsTaskTest {
ProcessInstance processInstance = new ProcessInstance();
processInstance.setId(10112);
processInstance.setProcessDefinitionId(100001);
processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
return processInstance;
}

2
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java

@ -176,7 +176,7 @@ public class DependentTaskTest {
ProcessInstance processInstance = new ProcessInstance();
processInstance.setId(10111);
processInstance.setProcessDefinitionId(0);
processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
return processInstance;
}

21
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java

@ -72,8 +72,6 @@ public class MasterExecThreadTest {
applicationContext = mock(ApplicationContext.class);
config = new MasterConfig();
config.setMasterExecTaskNum(1);
SpringApplicationContext springApplicationContext = new SpringApplicationContext();
springApplicationContext.setApplicationContext(applicationContext);
Mockito.when(applicationContext.getBean(MasterConfig.class)).thenReturn(config);
processInstance = mock(ProcessInstance.class);
@ -84,14 +82,17 @@ public class MasterExecThreadTest {
Mockito.when(processInstance.getScheduleTime()).thenReturn(DateUtils.stringToDate("2020-01-01 00:00:00"));
Map<String, String> cmdParam = new HashMap<>();
cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, "2020-01-01 00:00:00");
cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, "2020-01-31 23:00:00");
cmdParam.put(CMDPARAM_COMPLEMENT_DATA_END_DATE, "2020-01-20 23:00:00");
Mockito.when(processInstance.getCommandParam()).thenReturn(JSONUtils.toJsonString(cmdParam));
ProcessDefinition processDefinition = new ProcessDefinition();
processDefinition.setGlobalParamMap(Collections.EMPTY_MAP);
processDefinition.setGlobalParamList(Collections.EMPTY_LIST);
Mockito.when(processInstance.getProcessDefinition()).thenReturn(processDefinition);
masterExecThread = PowerMockito.spy(new MasterExecThread(processInstance, processService,null));
masterExecThread = PowerMockito.spy(new MasterExecThread(
processInstance
, processService
,null, null, config));
// prepareProcess init dag
Field dag = MasterExecThread.class.getDeclaredField("dag");
dag.setAccessible(true);
@ -114,11 +115,11 @@ public class MasterExecThreadTest {
Method method = MasterExecThread.class.getDeclaredMethod("executeComplementProcess");
method.setAccessible(true);
method.invoke(masterExecThread);
// one create save, and 1-30 for next save, and last day 31 no save
verify(processService, times(31)).saveProcessInstance(processInstance);
// one create save, and 1-30 for next save, and last day 20 no save
verify(processService, times(20)).saveProcessInstance(processInstance);
}catch (Exception e){
e.printStackTrace();
Assert.assertTrue(false);
Assert.fail();
}
}
@ -133,10 +134,10 @@ public class MasterExecThreadTest {
Method method = MasterExecThread.class.getDeclaredMethod("executeComplementProcess");
method.setAccessible(true);
method.invoke(masterExecThread);
// one create save, and 15(1 to 31 step 2) for next save, and last day 31 no save
verify(processService, times(15)).saveProcessInstance(processInstance);
// one create save, and 9(1 to 20 step 2) for next save, and last day 31 no save
verify(processService, times(9)).saveProcessInstance(processInstance);
}catch (Exception e){
Assert.assertTrue(false);
Assert.fail();
}
}

21
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java

@ -17,11 +17,18 @@
package org.apache.dolphinscheduler.server.master.consumer;
import java.util.Date;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.Priority;
import org.apache.dolphinscheduler.dao.entity.*;
import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.master.dispatch.ExecutorDispatcher;
import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager;
@ -32,8 +39,10 @@ import org.apache.dolphinscheduler.server.zk.SpringZKServer;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.queue.TaskPriorityQueue;
import org.apache.dolphinscheduler.service.zk.CuratorZookeeperClient;
import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator;
import org.apache.dolphinscheduler.service.zk.ZookeeperConfig;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -42,11 +51,9 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import java.util.Date;
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(classes={DependencyConfig.class, SpringApplicationContext.class, SpringZKServer.class,
@ContextConfiguration(classes={DependencyConfig.class, SpringApplicationContext.class, SpringZKServer.class, CuratorZookeeperClient.class,
NettyExecutorManager.class, ExecutorDispatcher.class, ZookeeperRegistryCenter.class, TaskPriorityQueueConsumer.class,
ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class, MasterConfig.class})
public class TaskPriorityQueueConsumerTest {
@ -250,5 +257,11 @@ public class TaskPriorityQueueConsumerTest {
taskPriorityQueueConsumer.taskInstanceIsFinalState(1);
}
@After
public void close() {
Stopper.stop();
}
}

2
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java

@ -78,5 +78,7 @@ public class ExecutorDispatcherTest {
ExecutionContext executionContext = ExecutionContextTestUtils.getExecutionContext(port);
executorDispatcher.dispatch(executionContext);
workerRegistry.unRegistry();
}
}

1
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java

@ -74,5 +74,6 @@ public class RoundRobinHostManagerTest {
Host host = roundRobinHostManager.select(context);
Assert.assertTrue(StringUtils.isNotEmpty(host.getAddress()));
Assert.assertTrue(host.getAddress().equalsIgnoreCase(NetUtils.getHost() + ":" + workerConfig.getListenPort()));
workerRegistry.unRegistry();
}
}

4
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java

@ -43,7 +43,7 @@ public class TaskResponseServiceTest {
@Test
public void testAdd(){
TaskResponseEvent taskResponseEvent = TaskResponseEvent.newAck(ExecutionStatus.RUNNING_EXEUTION, new Date(),
TaskResponseEvent taskResponseEvent = TaskResponseEvent.newAck(ExecutionStatus.RUNNING_EXECUTION, new Date(),
"", "", "", 1);
taskResponseService.addResponse(taskResponseEvent);
Assert.assertTrue(taskResponseService.getEventQueue().size() == 1);
@ -57,7 +57,7 @@ public class TaskResponseServiceTest {
@Test
public void testStop(){
TaskResponseEvent taskResponseEvent = TaskResponseEvent.newAck(ExecutionStatus.RUNNING_EXEUTION, new Date(),
TaskResponseEvent taskResponseEvent = TaskResponseEvent.newAck(ExecutionStatus.RUNNING_EXECUTION, new Date(),
"", "", "", 1);
taskResponseService.addResponse(taskResponseEvent);
taskResponseService.stop();

1
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryTest.java

@ -58,6 +58,7 @@ public class MasterRegistryTest {
String masterNodePath = masterPath + "/" + (Constants.LOCAL_ADDRESS + ":" + masterConfig.getListenPort());
String heartbeat = zookeeperRegistryCenter.getZookeeperCachedOperator().get(masterNodePath);
Assert.assertEquals(HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH, heartbeat.split(",").length);
masterRegistry.unRegistry();
}
@Test

71
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java

@ -17,66 +17,75 @@
package org.apache.dolphinscheduler.server.master.runner;
import java.util.HashSet;
import java.util.Set;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.master.consumer.TaskPriorityQueueConsumer;
import org.apache.dolphinscheduler.server.master.dispatch.ExecutorDispatcher;
import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager;
import org.apache.dolphinscheduler.server.registry.DependencyConfig;
import org.apache.dolphinscheduler.server.registry.ZookeeperNodeManager;
import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter;
import org.apache.dolphinscheduler.server.zk.SpringZKServer;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.zk.CuratorZookeeperClient;
import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator;
import org.apache.dolphinscheduler.service.zk.ZookeeperConfig;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.springframework.context.ApplicationContext;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import java.util.HashSet;
import java.util.Set;
import com.google.common.collect.Sets;
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(classes={DependencyConfig.class, SpringApplicationContext.class, SpringZKServer.class,
NettyExecutorManager.class, ExecutorDispatcher.class, ZookeeperRegistryCenter.class, TaskPriorityQueueConsumer.class,
ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class, CuratorZookeeperClient.class})
@RunWith(MockitoJUnitRunner.Silent.class)
@PrepareForTest(MasterTaskExecThread.class)
public class MasterTaskExecThreadTest {
private MasterTaskExecThread masterTaskExecThread;
private SpringApplicationContext springApplicationContext;
private ZookeeperRegistryCenter zookeeperRegistryCenter;
@Before
public void setUp() {
ApplicationContext applicationContext = PowerMockito.mock(ApplicationContext.class);
this.springApplicationContext = new SpringApplicationContext();
springApplicationContext.setApplicationContext(applicationContext);
this.zookeeperRegistryCenter = PowerMockito.mock(ZookeeperRegistryCenter.class);
PowerMockito.when(SpringApplicationContext.getBean(ZookeeperRegistryCenter.class))
.thenReturn(this.zookeeperRegistryCenter);
this.masterTaskExecThread = new MasterTaskExecThread(null);
}
@Test
public void testExistsValidWorkerGroup1(){
ZookeeperRegistryCenter zookeeperRegistryCenter = Mockito.mock(ZookeeperRegistryCenter.class);
Mockito.when(zookeeperRegistryCenter.getWorkerGroupDirectly()).thenReturn(null);
MasterTaskExecThread masterTaskExecThread = new MasterTaskExecThread(null);
masterTaskExecThread.existsValidWorkerGroup("default");
Mockito.when(zookeeperRegistryCenter.getWorkerGroupDirectly()).thenReturn(Sets.newHashSet());
boolean b = masterTaskExecThread.existsValidWorkerGroup("default");
Assert.assertFalse(b);
}
@Test
public void testExistsValidWorkerGroup2(){
ZookeeperRegistryCenter zookeeperRegistryCenter = Mockito.mock(ZookeeperRegistryCenter.class);
Set<String> workerGorups = new HashSet<>();
workerGorups.add("test1");
workerGorups.add("test2");
Mockito.when(zookeeperRegistryCenter.getWorkerGroupDirectly()).thenReturn(workerGorups);
MasterTaskExecThread masterTaskExecThread = new MasterTaskExecThread(null);
masterTaskExecThread.existsValidWorkerGroup("default");
boolean b = masterTaskExecThread.existsValidWorkerGroup("default");
Assert.assertFalse(b);
}
@Test
public void testExistsValidWorkerGroup3(){
ZookeeperRegistryCenter zookeeperRegistryCenter = Mockito.mock(ZookeeperRegistryCenter.class);
Set<String> workerGorups = new HashSet<>();
workerGorups.add("test1");
Mockito.when(zookeeperRegistryCenter.getWorkerGroupDirectly()).thenReturn(workerGorups);
Mockito.when(zookeeperRegistryCenter.getWorkerGroupNodesDirectly("test1")).thenReturn(workerGorups);
MasterTaskExecThread masterTaskExecThread = new MasterTaskExecThread(null);
masterTaskExecThread.existsValidWorkerGroup("test1");
boolean b = masterTaskExecThread.existsValidWorkerGroup("test1");
Assert.assertTrue(b);
}
@Test
@ -84,17 +93,15 @@ public class MasterTaskExecThreadTest {
ProcessService processService = Mockito.mock(ProcessService.class);
ApplicationContext applicationContext = Mockito.mock(ApplicationContext.class);
SpringApplicationContext springApplicationContext = new SpringApplicationContext();
springApplicationContext.setApplicationContext(applicationContext);
Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService);
Mockito.when(this.springApplicationContext.getBean(ProcessService.class))
.thenReturn(processService);
TaskInstance taskInstance = getTaskInstance();
Mockito.when(processService.findTaskInstanceById(252612))
.thenReturn(taskInstance);
Mockito.when(processService.updateTaskInstance(taskInstance))
.thenReturn(true);
.thenReturn(true);
MasterTaskExecThread masterTaskExecThread = new MasterTaskExecThread(taskInstance);
masterTaskExecThread.pauseTask();

12
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManagerTest.java

@ -17,6 +17,9 @@
package org.apache.dolphinscheduler.server.registry;
import java.util.Map;
import java.util.Set;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
@ -24,6 +27,7 @@ import org.apache.dolphinscheduler.server.master.registry.MasterRegistry;
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistry;
import org.apache.dolphinscheduler.server.zk.SpringZKServer;
import org.apache.dolphinscheduler.service.zk.CuratorZookeeperClient;
import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator;
import org.apache.dolphinscheduler.service.zk.ZookeeperConfig;
import org.junit.Assert;
@ -33,16 +37,13 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import java.util.Map;
import java.util.Set;
/**
* zookeeper node manager test
*/
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(classes={DependencyConfig.class, SpringZKServer.class, MasterRegistry.class,WorkerRegistry.class,
ZookeeperRegistryCenter.class, MasterConfig.class, WorkerConfig.class,
ZookeeperCachedOperator.class, ZookeeperConfig.class, ZookeeperNodeManager.class})
ZookeeperCachedOperator.class, ZookeeperConfig.class, ZookeeperNodeManager.class, CuratorZookeeperClient.class})
public class ZookeeperNodeManagerTest {
@Autowired
@ -75,6 +76,7 @@ public class ZookeeperNodeManagerTest {
Assert.assertTrue(CollectionUtils.isNotEmpty(masterNodes));
Assert.assertEquals(1, masterNodes.size());
Assert.assertEquals(NetUtils.getHost() + ":" + masterConfig.getListenPort(), masterNodes.iterator().next());
workerRegistry.unRegistry();
}
@Test
@ -88,6 +90,7 @@ public class ZookeeperNodeManagerTest {
Map<String, Set<String>> workerGroupNodes = zookeeperNodeManager.getWorkerGroupNodes();
Assert.assertEquals(1, workerGroupNodes.size());
Assert.assertEquals("default".trim(), workerGroupNodes.keySet().iterator().next());
workerRegistry.unRegistry();
}
@Test
@ -103,5 +106,6 @@ public class ZookeeperNodeManagerTest {
Assert.assertTrue(CollectionUtils.isNotEmpty(workerNodes));
Assert.assertEquals(1, workerNodes.size());
Assert.assertEquals(NetUtils.getHost() + ":" + workerConfig.getListenPort(), workerNodes.iterator().next());
workerRegistry.unRegistry();
}
}

11
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTest.java

@ -16,7 +16,8 @@
*/
package org.apache.dolphinscheduler.server.worker.processor;
import io.netty.channel.Channel;
import java.util.Date;
import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.remote.NettyRemotingClient;
@ -40,6 +41,7 @@ import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistry;
import org.apache.dolphinscheduler.server.zk.SpringZKServer;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.zk.CuratorZookeeperClient;
import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator;
import org.apache.dolphinscheduler.service.zk.ZookeeperConfig;
import org.junit.Assert;
@ -50,8 +52,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import java.io.IOException;
import java.util.Date;
import io.netty.channel.Channel;
/**
* test task call back service
@ -73,7 +74,8 @@ import java.util.Date;
TaskResponseService.class,
TaskAckProcessor.class,
TaskResponseProcessor.class,
TaskExecuteProcessor.class})
TaskExecuteProcessor.class,
CuratorZookeeperClient.class})
public class TaskCallbackServiceTest {
@Autowired
@ -189,6 +191,7 @@ public class TaskCallbackServiceTest {
nettyRemotingServer.close();
nettyRemotingClient.close();
masterRegistry.unRegistry();
}
@Test

1
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryTest.java

@ -143,6 +143,7 @@ public class WorkerRegistryTest {
Assert.assertEquals(0, testWorkerGroupPathZkChildren.size());
Assert.assertEquals(0, defaultWorkerGroupPathZkChildren.size());
workerRegistry.unRegistry();
}
@Test

1
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogPromise.java

@ -100,6 +100,7 @@ public class LogPromise {
try {
latch.await(timeout, TimeUnit.MILLISECONDS);
} catch (InterruptedException ignore) {
Thread.currentThread().interrupt();
}
PROMISES.remove(opaque);
return this.result;

19
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java

@ -37,11 +37,8 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import java.io.File;
import java.util.*;
import java.util.stream.Collectors;
import static java.util.stream.Collectors.toSet;
import static org.apache.dolphinscheduler.common.Constants.*;
@ -54,7 +51,7 @@ public class ProcessService {
private final Logger logger = LoggerFactory.getLogger(getClass());
private final int[] stateArray = new int[]{ExecutionStatus.SUBMITTED_SUCCESS.ordinal(),
ExecutionStatus.RUNNING_EXEUTION.ordinal(),
ExecutionStatus.RUNNING_EXECUTION.ordinal(),
ExecutionStatus.READY_PAUSE.ordinal(),
ExecutionStatus.READY_STOP.ordinal()};
@ -107,7 +104,7 @@ public class ProcessService {
* @param command found command
* @return process instance
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public ProcessInstance handleCommand(Logger logger, String host, int validThreadNum, Command command) {
ProcessInstance processInstance = constructProcessInstance(command, host);
//cannot construct process instance, return null;
@ -133,7 +130,7 @@ public class ProcessService {
* @param command command
* @param message message
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public void moveToErrorCommand(Command command, String message) {
ErrorCommand errorCommand = new ErrorCommand(command, message);
this.errorCommandMapper.insert(errorCommand);
@ -454,7 +451,7 @@ public class ProcessService {
Command command,
Map<String, String> cmdParam){
ProcessInstance processInstance = new ProcessInstance(processDefinition);
processInstance.setState(ExecutionStatus.RUNNING_EXEUTION);
processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
processInstance.setRecovery(Flag.NO);
processInstance.setStartTime(new Date());
processInstance.setRunTimes(1);
@ -616,7 +613,7 @@ public class ProcessService {
}
processInstance.setHost(host);
ExecutionStatus runStatus = ExecutionStatus.RUNNING_EXEUTION;
ExecutionStatus runStatus = ExecutionStatus.RUNNING_EXECUTION;
int runTime = processInstance.getRunTimes();
switch (commandType){
case START_PROCESS:
@ -827,7 +824,7 @@ public class ProcessService {
* @param taskInstance taskInstance
* @return task instance
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public TaskInstance submitTask(TaskInstance taskInstance){
ProcessInstance processInstance = this.findProcessInstanceDetailById(taskInstance.getProcessInstanceId());
logger.info("start submit task : {}, instance id:{}, state: {}",
@ -1068,7 +1065,7 @@ public class ProcessService {
// running or killed
// the task already exists in task queue
// return state
state == ExecutionStatus.RUNNING_EXEUTION
state == ExecutionStatus.RUNNING_EXECUTION
|| state == ExecutionStatus.KILL
|| checkTaskExistsInTaskQueue(taskInstance)
){
@ -1477,7 +1474,7 @@ public class ProcessService {
* process need failover process instance
* @param processInstance processInstance
*/
@Transactional(rollbackFor = Exception.class)
@Transactional(rollbackFor = RuntimeException.class)
public void processNeedFailoverProcessInstances(ProcessInstance processInstance){
//1 update processInstance host is null
processInstance.setHost(Constants.NULL);

1
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/ZookeeperCachedOperator.java

@ -83,6 +83,7 @@ public class ZookeeperCachedOperator extends ZookeeperOperator {
try {
Thread.sleep(500);
} catch (InterruptedException ignore) {
Thread.currentThread().interrupt();
}
super.close();
}

4
dolphinscheduler-service/src/main/resources/logback-zookeeper.xml

@ -30,7 +30,7 @@
</appender>
<appender name="LOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${log.base}/dolphinscheduler-alert.log</file>
<file>${log.base}/dolphinscheduler-zookeeper.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>${log.base}/dolphinscheduler-alert.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<maxHistory>20</maxHistory>
@ -49,4 +49,4 @@
<appender-ref ref="LOGFILE"/>
</root>
</configuration>
</configuration>

1
dolphinscheduler-ui/build/config.js

@ -116,7 +116,6 @@ const pages = glob.sync(['*/!(_*).html'], { cwd: viewDir }).map(p => {
minify: minifierConfig
})
})
const baseConfig = {
entry: jsEntry,
output: {

6
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js

@ -153,7 +153,7 @@ const tasksState = {
icoUnicode: 'ans-icon-dot-circle',
isSpin: false
},
RUNNING_EXEUTION: {
RUNNING_EXECUTION: {
id: 1,
desc: `${i18n.$t('Executing')}`,
color: '#0097e0',
@ -243,6 +243,10 @@ const tasksType = {
desc: 'SHELL',
color: '#646464'
},
WATERDROP: {
desc: 'WATERDROP',
color: '#646465'
},
SUB_PROCESS: {
desc: 'SUB_PROCESS',
color: '#0097e0'

3
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss

@ -74,6 +74,9 @@
.icos-SHELL {
background: url("../img/toolbar_SHELL.png") no-repeat 50% 50%;
}
.icos-WATERDROP {
background: url("../img/toolbar_WATERDROP.png") no-repeat 50% 50%;
}
.icos-SUB_PROCESS {
background: url("../img/toolbar_SUB_PROCESS.png") no-repeat 50% 50%;
}

12
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue

@ -38,7 +38,7 @@
type="text"
v-model="name"
:disabled="isDetails"
:placeholder="$t('Please enter name(required)')"
:placeholder="$t('Please enter name (required)')"
maxlength="100"
@on-blur="_verifName()"
autocomplete="off">
@ -162,6 +162,14 @@
ref="SHELL"
:backfill-item="backfillItem">
</m-shell>
<!-- waterdrop node -->
<m-waterdrop
v-if="taskType === 'WATERDROP'"
@on-params="_onParams"
@on-cache-params="_onCacheParams"
ref="WATERDROP"
:backfill-item="backfillItem">
</m-waterdrop>
<!-- sub_process node -->
<m-sub-process
v-if="taskType === 'SUB_PROCESS'"
@ -274,6 +282,7 @@
import mSql from './tasks/sql'
import i18n from '@/module/i18n'
import mShell from './tasks/shell'
import mWaterdrop from './tasks/waterdrop'
import mSpark from './tasks/spark'
import mFlink from './tasks/flink'
import mPython from './tasks/python'
@ -701,6 +710,7 @@
components: {
mMr,
mShell,
mWaterdrop,
mSubProcess,
mProcedure,
mSql,

6
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/nodeStatus.vue

@ -28,7 +28,7 @@
<template v-if="isInstance">
<span class="instance-state">
<em class="iconfont ans-icon-success-solid" :class="'icon-' + el.state" v-if="el.state === 'SUCCESS'" data-toggle="tooltip" data-container="body" :title="$t('success')"></em>
<em class="iconfont ans-icon-clock" :class="'icon-' + el.state" v-if="el.state === 'RUNNING_EXEUTION'" data-toggle="tooltip" data-container="body" :title="$t('waiting')"></em>
<em class="iconfont ans-icon-clock" :class="'icon-' + el.state" v-if="el.state === 'RUNNING_EXECUTION'" data-toggle="tooltip" data-container="body" :title="$t('waiting')"></em>
<em class="iconfont ans-icon-fail-solid" :class="'icon-' + el.state" v-if="el.state === 'FAILURE'" data-toggle="tooltip" data-container="body" :title="$t('failed')"></em>
</span>
</template>
@ -79,7 +79,7 @@
// btn loading
this.isLoading = true
this.$emit('dependItemListEvent', _.concat(this.dependItemList, this._rtNewParams()))
// remove tooltip
this._removeTip()
},
@ -223,4 +223,4 @@
}
}
}
</style>
</style>

467
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/waterdrop.vue

@ -0,0 +1,467 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
<template>
<div class="shell-model">
<!--deploy mode-->
<div class="list-box-4p">
<div class="clearfix list">
<span class="sp1">{{$t('Deploy Mode')}}</span>
<span class="sp2">
<x-radio-group v-model="deployMode">
<x-radio :label="'client'" :disabled="isDetails"></x-radio>
<x-radio :label="'cluster'" :disabled="isDetails"></x-radio>
<x-radio :label="'local'" :disabled="isDetails"></x-radio>
</x-radio-group>
</span>
<span class="sp1 sp3">{{$t('Queue')}}</span>
<span class="sp4">
<x-input
:disabled="isDetails"
type="input"
v-model="queue"
:placeholder="$t('Please enter queue value')"
style="width: 60%;"
autocomplete="off">
</x-input>
</span>
</div>
</div>
<!--master-->
<div class="list-box-4p" v-if="deployMode !== 'local'">
<div class="clearfix list">
<span class="sp1">{{$t('Master')}}</span>
<span class="sp4">
<x-select
style="width: 130px;"
v-model="master"
:disabled="isDetails">
<x-option
v-for="city in masterType"
:key="city.code"
:value="city.code"
:label="city.code">
</x-option>
</x-select>
</span>
<span v-if="masterUrlState">
<x-input
:disabled="isDetails"
type="input"
v-model="masterUrl"
:placeholder="$t('Please Enter Url')"
style="width: 60%;"
autocomplete="off">
</x-input>
</span>
</div>
</div>
<!--config file-->
<m-list-box>
<div slot="text">{{$t('Resources')}}</div>
<div slot="content">
<treeselect v-model="resourceList" :disable-branch-nodes="true" :multiple="true" :options="options" :normalizer="normalizer" :disabled="isDetails" :value-consists-of="valueConsistsOf" :placeholder="$t('Please select resources')">
<div slot="value-label" slot-scope="{ node }">{{ node.raw.fullName }}</div>
</treeselect>
</div>
</m-list-box>
<!--custom parameters-->
<m-list-box>
<div slot="text">{{$t('Custom Parameters')}}</div>
<div slot="content">
<m-local-params
ref="refLocalParams"
@on-local-params="_onLocalParams"
:udp-list="localParams"
:hide="false">
</m-local-params>
</div>
</m-list-box>
</div>
</template>
<script>
import _ from 'lodash'
import i18n from '@/module/i18n'
import mListBox from './_source/listBox'
import mScriptBox from './_source/scriptBox'
import mResources from './_source/resources'
import mLocalParams from './_source/localParams'
import disabledState from '@/module/mixin/disabledState'
import Treeselect from '@riophae/vue-treeselect'
import '@riophae/vue-treeselect/dist/vue-treeselect.css'
export default {
name: 'waterdrop',
data () {
return {
valueConsistsOf: 'LEAF_PRIORITY',
// script
rawScript: '',
// waterdrop script
baseScript: 'sh ${WATERDROP_HOME}/bin/start-waterdrop.sh',
// resourceNameVal
resourceNameVal : [],
// Custom parameter
localParams: [],
// resource(list)
resourceList: [],
// Deployment method
deployMode: 'client',
// Deployment master
queue: 'default',
// Deployment master
master: 'yarn',
// Spark version(LIst)
masterType: [{ code: 'yarn' }, { code: 'local' }, { code: 'spark://' }, { code: 'mesos://' }],
// Deployment masterUrl state
masterUrlState:false,
// Deployment masterUrl
masterUrl: '',
// Cache ResourceList
cacheResourceList: [],
// define options
options: [],
normalizer(node) {
return {
label: node.name
}
},
allNoResources: [],
noRes: []
}
},
mixins: [disabledState],
props: {
backfillItem: Object
},
methods: {
/**
* return localParams
*/
_onLocalParams (a) {
this.localParams = a
},
/**
* return resourceList
*
*/
_onResourcesData (a) {
this.resourceList = a
},
/**
* cache resourceList
*/
_onCacheResourcesData (a) {
this.cacheResourceList = a
},
/**
* verification
*/
_verification () {
// localParams Subcomponent verification
if (!this.$refs.refLocalParams._verifProp()) {
return false
}
// noRes
if (this.noRes.length>0) {
this.$message.warning(`${i18n.$t('Please delete all non-existent resources')}`)
return false
}
// noRes
if (!this.resourceNameVal.resourceList) {
this.$message.warning(`${i18n.$t('Please select the waterdrop resources')}`)
return false
}
if (this.resourceNameVal.resourceList && this.resourceNameVal.resourceList.length==0) {
this.$message.warning(`${i18n.$t('Please select the waterdrop resources')}`)
return false
}
// Process resourcelist
let dataProcessing= _.map(this.resourceList, v => {
return {
id: v
}
})
//verify deploy mode
let deployMode = this.deployMode
let master = this.master
let masterUrl = this.masterUrl
if(this.deployMode == 'local'){
master = 'local'
masterUrl = ''
deployMode = 'client'
}
// get local params
let locparams = ''
this.localParams.forEach(v=>{
locparams = locparams + ' --variable ' + v.prop + '=' + v.value
}
)
// get waterdrop script
let tureScript = ''
this.resourceNameVal.resourceList.forEach(v=>{
tureScript = tureScript + this.baseScript +
' --master '+ master + masterUrl +
' --deploy-mode '+ deployMode +
' --queue '+ this.queue +
' --config ' + v.res +
locparams + ' \n'
})
// storage
this.$emit('on-params', {
resourceList: dataProcessing,
localParams: this.localParams,
rawScript: tureScript,
})
return true
},
diGuiTree(item) { // Recursive convenience tree structure
item.forEach(item => {
item.children === '' || item.children === undefined || item.children === null || item.children.length === 0?
this.operationTree(item) : this.diGuiTree(item.children);
})
},
operationTree(item) {
if(item.dirctory) {
item.isDisabled =true
}
delete item.children
},
searchTree(element, id) {
// id
if (element.id == id) {
return element;
} else if (element.children != null) {
var i;
var result = null;
for (i = 0; result == null && i < element.children.length; i++) {
result = this.searchTree(element.children[i], id);
}
return result;
}
return null;
},
dataProcess(backResource) {
let isResourceId = []
let resourceIdArr = []
if(this.resourceList.length>0) {
this.resourceList.forEach(v=>{
this.options.forEach(v1=>{
if(this.searchTree(v1,v)) {
isResourceId.push(this.searchTree(v1,v))
}
})
})
resourceIdArr = isResourceId.map(item=>{
return item.id
})
Array.prototype.diff = function(a) {
return this.filter(function(i) {return a.indexOf(i) < 0;});
};
let diffSet = this.resourceList.diff(resourceIdArr);
let optionsCmp = []
if(diffSet.length>0) {
diffSet.forEach(item=>{
backResource.forEach(item1=>{
if(item==item1.id || item==item1.res) {
optionsCmp.push(item1)
}
})
})
}
let noResources = [{
id: -1,
name: $t('Unauthorized or deleted resources'),
fullName: '/'+$t('Unauthorized or deleted resources'),
children: []
}]
if(optionsCmp.length>0) {
this.allNoResources = optionsCmp
optionsCmp = optionsCmp.map(item=>{
return {id: item.id,name: item.name,fullName: item.res}
})
optionsCmp.forEach(item=>{
item.isNew = true
})
noResources[0].children = optionsCmp
this.options = this.options.concat(noResources)
}
}
}
},
watch: {
//Watch the cacheParams
cacheParams (val) {
this.resourceNameVal = val
this.$emit('on-cache-params', val);
},
"master": {
handler(code) {
if(code == 'spark://'){
this.masterUrlState = true;
}else if(code == 'mesos://'){
this.masterUrlState = true;
}else{
this.masterUrlState = false;
this.masterUrl = ''
}
}
},
},
computed: {
cacheParams () {
let isResourceId = []
let resourceIdArr = []
if(this.resourceList.length>0) {
this.resourceList.forEach(v=>{
this.options.forEach(v1=>{
if(this.searchTree(v1,v)) {
isResourceId.push(this.searchTree(v1,v))
}
})
})
resourceIdArr = isResourceId.map(item=>{
return {id: item.id,name: item.name,res: item.fullName}
})
}
let result = []
resourceIdArr.forEach(item=>{
this.allNoResources.forEach(item1=>{
if(item.id==item1.id) {
// resultBool = true
result.push(item1)
}
})
})
this.noRes = result
return {
resourceList: resourceIdArr,
localParams: this.localParams,
deployMode: this.deployMode,
master: this.master,
masterUrl: this.masterUrl,
queue:this.queue,
}
}
},
created () {
let item = this.store.state.dag.resourcesListS
this.diGuiTree(item)
this.options = item
let o = this.backfillItem
// Non-null objects represent backfill
if (!_.isEmpty(o)) {
this.master = o.params.master || 'yarn'
this.deployMode = o.params.deployMode || 'client'
this.masterUrl = o.params.masterUrl || ''
this.queue = o.params.queue || 'default'
this.rawScript = o.params.rawScript || ''
// backfill resourceList
let backResource = o.params.resourceList || []
let resourceList = o.params.resourceList || []
if (resourceList.length) {
_.map(resourceList, v => {
if(!v.id) {
this.store.dispatch('dag/getResourceId',{
type: 'FILE',
fullName: '/'+v.res
}).then(res => {
this.resourceList.push(res.id)
this.dataProcess(backResource)
}).catch(e => {
this.resourceList.push(v.res)
this.dataProcess(backResource)
})
} else {
this.resourceList.push(v.id)
this.dataProcess(backResource)
}
})
}
// backfill localParams
let localParams = o.params.localParams || []
if (localParams.length) {
this.localParams = localParams
}
}
},
mounted () {
},
destroyed () {
},
components: { mLocalParams, mListBox, mResources, mScriptBox, Treeselect }
}
</script>
<style lang="scss" rel="stylesheet/scss" scope>
.scriptModal {
.ans-modal-box-content-wrapper {
width: 90%;
.ans-modal-box-close {
right: -12px;
top: -16px;
color: #fff;
}
}
}
.ans-modal-box-close {
z-index: 100;
}
.ans-modal-box-max {
position: absolute;
right: -12px;
top: -16px;
}
.vue-treeselect--disabled {
.vue-treeselect__control {
background-color: #ecf3f8;
.vue-treeselect__single-value {
color: #6d859e;
}
}
}
.list-box-4p {
.list {
margin-bottom: 14px;
.sp1 {
float: left;
width: 112px;
text-align: right;
margin-right: 10px;
font-size: 14px;
color: #777;
display: inline-block;
padding-top: 6px;
}
.sp2 {
float: left;
margin-right: 4px;
padding-top: 6px;
}
.sp3 {
width: 90px;
}
.sp4 {
float: left;
margin-right: 4px;
}
}
}
</style>

17
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue

@ -26,7 +26,7 @@
type="text"
v-model="name"
:disabled="router.history.current.name === 'projects-instance-details'"
:placeholder="$t('Please enter name(required)')">
:placeholder="$t('Please enter name (required)')">
</x-input>
</div>
@ -169,17 +169,12 @@
this.$emit('onUdp')
}
// Edit => direct storage
if (this.store.state.dag.name) {
// verify that the name exists
this.store.dispatch('dag/verifDAGName', this.name).then(res => {
_verif()
} else {
// New First verify that the name exists
this.store.dispatch('dag/verifDAGName', this.name).then(res => {
_verif()
}).catch(e => {
this.$message.error(e.msg || '')
})
}
}).catch(e => {
this.$message.error(e.msg || '')
})
},
/**
* Close the popup

BIN
dolphinscheduler-ui/src/js/conf/home/pages/dag/img/toolbar_WATERDROP.png

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.6 KiB

4
dolphinscheduler-ui/src/js/conf/home/pages/projects/index.vue

@ -17,7 +17,9 @@
<template>
<div class="main-layout-box" :class="!isProjectsList ? '' : 'no'">
<m-secondary-menu :type="'projects'" v-if="!isProjectsList"></m-secondary-menu>
<router-view></router-view>
<transition name="fade">
<router-view></router-view>
</transition>
</div>
</template>
<script>

2
dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/_source/instanceConditions/common.js

@ -28,7 +28,7 @@ const stateType = [
code: 'SUBMITTED_SUCCESS',
label: `${i18n.$t('Submitted successfully')}`
}, {
code: 'RUNNING_EXEUTION',
code: 'RUNNING_EXECUTION',
label: `${i18n.$t('Running')}`
}, {
code: 'READY_PAUSE',

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save